--- /dev/null
+demo/dependencies/*
+demo/demo
--- /dev/null
+A copy of the GDAX order book for the product given during construction, exposed as two maps, one for bids and one for offers, each mapping price levels to order quantities, continually updated in real time via the `level2` channel of the Websocket feed of the GDAX API.
+
+Spawns a separate thread to receive updates from the GDAX WebSocket Feed and process them into the maps.
+
+To ensure high performance, implemented using concurrent data structures from libcds. The price->quantity maps are instances of cds::container::SkipListMap, whose doc says it is lock-free.
--- /dev/null
+#!/bin/bash
+cd demo/
+make
--- /dev/null
+INCDIRS = -I ..
+INCDIRS += -I ./dependencies/libcds-2.3.2
+INCDIRS += -I ./dependencies/rapidjson-1.1.0/include
+INCDIRS += -I ./dependencies/websocketpp-0.7.0
+
+CDSLIBDIR = ./dependencies/libcds-2.3.2/build-release/bin
+
+LIBS = -lboost_system # needed for boost::asio, used by websocketpp
+ # according to https://groups.google.com/forum/#!topic/websocketpp/Ou30S05FKoM
+ # one could use standalone Asio from think-async.com
+LIBS += -lpthread
+LIBS += -lssl # needed for websocketpp's TLS support (req'd by GDAX)
+LIBS += -lcrypto # needed for websocketpp's TLS support (req'd by GDAX)
+LIBS += -lcds
+
+test: demo
+ LD_LIBRARY_PATH=$(CDSLIBDIR) ./demo
+
+debug: demo
+ LD_LIBRARY_PATH=$(CDSLIBDIR) gdb ./demo
+
+demo: demo.cpp ../gdax-orderbook.hpp \
+ | dependencies/libcds-2.3.2/build-release/bin/libcds.so \
+ dependencies/rapidjson-1.1.0 \
+ dependencies/websocketpp-0.7.0
+ g++ demo.cpp -std=c++11 -o demo $(INCDIRS) -L $(CDSLIBDIR) $(LIBS)
+
+dependencies/libcds-2.3.2/build-release/bin/libcds.so: \
+ | dependencies/libcds-2.3.2
+ cd dependencies/libcds-2.3.2 ; if [ ! -d build-release ]; then mkdir build-release; fi
+ cd dependencies/libcds-2.3.2/build-release ; cmake -DCMAKE\_BUILD_TYPE=RELEASE ../
+ cd dependencies/libcds-2.3.2/build-release ; make -j4
+
+dependencies/libcds-2.3.2: | dependencies/libcds-2.3.2.tar.gz
+ tar xzvf dependencies/libcds-2.3.2.tar.gz -C dependencies
+
+dependencies/libcds-2.3.2.tar.gz: dependencies
+ wget -O $@ https://github.com/khizmax/libcds/archive/v2.3.2.tar.gz
+
+
+dependencies/rapidjson-1.1.0: | dependencies/rapidjson-1.1.0.tar.gz
+ tar xzvf dependencies/rapidjson-1.1.0.tar.gz -C dependencies
+
+dependencies/rapidjson-1.1.0.tar.gz: dependencies
+ wget -O $@ https://github.com/Tencent/rapidjson/archive/v1.1.0.tar.gz
+
+
+dependencies/websocketpp-0.7.0: | dependencies/websocketpp-0.7.0.tar.gz
+ tar xzvf dependencies/websocketpp-0.7.0.tar.gz -C dependencies
+
+dependencies/websocketpp-0.7.0.tar.gz: dependencies
+ wget -O $@ https://github.com/zaphoyd/websocketpp/archive/0.7.0.tar.gz
+
+
+dependencies:
+ mkdir dependencies
+
+clean:
+ rm -rf demo dependencies
--- /dev/null
+Sample output:
+
+```
+[2018-06-14 13:19:21] [connect] Successful connection
+[2018-06-14 13:19:23] [connect] WebSocket Connection 104.16.107.31:443 v-2 "WebSocket++/0.7.0" / 101
+current best bid: Ξ0.01086 @ $513.98/Ξ ; current best offer: Ξ148.995 @ $513.99/Ξ
+waiting 5 seconds for the market to shift
+current best bid: Ξ0.01086 @ $513.98/Ξ ; current best offer: Ξ0.337 @ $513.71/Ξ
+running for 90 seconds, with 5 threads constantly iterating over the whole order book.
+histogram of times to iterate over the whole book:
+ 0- 4 ms: *
+ 5- 9 ms: *********************************************************************
+ 10- 14 ms: ***********************************
+ 15- 19 ms: *
+ 20- 24 ms:
+ 25- 29 ms:
+ 30- 34 ms:
+ 35- 39 ms:
+ 40- 44 ms:
+ 45- 49 ms:
+ 50- 54 ms:
+ 55- 59 ms:
+ 60- 64 ms:
+ 65- 69 ms:
+ 70- 74 ms:
+ 75- 79 ms:
+ 80- 84 ms:
+ 85- 89 ms:
+ 90- 94 ms:
+ 95- 99 ms:
+100-104 ms:
+105-109 ms:
+110-114 ms:
+115-119 ms:
+120-124 ms:
+125-129 ms:
+130-134 ms:
+135-139 ms:
+140-144 ms:
+145-149 ms:
+```
--- /dev/null
+#include <chrono>
+#include <iostream>
+#include <thread>
+
+#include "gdax-orderbook.hpp"
+
+void printBestBidAndOffer(GDAXOrderBook & book)
+{
+ std::cout << "current best bid: Ξ" << book.bids.begin()->second << " @ $"
+ << book.bids.begin()->first/100.0 << "/Ξ ; ";
+ std::cout << "current best offer: Ξ" << book.offers.begin()->second << " @ $"
+ << book.offers.begin()->first/100.0 << "/Ξ"
+ << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+ GDAXOrderBook book("ETH-USD");
+
+ printBestBidAndOffer(book);
+
+ size_t secondsToSleep = 5;
+ std::cout << "waiting " << secondsToSleep << " seconds for the market to "
+ "shift" << std::endl;
+ std::this_thread::sleep_for(std::chrono::seconds(secondsToSleep));
+
+ printBestBidAndOffer(book);
+
+ size_t histogram[30];
+ for ( auto bucket : histogram ) bucket = 0;
+
+ size_t numThreads = 5;
+ secondsToSleep = 90;
+ std::cout << "running for " << secondsToSleep << " seconds, with " <<
+ numThreads << " threads constantly iterating over the whole order "
+ "book." << std::endl;
+ bool keepIterating = true;
+ {
+ std::vector<std::future<void>> futures;
+ for (size_t i = 0 ; i < numThreads ; ++i)
+ {
+ futures.emplace_back(
+ std::async(std::launch::async,
+ [&book, &keepIterating, &histogram] ()
+ {
+ GDAXOrderBook::ensureThreadAttached();
+
+ GDAXOrderBook::bids_map_t::iterator bidIter;
+ GDAXOrderBook::offers_map_t::iterator offerIter;
+ std::chrono::steady_clock::time_point start, finish;
+
+ while(keepIterating == true)
+ {
+ start = std::chrono::steady_clock::now();
+
+ bidIter = book.bids.begin();
+ while(bidIter != book.bids.end()) { ++bidIter; }
+
+ offerIter = book.offers.begin();
+ while(offerIter != book.offers.end()) { ++offerIter; }
+
+ finish = std::chrono::steady_clock::now();
+
+ histogram[
+ static_cast<size_t>(
+ std::chrono::duration<double, std::milli>(
+ std::chrono::steady_clock::now() - start
+ ).count()/5
+ )
+ ]++;
+ }
+ }));
+ }
+
+ std::this_thread::sleep_for(std::chrono::seconds(secondsToSleep));
+
+ keepIterating = false;
+ }
+
+ // find the largest histogram bucket so we can determine the scale factor
+ // for all of the buckets
+ double scaleFactor =
+ [&histogram]() {
+ size_t countOfBiggestBucket = 0;
+ for ( size_t & i : histogram )
+ {
+ countOfBiggestBucket = std::max(i, countOfBiggestBucket);
+ }
+ return countOfBiggestBucket;
+ }()/68; // 80 column display, minus chars used for row headers, =68
+ std::cout << "histogram of times to iterate over the whole book:" << std::endl;
+ for ( int i=0 ; i < sizeof(histogram)/sizeof(histogram[0]) ; ++i )
+ {
+ std::cout
+ << std::right << std::setw(3) << std::setfill(' ') << i*5
+ << "-"
+ << std::right << std::setw(3) << std::setfill(' ') << (i+1)*5-1
+ << " ms: ";
+ for ( int j=0 ; j<histogram[i]/scaleFactor ; ++j )
+ {
+ std::cout << "*";
+ }
+ std::cout << std::endl;
+ }
+}
--- /dev/null
+#ifndef GDAX_ORDERBOOK_HPP
+#define GDAX_ORDERBOOK_HPP
+
+#include <cstring>
+#include <future>
+#include <iostream>
+#include <string>
+
+#include <cds/container/skip_list_map_hp.h>
+#include <cds/gc/hp.h>
+#include <cds/init.h>
+
+#include <rapidjson/document.h>
+
+#include <websocketpp/client.hpp>
+#include <websocketpp/concurrency/none.hpp>
+#include <websocketpp/config/asio_client.hpp>
+
+/**
+ * A copy of the GDAX order book for the currency pair product given during
+ * construction, exposed as two maps, one for bids and one for offers, each
+ * mapping price levels to order quantities, continually updated in real time
+ * via the `level2` channel of the Websocket feed of the GDAX API.
+ *
+ * Spawns a separate thread to receive updates from the GDAX WebSocket Feed and
+ * process them into the maps.
+ *
+ * To ensure high performance, implemented using concurrent data structures
+ * from libcds. The price->quantity maps are instances of
+ * cds::container::SkipListMap, whose doc says it is lock-free.
+ */
+class GDAXOrderBook {
+private:
+ // libcds requires paired Initialize/Terminate calls
+ struct CDSInitializer {
+ CDSInitializer() { cds::Initialize(); }
+ ~CDSInitializer() { cds::Terminate(); }
+ } m_cdsInitializer;
+
+ // a libcds garbage collector is required for our map structures
+ cds::gc::HP m_cdsGarbageCollector;
+
+public:
+ /**
+ * libcds requires each and every thread to first "attach" itself to the
+ * lib before using any data structures. This method is used internally,
+ * and is called during construction on behalf of the constructing thread,
+ * and should be called by any additional client threads that access the
+ * price->quanitity maps.
+ */
+ static void ensureThreadAttached()
+ {
+ if (cds::threading::Manager::isThreadAttached() == false)
+ cds::threading::Manager::attachThread();
+ }
+
+ GDAXOrderBook(std::string const& product = "BTC-USD")
+ : m_cdsGarbageCollector(67*2),
+ // per SkipListMap doc, 67 hazard pointers per instance
+ m_threadTerminator(
+ std::async(
+ std::launch::async,
+ &GDAXOrderBook::handleUpdates,
+ this,
+ product))
+ {
+ ensureThreadAttached();
+ m_bookInitialized.get_future().wait();
+ }
+
+ using Price = unsigned int; // cents
+ using Size = double;
+ using offers_map_t = cds::container::SkipListMap<cds::gc::HP, Price, Size>;
+ using bids_map_t =
+ cds::container::SkipListMap<
+ cds::gc::HP,
+ Price,
+ Size,
+ // reverse map ordering so best (highest) bid is at begin()
+ typename cds::container::skip_list::make_traits<
+ cds::opt::less<std::greater<Price>>>::type>;
+ // *map_t::get(Price) returns an std::pair<Price, Size>*
+ bids_map_t bids;
+ offers_map_t offers;
+
+ ~GDAXOrderBook() { m_client.stop(); }
+
+private:
+ struct websocketppConfig
+ : public websocketpp::config::asio_tls_client
+ {
+ typedef websocketpp::concurrency::none concurrency_type;
+ // we only have one thread using the WebSocket
+ };
+ using websocketclient_t = websocketpp::client<websocketppConfig>;
+ websocketclient_t m_client;
+
+ std::promise<void> m_bookInitialized; // to signal constructor to finish
+
+ std::future<void> m_threadTerminator; // for graceful thread destruction
+
+ /**
+ * Initiates WebSocket connection, subscribes to order book updates for the
+ * given product, installs a message handler which will receive updates
+ * and process them into the maps, and starts the asio event loop.
+ */
+ void handleUpdates(std::string const& product)
+ {
+ ensureThreadAttached();
+
+ rapidjson::Document json;
+
+ try {
+ m_client.clear_access_channels(websocketpp::log::alevel::all);
+ m_client.set_access_channels(
+ websocketpp::log::alevel::connect |
+ websocketpp::log::alevel::disconnect);
+
+ m_client.clear_error_channels(websocketpp::log::elevel::all);
+ m_client.set_error_channels(
+ websocketpp::log::elevel::info |
+ websocketpp::log::elevel::warn |
+ websocketpp::log::elevel::rerror |
+ websocketpp::log::elevel::fatal);
+
+ m_client.init_asio();
+
+ m_client.set_tls_init_handler(
+ [](websocketpp::connection_hdl)
+ {
+ websocketpp::lib::shared_ptr<boost::asio::ssl::context>
+ context = websocketpp::lib::make_shared<
+ boost::asio::ssl::context>(
+ boost::asio::ssl::context::tlsv1);
+
+ try {
+ context->set_options(
+ boost::asio::ssl::context::default_workarounds |
+ boost::asio::ssl::context::no_sslv2 |
+ boost::asio::ssl::context::no_sslv3 |
+ boost::asio::ssl::context::single_dh_use);
+ } catch (std::exception& e) {
+ std::cerr << "set_tls_init_handler() failed to set"
+ " context options: " << e.what() << std::endl;
+ }
+ return context;
+ });
+
+ m_client.set_open_handler(
+ [this, &product](websocketpp::connection_hdl handle)
+ {
+ // subscribe to updates to product's order book
+ websocketpp::lib::error_code errorCode;
+ this->m_client.send(handle,
+ "{"
+ "\"type\": \"subscribe\","
+ "\"product_ids\": [" "\""+product+"\"" "],"
+ "\"channels\": [" "\"level2\"" "]"
+ "}", websocketpp::frame::opcode::text, errorCode);
+ if (errorCode) {
+ std::cerr << "error sending subscription: " +
+ errorCode.message() << std::endl;
+ }
+ });
+
+ m_client.set_message_handler(
+ [this, &json] (websocketpp::connection_hdl,
+ websocketppConfig::message_type::ptr msg)
+ {
+ json.Parse(msg->get_payload().c_str());
+ const char *const type = json["type"].GetString();
+ if ( strcmp(type, "l2update") == 0 )
+ {
+ processUpdates(json, bids, offers);
+ }
+ else if ( strcmp(type, "snapshot") == 0 )
+ {
+ processSnapshot(json, bids, offers, m_bookInitialized);
+ }
+ });
+
+ websocketpp::lib::error_code errorCode;
+ auto connection =
+ m_client.get_connection("wss://ws-feed.gdax.com", errorCode);
+ if (errorCode) {
+ std::cerr << "failed websocketclient_t::get_connection(): " <<
+ errorCode.message() << std::endl;
+ }
+
+ m_client.connect(connection);
+
+ m_client.run();
+ } catch (websocketpp::exception const & e) {
+ std::cerr << "handleUpdates() failed: " << e.what() << std::endl;
+ }
+ }
+
+ /**
+ * Simply delegates snapshot processing to a helper function (different
+ * template instantiations of the same function, one for each type of map
+ * (bid, offer)), and signals when the snapshot has been processed.
+ */
+ static void processSnapshot(
+ rapidjson::Document & json,
+ bids_map_t & bids,
+ offers_map_t & offers,
+ std::promise<void> & finished)
+ {
+ processSnapshotHalf(json, "bids", bids);
+ processSnapshotHalf(json, "asks", offers);
+ finished.set_value();
+ }
+
+ /**
+ * Helper to permit code re-use on either type of map (bids or offers).
+ * Traverses already-parsed json document and inserts initial-price
+ * snapshots for entire half (bids or offers) of the order book.
+ */
+ template<typename map_t>
+ static void processSnapshotHalf(
+ rapidjson::Document const& json,
+ const char *const bidsOrOffers,
+ map_t & map)
+ {
+ for (auto j = 0 ; j < json[bidsOrOffers].Size() ; ++j)
+ {
+ Price price = std::stod(json[bidsOrOffers][j][0].GetString())*100;
+ Size size = std::stod(json[bidsOrOffers][j][1].GetString());
+
+ map.insert(price, size);
+ }
+ }
+
+ /**
+ * Traverses already-parsed json document, and, assuming it's a "l2update"
+ * document, updates price->quantity maps based on the order book changes
+ * that have occurred.
+ */
+ static void processUpdates(
+ rapidjson::Document & json,
+ bids_map_t & bids,
+ offers_map_t & offers)
+ {
+ for (auto i = 0 ; i < json["changes"].Size() ; ++i)
+ {
+ const char* buyOrSell = json["changes"][i][0].GetString(),
+ * price = json["changes"][i][1].GetString(),
+ * size = json["changes"][i][2].GetString();
+
+ if ( strcmp(buyOrSell, "buy") == 0 )
+ {
+ updateMap(price, size, bids);
+ }
+ else
+ {
+ updateMap(price, size, offers);
+ }
+ }
+ }
+
+ /**
+ * Helper to permit code re-use on either type of map (bids or offers).
+ * Simply updates a single map entry with the specified price/size.
+ */
+ template<typename map_t>
+ static void updateMap(
+ const char *const price,
+ const char *const size,
+ map_t & map)
+ {
+ if (std::stod(size) == 0) { map.erase(std::stod(price)); }
+ else
+ {
+ map.update(
+ std::stod(price)*100,
+ [size](bool & bNew,
+ std::pair<const Price, Size> & pair)
+ {
+ pair.second = std::stod(size);
+ });
+ }
+ }
+};
+
+#endif // GDAX_ORDERBOOK_HPP
--- /dev/null
+#!/bin/bash
+cd demo/
+LD_LIBRARY_PATH=./dependencies/libcds-2.3.2/build-release/bin ./demo
+
+
--- /dev/null
+tags
+doc/*
+**/*.o
+src/*.so
+binaries/*
+**/*.gcda
+**/*.gcno
+**/*.tap
+data/*
+src/test/key_list*
+src/test/mb_test*
+tmp/*
--- /dev/null
+sudo: required
+
+# Placeholder installs
+before_install:
+ - sudo apt-get install libao-dev
+ - sudo apt-get install libcurl4-openssl-dev
+ - sudo apt-get install -y cmake libgtest-dev valgrind gcovr
+ - sudo apt-get install -y exuberant-ctags doxygen graphviz
+ - sudo pip install cpp-coveralls
+ - sudo pip install -U cpp-coveralls
+ - cd /var/tmp
+ - sudo wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz
+ - sudo tar xf release-1.7.0.tar.gz
+ - cd googletest-release-1.7.0
+ - sudo cmake -DBUILD_SHARED_LIBS=ON .
+ - sudo make
+ - sudo cp -a include/gtest /usr/include
+ - sudo cp -a libgtest_main.so libgtest.so /usr/lib/
+ - cd /var/tmp
+ - sudo wget https://raw.githubusercontent.com/kinow/gtest-tap-listener/master/src/tap.h
+ - sudo cp /var/tmp/tap.h /usr/include/gtest/tap.h
+ - which valgrind
+ - cd $TRAVIS_BUILD_DIR
+
+# Enable C++ support
+language: cpp
+
+# Compiler selection
+compiler:
+ - gcc
+
+# Skip install phase
+install: true
+
+# Build steps
+script:
+ - make clean build
+ - sudo make install
+ - make unit-test
+
+# Run examples
+after_success:
+ - cd $TRAVIS_BUILD_DIR/examples
+ - make
+ - export LD_LIBRARY_PATH=/usr/local/lib/
+ - ./mb_insert_test
+ - ./mb_iterator_test
+ - ./mb_longest_prefix_test
+ - ./mb_lookup_test
+ - ./mb_multi_proc_test
+ - ./mb_remove_test
+ - cd $TRAVIS_BUILD_DIR
+ - coveralls --root src/ -E "./src/unittest/*" -E "./src/test/*"
--- /dev/null
+Corporate Contributors
+======================
+Copyright (c) 2018 Cisco Systems Inc.
+
+Initial Author
+==============
+Changxue Deng
+
+Individual Contributors
+=======================
+
+Changxue Deng
+Satyajit Padalkar
+Bahula Gupta
+Shridhar Bhalerao
+Amer Tahir
+Ted Bedwell
+Andy Mullican
+
+Credits
+=======
+Hao Wang
--- /dev/null
+# This is a multistage image build. Run the following command from the
+# top directory of your repo clone.
+# docker build --rm -t chxdeng/mabain:latest .
+FROM alpine:latest as builder
+
+# Lets build Mabain in the alpine environment
+WORKDIR /build
+COPY . /build/
+RUN apk update && apk add --no-cache g++ musl-dev make \
+ readline-dev ncurses-dev
+RUN make distclean build install
+# Lets run the unit-test build
+RUN apk add --no-cache gtest gtest-dev openssl-dev gcovr && wget https://github.com/kinow/gtest-tap-listener/raw/master/src/tap.h -O /usr/include/gtest/tap.h
+RUN make unit-test
+
+
+# Now lets build the runtime
+FROM alpine:latest
+LABEL maintainer="Ted Bedwell tebedwel@cisco.com"
+ENV MABAIN_INSTALL_DIR=/usr/local
+RUN apk update && apk add --no-cache musl libstdc++ readline ncurses && mkdir /data && \
+ mkdir -p $MABAIN_INSTALL_DIR/include/mabain
+COPY --from=builder $MABAIN_INSTALL_DIR/include/mabain/* \
+ $MABAIN_INSTALL_DIR/include/mabain/
+
+COPY --from=builder $MABAIN_INSTALL_DIR/lib/*mabain*.* \
+ $MABAIN_INSTALL_DIR/lib/
+
+COPY --from=builder $MABAIN_INSTALL_DIR/bin/mbc \
+ $MABAIN_INSTALL_DIR/bin/
+
+VOLUME /data
--- /dev/null
+The GNU General Public License, Version 2, June 1991 (GPLv2)
+============================================================
+
+> Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+> 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+Everyone is permitted to copy and distribute verbatim copies of this license
+document, but changing it is not allowed.
+
+
+Preamble
+--------
+
+The licenses for most software are designed to take away your freedom to share
+and change it. By contrast, the GNU General Public License is intended to
+guarantee your freedom to share and change free software--to make sure the
+software is free for all its users. This General Public License applies to most
+of the Free Software Foundation's software and to any other program whose
+authors commit to using it. (Some other Free Software Foundation software is
+covered by the GNU Lesser General Public License instead.) You can apply it to
+your programs, too.
+
+When we speak of free software, we are referring to freedom, not price. Our
+General Public Licenses are designed to make sure that you have the freedom to
+distribute copies of free software (and charge for this service if you wish),
+that you receive source code or can get it if you want it, that you can change
+the software or use pieces of it in new free programs; and that you know you can
+do these things.
+
+To protect your rights, we need to make restrictions that forbid anyone to deny
+you these rights or to ask you to surrender the rights. These restrictions
+translate to certain responsibilities for you if you distribute copies of the
+software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether gratis or for a
+fee, you must give the recipients all the rights that you have. You must make
+sure that they, too, receive or can get the source code. And you must show them
+these terms so they know their rights.
+
+We protect your rights with two steps: (1) copyright the software, and (2) offer
+you this license which gives you legal permission to copy, distribute and/or
+modify the software.
+
+Also, for each author's protection and ours, we want to make certain that
+everyone understands that there is no warranty for this free software. If the
+software is modified by someone else and passed on, we want its recipients to
+know that what they have is not the original, so that any problems introduced by
+others will not reflect on the original authors' reputations.
+
+Finally, any free program is threatened constantly by software patents. We wish
+to avoid the danger that redistributors of a free program will individually
+obtain patent licenses, in effect making the program proprietary. To prevent
+this, we have made it clear that any patent must be licensed for everyone's free
+use or not licensed at all.
+
+The precise terms and conditions for copying, distribution and modification
+follow.
+
+
+Terms And Conditions For Copying, Distribution And Modification
+---------------------------------------------------------------
+
+**0.** This License applies to any program or other work which contains a notice
+placed by the copyright holder saying it may be distributed under the terms of
+this General Public License. The "Program", below, refers to any such program or
+work, and a "work based on the Program" means either the Program or any
+derivative work under copyright law: that is to say, a work containing the
+Program or a portion of it, either verbatim or with modifications and/or
+translated into another language. (Hereinafter, translation is included without
+limitation in the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not covered by
+this License; they are outside its scope. The act of running the Program is not
+restricted, and the output from the Program is covered only if its contents
+constitute a work based on the Program (independent of having been made by
+running the Program). Whether that is true depends on what the Program does.
+
+**1.** You may copy and distribute verbatim copies of the Program's source code
+as you receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice and
+disclaimer of warranty; keep intact all the notices that refer to this License
+and to the absence of any warranty; and give any other recipients of the Program
+a copy of this License along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and you may at
+your option offer warranty protection in exchange for a fee.
+
+**2.** You may modify your copy or copies of the Program or any portion of it,
+thus forming a work based on the Program, and copy and distribute such
+modifications or work under the terms of Section 1 above, provided that you also
+meet all of these conditions:
+
+* **a)** You must cause the modified files to carry prominent notices stating
+ that you changed the files and the date of any change.
+
+* **b)** You must cause any work that you distribute or publish, that in whole
+ or in part contains or is derived from the Program or any part thereof, to
+ be licensed as a whole at no charge to all third parties under the terms of
+ this License.
+
+* **c)** If the modified program normally reads commands interactively when
+ run, you must cause it, when started running for such interactive use in the
+ most ordinary way, to print or display an announcement including an
+ appropriate copyright notice and a notice that there is no warranty (or
+ else, saying that you provide a warranty) and that users may redistribute
+ the program under these conditions, and telling the user how to view a copy
+ of this License. (Exception: if the Program itself is interactive but does
+ not normally print such an announcement, your work based on the Program is
+ not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If identifiable
+sections of that work are not derived from the Program, and can be reasonably
+considered independent and separate works in themselves, then this License, and
+its terms, do not apply to those sections when you distribute them as separate
+works. But when you distribute the same sections as part of a whole which is a
+work based on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the entire whole,
+and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest your
+rights to work written entirely by you; rather, the intent is to exercise the
+right to control the distribution of derivative or collective works based on the
+Program.
+
+In addition, mere aggregation of another work not based on the Program with the
+Program (or with a work based on the Program) on a volume of a storage or
+distribution medium does not bring the other work under the scope of this
+License.
+
+**3.** You may copy and distribute the Program (or a work based on it, under
+Section 2) in object code or executable form under the terms of Sections 1 and 2
+above provided that you also do one of the following:
+
+* **a)** Accompany it with the complete corresponding machine-readable source
+ code, which must be distributed under the terms of Sections 1 and 2 above on
+ a medium customarily used for software interchange; or,
+
+* **b)** Accompany it with a written offer, valid for at least three years, to
+ give any third party, for a charge no more than your cost of physically
+ performing source distribution, a complete machine-readable copy of the
+ corresponding source code, to be distributed under the terms of Sections 1
+ and 2 above on a medium customarily used for software interchange; or,
+
+* **c)** Accompany it with the information you received as to the offer to
+ distribute corresponding source code. (This alternative is allowed only for
+ noncommercial distribution and only if you received the program in object
+ code or executable form with such an offer, in accord with Subsection b
+ above.)
+
+The source code for a work means the preferred form of the work for making
+modifications to it. For an executable work, complete source code means all the
+source code for all modules it contains, plus any associated interface
+definition files, plus the scripts used to control compilation and installation
+of the executable. However, as a special exception, the source code distributed
+need not include anything that is normally distributed (in either source or
+binary form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component itself
+accompanies the executable.
+
+If distribution of executable or object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the source code
+from the same place counts as distribution of the source code, even though third
+parties are not compelled to copy the source along with the object code.
+
+**4.** You may not copy, modify, sublicense, or distribute the Program except as
+expressly provided under this License. Any attempt otherwise to copy, modify,
+sublicense or distribute the Program is void, and will automatically terminate
+your rights under this License. However, parties who have received copies, or
+rights, from you under this License will not have their licenses terminated so
+long as such parties remain in full compliance.
+
+**5.** You are not required to accept this License, since you have not signed
+it. However, nothing else grants you permission to modify or distribute the
+Program or its derivative works. These actions are prohibited by law if you do
+not accept this License. Therefore, by modifying or distributing the Program (or
+any work based on the Program), you indicate your acceptance of this License to
+do so, and all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+**6.** Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the original
+licensor to copy, distribute or modify the Program subject to these terms and
+conditions. You may not impose any further restrictions on the recipients'
+exercise of the rights granted herein. You are not responsible for enforcing
+compliance by third parties to this License.
+
+**7.** If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues), conditions
+are imposed on you (whether by court order, agreement or otherwise) that
+contradict the conditions of this License, they do not excuse you from the
+conditions of this License. If you cannot distribute so as to satisfy
+simultaneously your obligations under this License and any other pertinent
+obligations, then as a consequence you may not distribute the Program at all.
+For example, if a patent license would not permit royalty-free redistribution of
+the Program by all those who receive copies directly or indirectly through you,
+then the only way you could satisfy both it and this License would be to refrain
+entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply and the
+section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any patents or
+other property right claims or to contest validity of any such claims; this
+section has the sole purpose of protecting the integrity of the free software
+distribution system, which is implemented by public license practices. Many
+people have made generous contributions to the wide range of software
+distributed through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing to
+distribute software through any other system and a licensee cannot impose that
+choice.
+
+This section is intended to make thoroughly clear what is believed to be a
+consequence of the rest of this License.
+
+**8.** If the distribution and/or use of the Program is restricted in certain
+countries either by patents or by copyrighted interfaces, the original copyright
+holder who places the Program under this License may add an explicit
+geographical distribution limitation excluding those countries, so that
+distribution is permitted only in or among countries not thus excluded. In such
+case, this License incorporates the limitation as if written in the body of this
+License.
+
+**9.** The Free Software Foundation may publish revised and/or new versions of
+the General Public License from time to time. Such new versions will be similar
+in spirit to the present version, but may differ in detail to address new
+problems or concerns.
+
+Each version is given a distinguishing version number. If the Program specifies
+a version number of this License which applies to it and "any later version",
+you have the option of following the terms and conditions either of that version
+or of any later version published by the Free Software Foundation. If the
+Program does not specify a version number of this License, you may choose any
+version ever published by the Free Software Foundation.
+
+**10.** If you wish to incorporate parts of the Program into other free programs
+whose distribution conditions are different, write to the author to ask for
+permission. For software which is copyrighted by the Free Software Foundation,
+write to the Free Software Foundation; we sometimes make exceptions for this.
+Our decision will be guided by the two goals of preserving the free status of
+all derivatives of our free software and of promoting the sharing and reuse of
+software generally.
+
+
+No Warranty
+-----------
+
+**11.** BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR
+THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE
+STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM
+"AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+**12.** IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR
+INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA
+BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER
+OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
--- /dev/null
+#mabain Makefile
+
+ifndef MABAIN_INSTALL_DIR
+MABAIN_INSTALL_DIR=/usr/local
+endif
+
+build:
+ make -C src build
+ make -C binaries build
+
+install: build
+ echo "mabain install directory: $(MABAIN_INSTALL_DIR)"
+ mkdir -p $(MABAIN_INSTALL_DIR)/include/mabain
+ cp src/db.h $(MABAIN_INSTALL_DIR)/include/mabain
+ cp src/mb_data.h $(MABAIN_INSTALL_DIR)/include/mabain
+ cp src/mabain_consts.h $(MABAIN_INSTALL_DIR)/include/mabain
+ cp src/lock.h $(MABAIN_INSTALL_DIR)/include/mabain
+ cp src/error.h $(MABAIN_INSTALL_DIR)/include/mabain
+ cp src/integer_4b_5b.h $(MABAIN_INSTALL_DIR)/include/mabain
+
+ mkdir -p $(MABAIN_INSTALL_DIR)/lib
+ cp src/libmabain.so $(MABAIN_INSTALL_DIR)/lib
+
+ mkdir -p $(MABAIN_INSTALL_DIR)/bin
+ cp binaries/mbc $(MABAIN_INSTALL_DIR)/bin
+
+uninstall:
+ rm -rf $(MABAIN_INSTALL_DIR)/include/mabain
+ rm -f $(MABAIN_INSTALL_DIR)/lib/libmabain.so
+ rm -f $(MABAIN_INSTALL_DIR)/bin/mbc
+
+clean:
+ -make -C src clean
+ -make -C binaries clean
+ -make -C examples clean
+
+distclean: clean
+ -rm -rf doc/*
+ -rm -f tags
+
+index:
+ -ctags -R *
+ -doxygen doxygen.conf
+ -echo "Generating documentation..."
+ -echo "Use following index file : "
+ -readlink -f doc/html/index.html
+ -echo "Done with doxygen"
+
+unit-test: build
+ make -C src unit-test
+
+.PHONY: docker
+docker:
+ docker build --rm -t chxdeng/mabain:latest .
+ docker rmi $$(docker images -q -f dangling=true)
--- /dev/null
+# mabain
+
+[](https://travis-ci.org/chxdeng/mabain)
+
+[](https://coveralls.io/github/chxdeng/mabain)
+
+## Mabain: a key-value store library
+
+Mabain is a light-weighted C++ library that provides a generic key-value store
+based on a radix tree implementation. It supports multi-thread and multi-process
+concurrency *(see Caveats below)*. Mabain can be used for exact and common
+prefix key match. Please see the examples in the `examples` directory.
+
+### Shared Memory and Memcap
+
+Mabain stores all data on disk. However, you can specify how much data can be
+mapped to shared memory using the memcap option. The memcap can be specified for
+the key space and the value space. For example see the `-km` and `-dm` options to
+the Mabain command line client below.
+
+### Multi-Thread/Multi-Process Concurrency
+
+Full multi-thread/multi-process concurrency is supported. Concurrent insertion
+and queries are supported internally using a lock-free mechanism. Programs using
+the library DO NOT need to perform any locking on concurrent insertion and
+lookup in the multi-thread or multi-process reader/writer scenario. *(see
+Caveats below)*
+
+### Multi-Thread/Multi-Process Insertion/Update
+
+When using mabain only one writer is allowed. However, all reader threads/processes can
+perform DB insertion/update using asynchronous queue in shared memory if the writer
+has ASYNC_WRITER_MODE specified in the option when opening the DB. In this case, a separate
+thread in the writer process is started for internal DB operations. All DB writing
+operations are performed sequentially in this thread.
+
+## Build and Install Mabain Library
+
+We now have two different build options. First is the traditional "Native
+Build", and *experimentally* a Docker build based on the very slim [Alpine
+Linux](https://wiki.alpinelinux.org/wiki/Docker) and
+[MSCL](https://www.musl-libc.org/) instead of glibc++.
+
+### Native Build & Install
+
+#### Build/Runtime Dependencies
+1. [GNU ncurses](https://www.gnu.org/software/ncurses/)
+2. [GNU readline](https://www.gnu.org/software/ncurses/)
+3. [GLIBC](https://www.gnu.org/software/libc/)
+4. [g++ Compiler that supports C++11](https://gcc.gnu.org/)
+
+You should be able to build Mabain on any modern linux machines. To build and
+install, ensure you meet the dependency requirements called out above run
+following command in the top level of your git clone:
+
+```
+make build
+```
+
+#### (Optional) - Run the unit tests
+
+If you would like to run the unit tests you will need some additional dependencies:
+
+##### Unit Test dependencies
+1. [Google Test](https://github.com/google/googletest)
+2. [OpenSSL](https://www.openssl.org/)
+3. [gcovr & gcov](https://github.com/gcovr/gcovr)
+4. [Gtest Tap Listener](https://github.com/kinow/gtest-tap-listener/)
+
+The command :
+
+```
+make unit-test
+```
+### Install Mabain
+
+By default, Mabain will atttempt to install into `/usr/local/`. If you would
+like to override this behavior define the `MABAIN_INSTALL_DIR` variable before
+running the following command:
+
+```
+make install
+```
+
+### Docker Build
+
+You can skip the previous build step if you would like to just build for a
+docker environment. From the top level of your repository clone just run:
+
+ make docker
+
+This will kick off a two stage build process. We will spin up a build container
+with all the build time dependencies. Once the build is complete, a thin
+container image is build with just the run-time dependencies.
+
+This image is intended to serve as a base layer for an application that would
+like to leverage Mabain. However, we can certainly validate that the docker
+container built properly.
+
+```
+docker run -it --rm -v$(pwd)/data:/data chxdeng/mabain /usr/local/bin/mbc -d /data -w
+```
+
+This should drop you into the mbc command-line client. It should look something
+like this:
+
+```
+mabain 1.1.0 shell
+database directory: /data/
+>>
+```
+
+We expose a VOLUME from the image `/data` where you should bind mount persistent
+storage. In the above command I have a `data` sub-directory in my current
+directory that I mount to `/data`. This way, my database is created in the host
+filesystem and persists between instantiation of containers based on this image.
+
+## Mabain Command-Line Client
+
+The command-line client source is in the `binaries` directory. It is installed
+into $MABAIN_INSTALL_DIR/bin via `make install`.
+
+```
+Usage: mbc -d mabain-directory [-im index-memcap] [-dm data-memcap] [-w] [-e query] [-s script-file]
+
+-d mabain databse directory
+-im index memcap
+-dm data memcap
+-w running in writer mode
+-e run query on command line
+-s run queries in a file
+```
+
+## Examples
+
+Please follow these steps to run the examples
+
+1. Build and install mabain library
+```
+make clean build install
+```
+
+2. Build examples
+```
+cd ./examples
+make
+```
+3. Create the temporary database directory for examples
+```
+mkdir ./tmp_dir
+./mb_insert_test
+```
+
+## Caveats
+
+* Only one writer is allowed. However, multiple readers can be running
+ concurrently.
+* Mabain DB handle is not thread-safe. Each thread must have open its own DB
+ instance when Using in multi-thread context.
+* The longest key supported is 256 bytes.
+* The value/data size can not be bigger than 32767 bytes.
+* Using Mabain on network storage (NAS, SAN, NFS, SMB, etc..) has not been
+ tested. Your mileage may vary
+* Please use `-D__BIG__ENDIAN__` in compilation flags when using Mabain on big
+ endian machines.
+
+## License
+
+Copyright (C) 2017 Cisco Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License, version 2,
+as published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+[](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
--- /dev/null
+#!/bin/bash
+make build
+sudo make install
+sudo make clean
+sudo make build
+sudo make install
+cd ./examples
+make
+mkdir ./tmp_dir
+
--- /dev/null
+db_comp_kc
+db_comp_ldb
+db_comp_lmdb
+db_comp_mb
--- /dev/null
+ifndef MABAIN_INSTALL_DIR
+MABAIN_INSTALL_DIR=/usr/local
+endif
+
+all: db_comp_ldb db_comp_lmdb db_comp_mb db_comp_kc
+
+CPP=g++
+
+CFLAGS = -O2 -g -std=c++11 -Wall -I/usr/local/include -I$(MABAIN_INSTALL_DIR)/include
+
+LDFLAGS = -lpthread -lcrypto -L$(MABAIN_INSTALL_DIR)/lib -lleveldb -lkyotocabinet -lmabain -llmdb
+
+db_comp_ldb: db_comp.cpp
+ $(CPP) $(CFLAGS) -DLEVEL_DB -o db_comp_ldb db_comp.cpp $(LDFLAGS)
+db_comp_lmdb: db_comp.cpp
+ $(CPP) $(CFLAGS) -DLMDB -o db_comp_lmdb db_comp.cpp $(LDFLAGS)
+db_comp_mb: db_comp.cpp
+ $(CPP) $(CFLAGS) -DMABAIN -o db_comp_mb db_comp.cpp $(LDFLAGS)
+db_comp_kc: db_comp.cpp
+ $(CPP) $(CFLAGS) -DKYOTO_CABINET -o db_comp_kc db_comp.cpp $(LDFLAGS)
+
+build: all
+
+clean:
+ - rm db_comp_ldb db_comp_kc db_comp_lmdb db_comp_mb
--- /dev/null
+#include <iostream>
+#include <string>
+#include <cstring>
+#include <fstream>
+#include <sys/time.h>
+#include <assert.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <openssl/sha.h>
+
+#ifdef LEVEL_DB
+#include <leveldb/db.h>
+#elif KYOTO_CABINET
+#include <kcpolydb.h>
+#include <kchashdb.h>
+#elif LMDB
+extern "C" {
+#include <lmdb.h>
+}
+#elif MABAIN
+#include <mabain/db.h>
+#endif
+
+#ifdef LEVEL_DB
+leveldb::DB *db = NULL;
+#elif KYOTO_CABINET
+kyotocabinet::HashDB *db = NULL;
+#elif LMDB
+MDB_env *env = NULL;
+MDB_dbi db;
+MDB_txn *txn = NULL;
+#elif MABAIN
+mabain::DB *db = NULL;
+#endif
+
+#define ONE_MILLION 1000000
+
+static const char *db_dir = "/var/tmp/db_test/";
+static int num_kv = 1 * ONE_MILLION;
+static int n_reader = 7;
+static int key_type = 0;
+static bool sync_on_write = false;
+static unsigned long long memcap = 1024ULL*1024*1024;
+
+static void get_sha256_str(int key, char *sha256_str)
+{
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ SHA256_CTX sha256;
+ SHA256_Init(&sha256);
+ SHA256_Update(&sha256, (unsigned char*)&key, 4);
+ SHA256_Final(hash, &sha256);
+ int i = 0;
+ for(i = 0; i < SHA256_DIGEST_LENGTH; i++)
+ {
+ sprintf(sha256_str + (i * 2), "%02x", hash[i]);
+ }
+ sha256_str[64] = 0;
+}
+
+static void get_sha1_str(int key, char *sha1_str)
+{
+ unsigned char hash[SHA_DIGEST_LENGTH];
+ SHA_CTX sha1;
+ SHA1_Init(&sha1);
+ SHA1_Update(&sha1, (unsigned char*)&key, 4);
+ SHA1_Final(hash, &sha1);
+ int i = 0;
+ for(i = 0; i < SHA_DIGEST_LENGTH; i++)
+ {
+ sprintf(sha1_str + (i * 2), "%02x", hash[i]);
+ }
+ sha1_str[32] = 0;
+}
+
+static void print_cpu_info()
+{
+ std::ifstream cpu_info("/proc/cpuinfo", std::fstream::in);
+ if(!cpu_info.is_open()) {
+ return;
+ }
+
+ std::string line;
+ std::string model_name;
+ std::string vendor_id;
+ std::string cpu_freq;
+ int n_processor = 0;
+ while (std::getline(cpu_info, line))
+ {
+ if(line.find("model name") != std::string::npos)
+ {
+ model_name = line;
+ n_processor++;
+ }
+ else if(line.find("vendor_id") != std::string::npos)
+ {
+ vendor_id = line;
+ }
+ else if(line.find("cpu MHz") != std::string::npos)
+ {
+ cpu_freq = line;
+ }
+ }
+ cpu_info.close();
+
+ std::cout << "number of CPUs\t: " << n_processor << "\n";
+ std::cout << vendor_id << "\n";
+ std::cout << model_name << "\n";
+ std::cout << cpu_freq << "\n";
+}
+
+static void InitTestDir()
+{
+ std::string cmd;
+
+ cmd = std::string("mkdir -p ") + db_dir;
+ if(system(cmd.c_str()) != 0) {
+ }
+
+#ifdef LEVEL_DB
+ std::cout << "===== using leveldb for testing\n";
+ std::string db_dir_tmp = std::string(db_dir) + "/leveldb/";
+#elif KYOTO_CABINET
+ std::cout << "===== using kyotocabinet for testing\n";
+ std::string db_dir_tmp = std::string(db_dir) + "/kyotocabinet/";
+#elif LMDB
+ std::cout << "===== using lmdb for testing\n";
+ std::string db_dir_tmp = std::string(db_dir) + "/lmdb/";
+#elif MABAIN
+ std::cout << "===== using mabain for testing\n";
+ std::string db_dir_tmp = std::string(db_dir) + "/mabain/";
+#endif
+ cmd = std::string("mkdir -p ") + db_dir_tmp;
+ if(system(cmd.c_str()) != 0) {
+ }
+ cmd = std::string("rm -rf ") + db_dir_tmp + "*";
+ if(system(cmd.c_str()) != 0) {
+ }
+
+}
+
+static void InitDB(bool writer_mode = true)
+{
+#ifdef LEVEL_DB
+ std::string db_dir_tmp = std::string(db_dir) + "/leveldb/";
+ leveldb::Options options;
+ options.create_if_missing = true;
+ leveldb::Status status = leveldb::DB::Open(options, db_dir_tmp, &db);
+ assert(status.ok());
+#elif KYOTO_CABINET
+ std::string db_file = std::string(db_dir) + "/kyotocabinet/dbbench_hashDB.kch";
+ db = new kyotocabinet::HashDB();
+ int open_options = kyotocabinet::PolyDB::OWRITER |
+ kyotocabinet::PolyDB::OCREATE;
+ db->tune_map(memcap);
+ if(!db->open(db_file, open_options)) {
+ fprintf(stderr, "open error: %s\n", db->error().name());
+ }
+#elif LMDB
+ std::string db_dir_tmp = std::string(db_dir) + "/lmdb";
+ mdb_env_create(&env);
+ mdb_env_set_mapsize(env, memcap);
+ mdb_env_open(env, db_dir_tmp.c_str(), 0, 0664);
+ mdb_txn_begin(env, NULL, 0, &txn);
+ mdb_open(txn, NULL, 0, &db);
+ mdb_txn_commit(txn);
+#elif MABAIN
+ std::string db_dir_tmp = std::string(db_dir) + "/mabain/";
+ int options = mabain::CONSTS::WriterOptions() | mabain::CONSTS::ASYNC_WRITER_MODE;
+ if(sync_on_write) {
+ options |= mabain::CONSTS::SYNC_ON_WRITE;
+ }
+ if(!writer_mode)
+ options = mabain::CONSTS::ReaderOptions();
+ db = new mabain::DB(db_dir_tmp.c_str(), options, (unsigned long long)(0.6666667*memcap),
+ (unsigned long long)(0.3333333*memcap));
+ assert(db->is_open());
+#endif
+}
+
+static void Add(int n)
+{
+ timeval start, stop;
+ char kv[65];
+
+ gettimeofday(&start,NULL);
+#if LMDB
+ if(!sync_on_write)
+ mdb_txn_begin(env, NULL, 0, &txn);
+#endif
+ for(int i = 0; i < n; i++) {
+ std::string key, val;
+ if(key_type == 0) {
+ key = std::to_string(i);
+ val = std::to_string(i);
+ } else {
+ if(key_type == 1) {
+ get_sha1_str(i, kv);
+ } else {
+ get_sha256_str(i, kv);
+ }
+ key = kv;
+ val = kv;
+ }
+
+#ifdef LEVEL_DB
+ leveldb::WriteOptions opts = leveldb::WriteOptions();
+ opts.sync = sync_on_write;
+ db->Put(opts, key, val);
+#elif KYOTO_CABINET
+ db->set(key.c_str(), val.c_str());
+#elif LMDB
+ MDB_val lmdb_key, lmdb_val;
+ lmdb_key.mv_size = key.size();
+ lmdb_key.mv_data = (void*) key.data();
+ lmdb_val.mv_size = val.size();
+ lmdb_val.mv_data = (void*) val.data();
+ MDB_cursor *mc;
+ if(sync_on_write) {
+ mdb_txn_begin(env, NULL, 0, &txn);
+ mdb_cursor_open(txn, db, &mc);
+ mdb_cursor_put(mc, &lmdb_key, &lmdb_val, 0);
+ mdb_cursor_close(mc);
+ mdb_txn_commit(txn);
+ } else {
+ mdb_cursor_open(txn, db, &mc);
+ mdb_cursor_put(mc, &lmdb_key, &lmdb_val, 0);
+ mdb_cursor_close(mc);
+ }
+#elif MABAIN
+ db->Add(key, val);
+#endif
+
+ if((i+1) % ONE_MILLION == 0) {
+ std::cout << "inserted: " << (i+1) << " key-value pairs\n";
+ }
+ }
+#if LMDB
+ if(!sync_on_write)
+ mdb_txn_commit(txn);
+#endif
+
+ gettimeofday(&stop,NULL);
+
+ uint64_t timediff = (stop.tv_sec - start.tv_sec)*1000000 + (stop.tv_usec - start.tv_usec);
+ std::cout << "===== " << timediff*1.0/n << " micro seconds per insertion\n";
+}
+
+static void Lookup(int n)
+{
+ timeval start, stop;
+ int nfound = 0;
+ char kv[65];
+#ifdef LMDB
+ MDB_val lmdb_key, lmdb_value;
+ MDB_cursor *cursor;
+ mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+ mdb_cursor_open(txn, db, &cursor);
+#endif
+
+ gettimeofday(&start,NULL);
+ for(int i = 0; i < n; i++) {
+ std::string key;
+ if(key_type == 0) {
+ key = std::to_string(i);
+ } else {
+ if(key_type == 1) {
+ get_sha1_str(i, kv);
+ } else {
+ get_sha256_str(i, kv);
+ }
+ key = kv;
+ }
+
+#ifdef LEVEL_DB
+ std::string value;
+ leveldb::Status s = db->Get(leveldb::ReadOptions(), key, &value);
+ if(s.ok()) nfound++;
+#elif KYOTO_CABINET
+ std::string value;
+ if(db->get(key, &value)) nfound++;
+#elif LMDB
+ lmdb_key.mv_data = (void*) key.data();
+ lmdb_key.mv_size = key.size();
+ if(mdb_cursor_get(cursor, &lmdb_key, &lmdb_value, MDB_SET) == 0)
+ nfound++;
+ //std::cout<<key<<":"<<std::string((char*)lmdb_value.mv_data, lmdb_value.mv_size)<<"\n";
+#elif MABAIN
+ mabain::MBData mbd;
+ int rval = db->Find(key, mbd);
+ //std::cout<<key<<":"<<std::string((char*)mbd.buff, mbd.data_len)<<"\n";
+ if(rval == 0) nfound++;
+#endif
+
+ if((i+1) % ONE_MILLION == 0) {
+ std::cout << "looked up: " << (i+1) << " keys\n";
+ }
+ }
+
+#ifdef LMDB
+ mdb_cursor_close(cursor);
+ mdb_txn_abort(txn);
+#endif
+ gettimeofday(&stop,NULL);
+
+ std::cout << "found " << nfound << " key-value pairs\n";
+ uint64_t timediff = (stop.tv_sec - start.tv_sec)*1000000 + (stop.tv_usec - start.tv_usec);
+ std::cout << "===== " << timediff*1.0/n << " micro seconds per lookup\n";
+}
+
+static void Delete(int n)
+{
+ timeval start, stop;
+ int nfound = 0;
+ char kv[65];
+
+#if LMDB
+ if(!sync_on_write)
+ mdb_txn_begin(env, NULL, 0, &txn);
+#endif
+
+ gettimeofday(&start,NULL);
+ for(int i = 0; i < n; i++) {
+ std::string key;
+ if(key_type == 0) {
+ key = std::to_string(i);
+ } else {
+ if(key_type == 1) {
+ get_sha1_str(i, kv);
+ } else {
+ get_sha256_str(i, kv);
+ }
+ key = kv;
+ }
+#ifdef LEVEL_DB
+ leveldb::WriteOptions opts = leveldb::WriteOptions();
+ opts.sync = sync_on_write;
+ leveldb::Status s = db->Delete(opts, key);
+ if(s.ok()) nfound++;
+#elif KYOTO_CABINET
+ std::string value;
+ if(db->remove(key)) nfound++;
+#elif LMDB
+ MDB_val lmdb_key;
+ lmdb_key.mv_size = key.size();
+ lmdb_key.mv_data = (void*) key.data();
+ if(sync_on_write) {
+ mdb_txn_begin(env, NULL, 0, &txn);
+ if(mdb_del(txn, db, &lmdb_key, NULL) == 0) nfound++;
+ mdb_txn_commit(txn);
+ } else {
+ if(mdb_del(txn, db, &lmdb_key, NULL) == 0) nfound++;
+ }
+#elif MABAIN
+ int rval = db->Remove(key);
+ if(rval == 0) nfound++;
+#endif
+
+ if((i+1) % ONE_MILLION == 0) {
+ std::cout << "deleted: " << (i+1) << " keys\n";
+ }
+ }
+
+#if LMDB
+ if(!sync_on_write)
+ mdb_txn_commit(txn);
+#endif
+ gettimeofday(&stop,NULL);
+
+ std::cout << "deleted " << nfound << " key-value pairs\n";
+ uint64_t timediff = (stop.tv_sec - start.tv_sec)*1000000 + (stop.tv_usec - start.tv_usec);
+ std::cout << "===== " << timediff*1.0/n << " micro seconds per deletion\n";
+}
+
+static void *Writer(void *arg)
+{
+ int num = *((int *) arg);
+ char kv[65];
+ int tid = static_cast<int>(syscall(SYS_gettid));
+
+ std::cout << "\n[writer : " << tid << "] started" << std::endl;
+ for(int i = 0; i < num; i++) {
+ std::string key, val;
+ if(key_type == 0) {
+ key = std::to_string(i);
+ val = std::to_string(i);
+ } else {
+ if(key_type == 1) {
+ get_sha1_str(i, kv);
+ } else {
+ get_sha256_str(i, kv);
+ }
+ key = kv;
+ val = kv;
+ }
+
+#ifdef LEVEL_DB
+ leveldb::WriteOptions opts = leveldb::WriteOptions();
+ opts.sync = sync_on_write;
+ db->Put(opts, key, val);
+#elif MABAIN
+ db->Add(key.c_str(), key.length(), val.c_str(), val.length());
+#ifdef DEFRAG
+ if((i+1) % (2*ONE_MILLION)== 0) {
+ std::cout<<"\nRC SCHEDULED " << std::endl;
+ db->CollectResource(1, 1);
+ }
+#endif
+#endif
+
+ if((i+1) % ONE_MILLION == 0) {
+ std::cout << "\nwriter inserted " << (i+1) << std::endl;
+ }
+ }
+
+ return NULL;
+}
+static void *Reader(void *arg)
+{
+ int num = *((int *) arg);
+ int i = 0;
+ int tid = static_cast<int>(syscall(SYS_gettid));
+ char kv[65];
+
+#if MABAIN
+ std::string db_dir_tmp = std::string(db_dir) + "/mabain/";
+ mabain::DB *db_r = new mabain::DB(db_dir_tmp.c_str(), mabain::CONSTS::ReaderOptions(),
+ (unsigned long long)(0.6666667*memcap),
+ (unsigned long long)(0.3333333*memcap));
+ assert(db_r->is_open());
+#endif
+
+ std::cout << "\n[reader : " << tid << "] started" << std::endl;
+ while(i < num) {
+ std::string key;
+ bool found = false;
+
+ if(key_type == 0) {
+ key = std::to_string(i);
+ } else {
+ if(key_type == 1) {
+ get_sha1_str(i, kv);
+ } else {
+ get_sha256_str(i, kv);
+ }
+ key = kv;
+ }
+
+ std::string value;
+#ifdef LEVEL_DB
+ leveldb::Status s = db->Get(leveldb::ReadOptions(), key, &value);
+ if(s.ok()) {
+ found = true;
+ }
+#elif MABAIN
+ mabain::MBData mbd;
+ int rval = db_r->Find(key, mbd);
+ if(rval == 0) {
+ value = std::string((const char *)mbd.buff, mbd.data_len);
+ found = true;
+ }
+#endif
+ if(found) {
+ if(key.compare(value) != 0) {
+ std::cout << "\nVALUE NOT MATCH for key:" << key << ":" << value << "\n";
+ abort();
+ }
+
+ i++;
+ if((i+1) % ONE_MILLION == 0) {
+ std::cout << "\n[reader : " << tid << "] found " << (i+1) << "\n";
+ }
+ }
+ }
+
+#if MABAIN
+ db_r->Close();
+ delete db_r;
+#endif
+ return NULL;
+}
+static void ConcurrencyTest(int num, int n_r)
+{
+#ifdef KYOTO_CABINET
+ std::cout << "===== concurrency test ignored for kyotocabinet\n";
+ return;
+#elif LMDB
+ std::cout << "===== concurrency test ignored for lmdb\n";
+ return;
+#endif
+
+ pthread_t wid;
+ timeval start, stop;
+
+ gettimeofday(&start,NULL);
+
+ // Start the writer
+ if(pthread_create(&wid, NULL, Writer, &num) != 0) {
+ std::cerr << "failed to create writer thread\n";
+ abort();
+ }
+
+ // start the readers
+ pthread_t rid[256];
+ assert(n_r <= 256);
+ for(int i = 0; i < n_r; i++) {
+ if(pthread_create(&rid[i], NULL, Reader, &num) != 0) {
+ std::cerr << "failed to create writer thread\n";
+ abort();
+ }
+ }
+
+ for(int i = 0; i < n_r; i++) {
+ pthread_join(rid[i], NULL);
+ }
+ pthread_join(wid, NULL);
+
+ gettimeofday(&stop,NULL);
+
+ uint64_t timediff = (stop.tv_sec - start.tv_sec)*1000000 + (stop.tv_usec - start.tv_usec);
+ std::cout << "===== " << timediff*1.0/num_kv << " micro seconds per concurrent insertion/lookup\n";
+}
+
+static void DestroyDB()
+{
+#ifdef LEVEL_DB
+ delete db;
+ db = NULL;
+#elif KYOTO_CABINET
+ db->close();
+ delete db;
+ db = NULL;
+#elif LMDB
+ mdb_close(env, db);
+ mdb_env_close(env);
+#elif MABAIN
+ db->Close();
+ delete db;
+ db = NULL;
+#endif
+}
+
+static void RemoveDB()
+{
+ std::string cmd = std::string("rm -rf ") + db_dir;
+#ifdef LEVEL_DB
+ cmd += "leveldb/*";
+#elif KYOTO_CABINET
+ cmd += "kyotocabinet/*";
+#elif LMDB
+ cmd += "lmdb/*";
+#elif MABAIN
+ cmd += "mabain/*";
+ mabain::DB::ClearResources(std::string("/var/tmp/"));
+#endif
+
+ if(system(cmd.c_str()) != 0) {
+ }
+}
+
+int main(int argc, char *argv[])
+{
+#ifdef MABAIN
+ mabain::DB::SetLogFile("/var/tmp/mabain_test/mabain.log");
+ // mabain::DB::SetLogLevel(2);
+#endif
+ for(int i = 1; i < argc; i++) {
+ if(strcmp(argv[i], "-n") == 0) {
+ if(++i >= argc) abort();
+ num_kv = atoi(argv[i]);
+ } else if(strcmp(argv[i], "-k") == 0) {
+ if(++i >= argc) abort();
+ if(strcmp(argv[i], "int") == 0) {
+ key_type = 0;
+ } else if(strcmp(argv[i], "sha1") == 0) {
+ key_type = 1;
+ } else if(strcmp(argv[i], "sha2") == 0) {
+ key_type = 2;
+ } else {
+ std::cerr << "invalid key type: " << argv[i] << "\n";
+ abort();
+ }
+ } else if(strcmp(argv[i], "-t") == 0) {
+ if(++i >= argc) abort();
+ n_reader = atoi(argv[i]);
+ } else if(strcmp(argv[i], "-d") == 0) {
+ if(++i >= argc) abort();
+ db_dir = argv[i];
+ } else if(strcmp(argv[i], "-s") == 0) {
+ sync_on_write = true;
+ } else if(strcmp(argv[i], "-m") == 0) {
+ if(++i >= argc) abort();
+ memcap = atoi(argv[i]);
+ } else {
+ std::cerr << "invalid argument: " << argv[i] << "\n";
+ }
+ }
+
+ print_cpu_info();
+ if(sync_on_write)
+ std::cout << "===== Disk sync is on\n";
+ else
+ std::cout << "===== Disk sync is off\n";
+ std::cout << "===== Memcap is " << memcap << "\n";
+
+ InitTestDir();
+ RemoveDB();
+
+ InitDB();
+ Add(num_kv);
+ DestroyDB();
+
+ InitDB(false);
+ Lookup(num_kv);
+ DestroyDB();
+
+ InitDB();
+ Delete(num_kv);
+ DestroyDB();
+
+ RemoveDB();
+
+ InitDB();
+ ConcurrencyTest(num_kv, n_reader);
+ DestroyDB();
+
+#ifdef MABAIN
+ mabain::DB::CloseLogFile();
+#endif
+ return 0;
+}
--- /dev/null
+1.GNU ncurses
+2.GNU readline
+3.GLIBC
+4.g++ Compiler that supports C++11
--- /dev/null
+# Doxyfile 1.8.11
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all text
+# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
+# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
+# for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME = "MABAIN"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER =
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF =
+
+# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
+# in the documentation. The maximum height of the logo should not exceed 55
+# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
+# the logo to the output directory.
+
+PROJECT_LOGO =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY =./doc
+
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS = NO
+
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC = YES
+
+# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF =
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB = NO
+
+# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES = YES
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
+# page for each member. If set to NO, the documentation of a member will be part
+# of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE = 4
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines.
+
+ALIASES =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C = NO
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
+# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
+# Fortran. In the later case the parser tries to guess whether the code is fixed
+# or free formatted code, this is the default for Fortran type files), VHDL. For
+# instance to make doxygen treat .inc files as Fortran files (default is PHP),
+# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+#
+# Note: For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT = YES
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC = NO
+
+# If one adds a struct or class to a group and this option is enabled, then also
+# any nested class or struct is added to the same group. By default this option
+# is disabled and one has to add nested compounds explicitly via \ingroup.
+# The default value is: NO.
+
+GROUP_NESTED_COMPOUNDS = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL = YES
+
+# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE = YES
+
+# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE = NO
+
+# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC = YES
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO,
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES = YES
+
+# This flag is only useful for Objective-C code. If set to YES, local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO, only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES = YES
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO, these declarations will be
+# included in the documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO, these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES, upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES, the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES = NO
+
+# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
+# append additional text to a page's title, such as Class Reference. If set to
+# YES the compound reference will be hidden.
+# The default value is: NO.
+
+HIDE_COMPOUND_REFERENCE= NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC = YES
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
+# list. This list is created by putting \todo commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
+# list. This list is created by putting \test commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES, the
+# list will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS = YES
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED = YES
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO, doxygen will only warn about wrong or incomplete
+# parameter documentation, but not about the absence of documentation.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC = NO
+
+# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
+# a warning is encountered.
+# The default value is: NO.
+
+WARN_AS_ERROR = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+INPUT =
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl,
+# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js.
+
+FILE_PATTERNS =
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS = */test/* */unittest/*
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS =
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+INPUT_FILTER =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+FILTER_PATTERNS =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES = YES
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# function all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION = YES
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see http://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS = YES
+
+# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
+# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the
+# cost of reduced performance. This can be particularly helpful with template
+# rich C++ code for which doxygen's built-in parser lacks the necessary type
+# information.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse-libclang=ON option for CMake.
+# The default value is: NO.
+
+CLANG_ASSISTED_PARSING = NO
+
+# If clang assisted parsing is enabled you can provide the compiler with command
+# line options that you would normally use when invoking the compiler. Note that
+# the include paths will already be set by doxygen for the files and directories
+# specified with INPUT and INCLUDE_PATH.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_OPTIONS =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX = 5
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the style sheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to YES can help to show when doxygen was last run and thus if the
+# documentation is up to date.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP = NO
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler (hhc.exe). If non-empty,
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated
+# (YES) or that it should be included in the master .chm file (NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated
+# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW = NO
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH = 250
+
+# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE = 10
+
+# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# http://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from http://www.mathjax.org before deployment.
+# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH = NO
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX = YES
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when enabling USE_PDFLATEX this option is only used for generating
+# bitmaps for formulas in the HTML output, but not in the Makefile that is
+# written to the output directory.
+# The default file is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME = makeindex
+
+# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. The package can be specified just
+# by its name or with the correct syntax as to be used with the LaTeX
+# \usepackage command. To get the times font for instance you can specify :
+# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
+# To use the option intlimits with the amsmath package you can specify:
+# EXTRA_PACKAGES=[intlimits]{amsmath}
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
+# string, for the replacement values of the other commands the user is referred
+# to HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER =
+
+# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# LaTeX style sheets that are included after the standard style sheets created
+# by doxygen. Using this option one can overrule certain style aspects. Doxygen
+# will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_STYLESHEET =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS = YES
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES, to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE = plain
+
+# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_TIMESTAMP = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT = rtf
+
+# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's config
+# file, i.e. a series of assignments. You only have to provide replacements,
+# missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's config file. A template extensions file can be generated
+# using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE =
+
+# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
+# with syntax highlighting in the RTF output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_SOURCE_CODE = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION = .3
+
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML = YES
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
+# AutoGen Definitions (see http://autogen.sf.net) file that captures the
+# structure of the code including all documentation. Note that this feature is
+# still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD = NO
+
+# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX = NO
+
+# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO, the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF = YES
+
+# If the SEARCH_INCLUDES tag is set to YES, the include files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH = .
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE =
+
+# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
+# the class index. If set to NO, only the inherited external classes will be
+# listed.
+# The default value is: NO.
+
+ALLEXTERNALS = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS = YES
+
+# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
+
+PERL_PATH = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH =
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH =
+
+# If set to YES the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: YES.
+
+HAVE_DOT = YES
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS = YES
+
+# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK = YES
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS = 0
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS = YES
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command. Disabling a call graph can be
+# accomplished by means of the command \hidecallgraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH = YES
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command. Disabling a caller graph can be
+# accomplished by means of the command \hidecallergraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH = YES
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot. For an explanation of the image formats see the section
+# output formats in the documentation of the dot tool (Graphviz (see:
+# http://www.graphviz.org/)).
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,
+# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,
+# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo,
+# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
+# png:gdiplus:gdiplus.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT = png
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+
+PLANTUML_JAR_PATH =
+
+# When using plantuml, the specified paths are searched for files specified by
+# the !include statement in a plantuml block.
+
+PLANTUML_INCLUDE_PATH =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS = NO
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND = YES
+
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP = YES
--- /dev/null
+ifndef MABAIN_INSTALL_DIR
+MABAIN_INSTALL_DIR=/usr/local
+endif
+
+CPP=g++
+
+all: mb_insert_test mb_lookup_test mb_longest_prefix_test \
+ mb_remove_test mb_iterator_test mb_multi_proc_test \
+ mb_rc_test mb_multi_thread_insert_test mb_memory_only_test
+
+CFLAGS = -I. -I$(MABAIN_INSTALL_DIR)/include -Wall -Werror -g -O0 -c -std=c++11
+LDFLAGS = -lpthread -lcrypto -L$(MABAIN_INSTALL_DIR)/lib -lmabain
+
+mb_insert_test: mb_insert_test.cpp
+ $(CPP) $(CFLAGS) mb_insert_test.cpp
+ $(CPP) mb_insert_test.o -o mb_insert_test $(LDFLAGS)
+mb_lookup_test: mb_lookup_test.cpp
+ $(CPP) $(CFLAGS) mb_lookup_test.cpp
+ $(CPP) mb_lookup_test.o -o mb_lookup_test $(LDFLAGS)
+mb_longest_prefix_test: mb_longest_prefix_test.cpp
+ $(CPP) $(CFLAGS) mb_longest_prefix_test.cpp
+ $(CPP) mb_longest_prefix_test.o -o mb_longest_prefix_test $(LDFLAGS)
+mb_remove_test: mb_remove_test.cpp
+ $(CPP) $(CFLAGS) mb_remove_test.cpp
+ $(CPP) mb_remove_test.o -o mb_remove_test $(LDFLAGS)
+mb_iterator_test: mb_iterator_test.cpp
+ $(CPP) $(CFLAGS) mb_iterator_test.cpp
+ $(CPP) mb_iterator_test.o -o mb_iterator_test $(LDFLAGS)
+mb_multi_proc_test: mb_multi_proc_test.cpp
+ $(CPP) $(CFLAGS) mb_multi_proc_test.cpp
+ $(CPP) mb_multi_proc_test.o -o mb_multi_proc_test $(LDFLAGS)
+mb_rc_test: mb_rc_test.cpp
+ $(CPP) $(CFLAGS) mb_rc_test.cpp
+ $(CPP) mb_rc_test.o -o mb_rc_test $(LDFLAGS)
+mb_multi_thread_insert_test: mb_multi_thread_insert_test.cpp
+ $(CPP) $(CFLAGS) mb_multi_thread_insert_test.cpp
+ $(CPP) mb_multi_thread_insert_test.o -o mb_multi_thread_insert_test $(LDFLAGS)
+mb_memory_only_test: mb_memory_only_test.cpp
+ $(CPP) $(CFLAGS) mb_memory_only_test.cpp
+ $(CPP) mb_memory_only_test.o -o mb_memory_only_test $(LDFLAGS)
+
+build: all
+clean:
+ -rm -f ./*.o ./mb_*_test
+ -rm -rf ./tmp_dir
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <mabain/db.h>
+
+using namespace mabain;
+
+const char *db_dir = "./tmp_dir/";
+
+// Insert key-value pair to mabain db
+int main(int argc, char *argv[])
+{
+ if(argc == 2) {
+ db_dir = argv[1];
+ }
+
+ mabain::DB::SetLogFile("/var/tmp/mabain_test/mabain.log");
+
+ DB db(db_dir, CONSTS::WriterOptions());
+ if(!db.is_open()) {
+ std::cerr << "failed to open mabain db: " << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ std::string key[3], value[3];
+ int rval;
+
+ key[0] = "Apple";
+ value[0] = "Red";
+ key[1] = "Orange";
+ value[1] = "Yellow";
+ key[2] = "Grape";
+ value[2] = "Purple";
+
+ for(int i = 0; i < 3; i++) {
+ rval = db.Add(key[i], value[i]);
+ if(rval != MBError::SUCCESS) {
+ std::cout << key[i] << ": " << MBError::get_error_str(rval) << std::endl;
+ }
+ }
+
+ db.PrintStats(std::cout);
+ db.Close();
+
+ mabain::DB::CloseLogFile();
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <mabain/db.h>
+
+using namespace mabain;
+
+#define ONE_MILLION 1000000
+
+const char *db_dir = "/var/tmp/mabain_test";
+
+// mabain db iterator
+int main(int argc, char *argv[])
+{
+ if(argc == 2) {
+ db_dir = argv[1];
+ }
+
+ DB db(db_dir, CONSTS::ReaderOptions());
+ if(!db.is_open()) {
+ std::cerr << "failed to open mabain db: " << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ std::string key, value;
+ int64_t count=0;
+ for(DB::iterator iter = db.begin(); iter != db.end(); ++iter) {
+ std::cout << iter.key << ": " << std::string((char*)iter.value.buff, iter.value.data_len) << "\n";
+ count++;
+ if(count % ONE_MILLION == 0) std::cout << "COUNT: " << count<< "\n";
+ }
+
+ db.Close();
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <mabain/db.h>
+
+using namespace mabain;
+
+const char *db_dir = "./tmp_dir/";
+
+// Perform exact key match
+int main(int argc, char *argv[])
+{
+ if(argc == 2) {
+ db_dir = argv[1];
+ }
+
+ DB db(db_dir, CONSTS::ReaderOptions());
+ if(!db.is_open()) {
+ std::cerr << "failed to open mabain db: " << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ MBData mbdata;
+ int rval;
+ std::string key[4];
+
+ key[0] = "Apple Pie";
+ key[1] = "Grape Juice";
+ key[2] = "Orange Paper";
+ key[3] = "Kiwikiwi";
+
+ for(int i = 0; i < 4; i++) {
+ rval = db.FindLongestPrefix(key[i], mbdata);
+ if(rval == MBError::SUCCESS) {
+ std::cout << key[i] << ": " << std::string((char*) mbdata.buff, mbdata.data_len) << "\n";
+ } else {
+ std::cout << key[i] << ": not found\n";
+ }
+ }
+
+ db.Close();
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <mabain/db.h>
+
+using namespace mabain;
+
+const char *db_dir = "./tmp_dir/";
+
+// Perform exact key match
+int main(int argc, char *argv[])
+{
+ if(argc == 2) {
+ db_dir = argv[1];
+ }
+
+ DB db(db_dir, CONSTS::ReaderOptions());
+ if(!db.is_open()) {
+ std::cerr << "failed to open mabain db: " << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ MBData mbdata;
+ int rval;
+ std::string key[4];
+
+ key[0] = "Apple";
+ key[1] = "Grape";
+ key[2] = "Orange";
+ key[3] = "Kiwi";
+
+ for(int i = 0; i < 4; i++) {
+ rval = db.Find(key[i], mbdata);
+ if(rval == MBError::SUCCESS) {
+ std::cout << key[i] << ": " << std::string((char *) mbdata.buff, mbdata.data_len) << "\n";
+ } else {
+ std::cout << key[i] << ": not found\n";
+ }
+ }
+
+ db.Close();
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <mabain/db.h>
+
+using namespace mabain;
+
+const char *db_name = "my-memory-only-test";
+
+// memory-only mode test
+// No db directory is required. Need an unique name instead.
+// Note MEMORY_ONLY_MODE does not support multi-process accessing the DB.
+int main(int argc, char *argv[])
+{
+ DB db(db_name, CONSTS::WriterOptions() | CONSTS::MEMORY_ONLY_MODE);
+ if(!db.is_open()) {
+ std::cerr << "failed to open mabain db: " << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ std::string key[3], value[3];
+ int rval;
+
+ key[0] = "Apple";
+ value[0] = "Red";
+ key[1] = "Orange";
+ value[1] = "Yellow";
+ key[2] = "Grape";
+ value[2] = "Purple";
+
+ for(int i = 0; i < 3; i++) {
+ rval = db.Add(key[i], value[i]);
+ if(rval == MBError::SUCCESS) {
+ std::cout << "Add " << key[i] << ": " << value[i] << std::endl;
+ }
+ }
+
+ MBData mbd;
+ for(int i = 0; i < 3; i++) {
+ rval = db.Find(key[i], mbd);
+ if(rval == MBError::SUCCESS) {
+ std::cout << "Lookup " << key[i] << ": " << std::string((char *)mbd.buff, mbd.data_len) << std::endl;
+ }
+ }
+
+ db.Close();
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+// Multiple writers/readers access the DB concurrently.
+
+#include <unistd.h>
+#include <sys/wait.h>
+#include <string.h>
+#include <assert.h>
+#include <sys/time.h>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <openssl/sha.h>
+#include <map>
+
+#include <mabain/db.h>
+
+#define ADD_TEST 0
+#define REMOVE_ONE_BY_ONE 1
+#define REMOVE_ALL 2
+#define SHRINK_TEST 3
+#define LOOKUP_TEST 4
+#define ITERATOR_TEST 5
+#define KEY_TYPE_INT 0
+#define KEY_TYPE_SHA256 1
+
+const char* mb_dir = "/var/tmp/mabain_test";
+
+using namespace mabain;
+
+static int stop_processing = 0;
+static int num_keys = 1000000;
+static int test_type = 0;
+static int64_t memcap_index = 256*1024*1024LL;
+static int64_t memcap_data = 256*1024*1024LL;
+static int key_type = KEY_TYPE_SHA256;
+//static int key_type = KEY_TYPE_INT;
+static std::map<std::string,int> checked;
+
+static const char* get_sha256_str(int key);
+
+static void SetTestStatus(bool success, bool stop_test = true)
+{
+ std::string cmd;
+ if(success) {
+ cmd = std::string("touch ") + mb_dir + "/_success";
+ } else {
+ cmd = std::string("rm ") + mb_dir + "/_success";
+ }
+ if(system(cmd.c_str()) != 0) {
+ }
+ if(stop_test) {
+ abort();
+ }
+}
+
+static void RemoveRandom(int rm_cnt, DB &db)
+{
+ int cnt = 0;
+ std::string kv;
+ int ikey = 0;
+ while(cnt <= rm_cnt) {
+ cnt++;
+
+ ikey = rand() % num_keys + 1;
+ switch(key_type) {
+ case KEY_TYPE_SHA256:
+ kv = get_sha256_str(ikey);
+ break;
+ case KEY_TYPE_INT:
+ default:
+ kv = std::string("key") + std::to_string(ikey);
+ break;
+ }
+
+ db.Remove(kv);
+ checked[kv] = 0;
+ if(cnt % 1000000 == 0)
+ std::cout << "Removed " << cnt << "\n";
+ }
+}
+
+static void PopulateDB(DB &db)
+{
+ int rval;
+ int nkeys = 0;
+ std::string kv;
+ while(nkeys < num_keys) {
+ nkeys++;
+
+ switch(key_type) {
+ case KEY_TYPE_SHA256:
+ kv = get_sha256_str(nkeys);
+ break;
+ case KEY_TYPE_INT:
+ default:
+ kv = std::string("key") + std::to_string(nkeys);
+ break;
+ }
+
+ rval = db.Add(kv, kv);
+ if(rval != MBError::SUCCESS)
+ {
+ std::cout << "failed to add " << kv << " " << rval << "\n";
+ SetTestStatus(false);
+ }
+ checked[kv] = 1;
+
+ if(nkeys % 1000000 == 0)
+ std::cout << "populate db: " << " inserted " << nkeys << " keys\n";
+ }
+}
+
+static void RemoveHalf(DB &db)
+{
+ int rval;
+ int nkeys = 0;
+ std::string kv;
+ while(nkeys < num_keys) {
+ nkeys++;
+ if(nkeys % 1000000 == 0)
+ std::cout << "Removed " << nkeys/2 << "\n";
+ if(nkeys % 2 == 0)
+ continue;
+
+ switch(key_type) {
+ case KEY_TYPE_SHA256:
+ kv = get_sha256_str(nkeys);
+ break;
+ case KEY_TYPE_INT:
+ default:
+ kv = std::string("key") + std::to_string(nkeys);
+ break;
+ }
+ rval = db.Remove(kv.c_str(), kv.length());
+ checked[kv] = 0;
+ if(rval != MBError::SUCCESS)
+ {
+ std::cout << "failed to remove " << nkeys << " " << kv << " " << rval << "\n";
+ SetTestStatus(false);
+ }
+ }
+}
+
+//Writer process
+static void Writer(int id)
+{
+ DB db(mb_dir, CONSTS::WriterOptions(), memcap_index, memcap_data);
+ if(!db.is_open()) {
+ std::cerr << "writer " << id << " failed to open mabain db: "
+ << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ if(test_type == SHRINK_TEST) {
+ db.CollectResource(0, 0);
+
+ db.Close();
+ exit(0);
+ }
+
+ int rval;
+ int nkeys = 0;
+ std::string kv;
+ int count = 0;
+ while(!stop_processing) {
+ nkeys++;
+
+ switch(key_type) {
+ case KEY_TYPE_SHA256:
+ kv = get_sha256_str(nkeys);
+ break;
+ case KEY_TYPE_INT:
+ default:
+ kv = std::string("key") + std::to_string(nkeys);
+ break;
+ }
+
+ if(test_type == REMOVE_ONE_BY_ONE) {
+ rval = db.Remove(kv.c_str(), kv.length());
+ if(rval != MBError::SUCCESS && rval != MBError::NOT_EXIST) {
+ std::cout << "failed to remove " << kv << "\n";
+ SetTestStatus(false);
+ } else {
+ count++;
+ }
+ } else if(test_type == REMOVE_ALL) {
+ if(nkeys == int(num_keys * 0.666666)) {
+ db.RemoveAll();
+ }
+ } else {
+ rval = db.Add(kv, kv);
+ if(rval != MBError::SUCCESS && rval != MBError::IN_DICT) {
+ std::cout << "failed to add " << kv << " " << rval << "\n";
+ SetTestStatus(false);
+ } else if(rval == MBError::SUCCESS) {
+ count++;
+ }
+ }
+
+ if(nkeys % 1000000 == 0)
+ std::cout << "writer " << id << " looped over " << nkeys << " keys\n";
+ if(nkeys >= num_keys) break;
+ }
+
+ db.Close();
+ exit(0);
+}
+
+//Reader process
+static void Reader(int id)
+{
+ DB db(mb_dir, CONSTS::ReaderOptions(), memcap_index, memcap_data);
+ if(!db.is_open()) {
+ std::cerr << "reader " << id << " failed to open mabain db: "
+ << db.StatusStr() << "\n";
+ SetTestStatus(false);
+ }
+
+ std::string key;
+ if(test_type == ITERATOR_TEST) {
+ int count = 0;
+ for(DB::iterator iter = db.begin(); iter != db.end(); ++iter) {
+ count++;
+ if(iter.key != std::string((char*)iter.value.buff, iter.value.data_len)) {
+ std::cout << "VALUE NOT MATCH " << iter.key << ": "
+ << std::string((char*)iter.value.buff, iter.value.data_len) << "\n";
+ SetTestStatus(false);
+ } else {
+ if(checked[iter.key] == 1)
+ checked[iter.key]++;
+ }
+ }
+ db.Close();
+ for(int i = 1; i <= num_keys; i++) {
+ switch(key_type) {
+ case KEY_TYPE_SHA256:
+ key = get_sha256_str(i);
+ break;
+ case KEY_TYPE_INT:
+ default:
+ key = std::string("key") + std::to_string(i);
+ break;
+ }
+ if(!(checked[key] == 0 || checked[key] == 2)) {
+ std::cerr << i << ": " << key << " " << checked[key] << " for id " << id << "\n";
+ SetTestStatus(false);
+ }
+ }
+ exit(0);
+ }
+
+ int ikey = 1;
+ MBData mb_data;
+ int rval;
+ int nfound = 0;
+ while(!stop_processing) {
+ switch(key_type) {
+ case KEY_TYPE_SHA256:
+ key = get_sha256_str(ikey);
+ break;
+ case KEY_TYPE_INT:
+ default:
+ key = std::string("key") + std::to_string(ikey);
+ break;
+ }
+ rval = db.Find(key, mb_data);
+
+ if(rval == MBError::SUCCESS) {
+ // check the value read from DB
+ if(memcmp(key.c_str(), mb_data.buff, mb_data.data_len)) {
+ std::cout << "READER " << id << " VALUE DOES NOT MATCH: "
+ << key << ":" << mb_data.match_len << " "
+ << std::string((const char *)mb_data.buff, mb_data.data_len) << "\n";
+ SetTestStatus(false);
+ }
+
+ ikey++;
+ nfound++;
+ }
+ else if(rval == MBError::NOT_EXIST) {
+ ikey++;
+ } else {
+ std::cerr << "ERROR: " << MBError::get_error_str(rval) << "\n";
+ SetTestStatus(false);
+ }
+
+ if(ikey % 1000000 == 0)
+ std::cout << "reader " << id << " looked up " << ikey << " keys\n";
+
+ if(ikey > num_keys) break;
+ }
+
+ stop_processing = 1;
+ db.Close();
+ exit(0);
+}
+
+static char sha256_str[65];
+static const char* get_sha256_str(int key)
+{
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ SHA256_CTX sha256;
+ SHA256_Init(&sha256);
+ SHA256_Update(&sha256, (unsigned char*)&key, 4);
+ SHA256_Final(hash, &sha256);
+ int i = 0;
+ for(i = 0; i < SHA256_DIGEST_LENGTH; i++)
+ {
+ sprintf(sha256_str + (i * 2), "%02x", hash[i]);
+ }
+ sha256_str[64] = 0;
+ return (const char*)sha256_str;
+}
+
+int main(int argc, char *argv[])
+{
+ int num_writer = 1; // There can be one writer at most.
+ int num_reader = 2;
+ test_type = -1;
+ srand(time(NULL));
+
+ mabain::DB::SetLogFile("/var/tmp/mabain_test/mabain.log");
+ if(argc >= 4) {
+ const char *test_tp = argv[1];
+ if(strcmp(test_tp, "add") == 0)
+ test_type = ADD_TEST;
+ else if(strcmp(test_tp, "remove") == 0)
+ test_type = REMOVE_ONE_BY_ONE;
+ else if(strcmp(test_tp, "remove-all") == 0)
+ test_type = REMOVE_ALL;
+ else if(strcmp(test_tp, "shrink") == 0)
+ test_type = SHRINK_TEST;
+ else if(strcmp(test_tp, "lookup") == 0)
+ test_type = LOOKUP_TEST;
+ else if(strcmp(test_tp, "iterator") == 0)
+ test_type = ITERATOR_TEST;
+ num_keys = atoi(argv[2]);
+ num_reader = atoi(argv[3]);
+ }
+ if(argc >= 5) {
+ mb_dir = argv[4];
+ }
+
+ SetTestStatus(false, false);
+ if(test_type < 0) {
+ test_type = ADD_TEST;
+ }
+ // Delete all keys from last run.
+ DB db(mb_dir, CONSTS::WriterOptions(), memcap_index, memcap_data);
+ if(!db.is_open()) {
+ std::cerr << " failed to open mabain db: "
+ << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ db.RemoveAll();
+ PopulateDB(db);
+ switch(test_type) {
+ case ADD_TEST:
+ RemoveRandom(int(num_keys * 0.7777777), db);
+ break;
+ case LOOKUP_TEST:
+ break;
+ case SHRINK_TEST:
+ RemoveHalf(db);
+ break;
+ case REMOVE_ONE_BY_ONE:
+ break;
+ case REMOVE_ALL:
+ break;
+ case ITERATOR_TEST:
+ RemoveRandom(int(num_keys * 0.666666), db);
+ break;
+ default:
+ abort();
+ }
+
+ db.Close();
+
+ std::vector<pid_t> pid_arr;
+ pid_t test_pid = getpid();
+
+ if(test_type == LOOKUP_TEST)
+ num_writer = 0;
+ assert(num_writer < 2);
+ struct timeval start_tm;
+ gettimeofday(&start_tm,NULL);
+ for(int i = 0; i < num_writer; i++) {
+ if(getpid() != test_pid) continue;
+
+ pid_t pid = fork();
+ if(pid < 0) break;
+
+ if(getpid() == test_pid) {
+ pid_arr.push_back(pid);
+ continue;
+ }
+
+ // Start child/writer process
+ Writer(i);
+ }
+
+ for(int i = 0; i < num_reader; i++) {
+ if(getpid() != test_pid) continue;
+
+ pid_t pid = fork();
+ if(pid < 0) break;
+
+ if(getpid() == test_pid) {
+ pid_arr.push_back(pid);
+ continue;
+ }
+
+ // Start child/reader process
+ Reader(i);
+ }
+
+ if(test_pid == getpid()) {
+ int status;
+ int i;
+ while(!stop_processing) {
+ // Check if all children exited
+ for(i = 0; i < (int) pid_arr.size(); i++) {
+ if(pid_arr[i] > 0) {
+ waitpid(pid_arr[i], &status, WUNTRACED | WCONTINUED);
+ pid_arr[i] = -1;
+
+ if(i == (int) pid_arr.size()-1) {
+ stop_processing = 1;
+ struct timeval stop_tm;
+ gettimeofday(&stop_tm,NULL);
+ //std::cout << "All children have exited!\n";
+ //std::cout << "time: " << ((stop_tm.tv_sec-start_tm.tv_sec)*1000000.0 +
+ // (stop_tm.tv_usec-start_tm.tv_usec))/num_keys << "\n";
+ }
+ }
+ }
+
+ usleep(1000);
+ }
+ }
+
+ mabain::DB::CloseLogFile();
+ SetTestStatus(true, false);
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <assert.h>
+#include <string.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <atomic>
+
+#include <mabain/db.h>
+
+#include "test_key.h"
+
+using namespace mabain;
+
+static int max_key = 1000000;
+static std::atomic<int> write_index;
+static bool stop_processing = false;
+static std::string mbdir = "/var/tmp/mabain_test/";
+
+static void* insert_thread(void *arg)
+{
+ int curr_key;
+ TestKey mkey(MABAIN_TEST_KEY_TYPE_INT);
+ std::string kv;
+ DB *db_r = new DB(mbdir.c_str(), CONSTS::ReaderOptions(), 128LL*1024*1024, 128LL*1024*1024);
+ assert(db_r->is_open());
+
+ while(!stop_processing) {
+ curr_key = write_index.fetch_add(1, std::memory_order_release);
+ kv = mkey.get_key(curr_key);
+ if(curr_key < max_key) {
+ assert(db_r->Add(kv, kv) == MBError::SUCCESS);
+ } else {
+ stop_processing = true;
+ break;
+ }
+ }
+
+ // Reader must unregister the async writer pointer
+ delete db_r;
+ return NULL;
+}
+
+static void SetTestStatus(bool success)
+{
+ std::string cmd;
+ if(success) {
+ cmd = std::string("touch ") + mbdir + "/_success";
+ } else {
+ cmd = std::string("rm ") + mbdir + "/_success >" + mbdir + "/out 2>" + mbdir + "/err";
+ }
+ if(system(cmd.c_str()) != 0) {
+ }
+}
+
+static void Lookup()
+{
+ TestKey mkey(MABAIN_TEST_KEY_TYPE_INT);
+ std::string kv;
+ DB *db_r = new DB(mbdir.c_str(), CONSTS::ReaderOptions(), 128LL*1024*1024, 128LL*1024*1024);
+ assert(db_r->is_open());
+ MBData mbd;
+
+ for(int i = 0; i < max_key; i++) {
+ kv = mkey.get_key(i);
+ assert(db_r->Find(kv, mbd) == MBError::SUCCESS);
+ assert(kv == std::string((const char *)mbd.buff, mbd.data_len));
+ }
+ db_r->Close();
+ delete db_r;
+}
+
+// Multiple threads performing DB insertion/deletion/updating
+int main(int argc, char *argv[])
+{
+ pthread_t pid[256];
+ int nthread = 4;
+ if(nthread > 256) {
+ abort();
+ }
+
+ if(argc > 1) {
+ mbdir = std::string(argv[1]);
+ std::cout << "Mabain test db directory " << mbdir << "\n";
+ }
+ if(argc > 2) {
+ max_key = atoi(argv[2]);
+ std::cout << "Setting number of keys to be " << max_key << "\n";
+ }
+
+ SetTestStatus(false);
+ mabain::DB::SetLogFile(mbdir + "/mabain.log");
+
+ write_index.store(0, std::memory_order_release);
+ // Writer needs to enable async writer mode.
+ int options = CONSTS::WriterOptions() | CONSTS::ASYNC_WRITER_MODE;
+ DB *db = new DB(mbdir.c_str(), options, 128LL*1024*1024, 128LL*1024*1024);
+ assert(db->is_open());
+ db->RemoveAll();
+
+ for(int i = 0; i < nthread; i++) {
+ if(pthread_create(&pid[i], NULL, insert_thread, db) != 0) {
+ std::cout << "failed to create thread\n";
+ abort();
+ }
+ }
+
+ while(!stop_processing) {
+ usleep(5);
+ }
+
+ for(int i = 0; i < nthread; i++) {
+ pthread_join(pid[i], NULL);
+ }
+
+ assert(db->Close() == MBError::SUCCESS);
+ delete db;
+
+ Lookup();
+
+ mabain::DB::CloseLogFile();
+ SetTestStatus(true);
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <assert.h>
+#include <string.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <atomic>
+
+#include <mabain/db.h>
+
+#include "test_key.h"
+
+using namespace mabain;
+
+#define DB_SIZE 128ULL*1024*1024
+
+static int max_key = 10000000;
+static std::atomic<int> write_index;
+static bool stop_processing = false;
+static std::string mbdir = "/var/tmp/mabain_test/";
+
+static int key_list_size = 10;
+std::string key_list[] = {
+ "facebook",
+ "facetime",
+ "faceoff",
+ "furious",
+ "falcons",
+ "fences",
+ "fireman",
+ "fantasticfour",
+ "fragile",
+ "frankenstein"
+};
+
+static void* insert_thread(void *arg)
+{
+ int curr_key;
+ TestKey mkey(MABAIN_TEST_KEY_TYPE_SHA_256);
+ std::string kv;
+ DB *db_r = new DB(mbdir.c_str(), CONSTS::ReaderOptions(), 128LL*1024*1024, 128LL*1024*1024);
+ // If a reader wants to perform DB update, the async writer pointer must be set.
+ assert(db_r->is_open());
+ assert(db_r->SetAsyncWriterPtr((DB *) arg) == MBError::SUCCESS);
+ assert(db_r->AsyncWriterEnabled());
+
+ while(!stop_processing) {
+ curr_key = write_index.fetch_add(1, std::memory_order_release);
+ kv = mkey.get_key(curr_key);
+ if(curr_key < max_key) {
+ assert(db_r->Add(kv, kv) == MBError::SUCCESS);
+ } else {
+ stop_processing = true;
+ break;
+ }
+ }
+
+ // Reader must unregister the async writer pointer
+ assert(db_r->UnsetAsyncWriterPtr((DB *) arg) == MBError::SUCCESS);
+ db_r->Close();
+ delete db_r;
+ return NULL;
+}
+
+static void SetTestStatus(bool success)
+{
+ std::string cmd;
+ if(success) {
+ cmd = std::string("touch ") + mbdir + "/_success";
+ } else {
+ cmd = std::string("rm ") + mbdir + "/_success >" + mbdir + "/out 2>" + mbdir + "/err";
+ }
+ if(system(cmd.c_str()) != 0) {
+ }
+}
+
+static void Lookup()
+{
+ TestKey mkey(MABAIN_TEST_KEY_TYPE_SHA_256);
+ std::string kv;
+ DB *db_r = new DB(mbdir.c_str(), CONSTS::ReaderOptions(), 128LL*1024*1024, 128LL*1024*1024);
+ assert(db_r->is_open());
+ MBData mbd;
+
+ for(int i = 0; i < 100; i++) {
+ kv = mkey.get_key(i);
+ assert(db_r->Find(kv, mbd) == MBError::SUCCESS);
+ assert(kv == std::string((const char *)mbd.buff, mbd.data_len));
+ }
+ db_r->Close();
+ delete db_r;
+}
+
+static void GarbageLookup()
+{
+ std::cout << "\nCalling Lookups during Garbage Collection" << std::endl;
+ std::string kv;
+ DB *db_r = new DB(mbdir.c_str(), CONSTS::ReaderOptions(), 128LL*1024*1024, 128LL*1024*1024);
+ assert(db_r->is_open());
+ MBData mbd;
+
+ for(int i = 0; i < key_list_size; i++) {
+ int rval = db_r->Find(key_list[i], mbd);
+ if(rval != MBError::SUCCESS) {
+ std::cout << key_list[i] << ": " << MBError::get_error_str(rval) << std::endl;
+ } else {
+ std::cout << key_list[i] << ": " << std::string((const char *)mbd.buff, mbd.data_len) << std::endl;
+ assert(rval == MBError::SUCCESS);
+ assert(key_list[i] == std::string((const char *)mbd.buff, mbd.data_len));
+ }
+ }
+
+ db_r->Close();
+ delete db_r;
+}
+
+static void Deletekeys()
+{
+ std::cout << "\nStart Deletekeys ()" << std::endl;
+ std::string kv;
+
+ int options = CONSTS::WriterOptions() | CONSTS::ReaderOptions() | CONSTS::ASYNC_WRITER_MODE;
+ DB *db = new DB(mbdir.c_str(), options, 128LL*1024*1024, 128LL*1024*1024);
+ assert(db->is_open());
+ MBData mbd;
+
+ TestKey mkey(MABAIN_TEST_KEY_TYPE_SHA_256);
+ for(int i = 0; i < 10000; i++) {
+ kv = mkey.get_key(i);
+ assert(db->Remove(kv) == MBError::SUCCESS);
+ }
+
+ db->Close();
+ delete db;
+}
+
+
+static void GarbageCollectResources()
+{
+ std::cout << "\nKick-start Garbage Collection" << std::endl;
+ TestKey mkey(MABAIN_TEST_KEY_TYPE_SHA_256);
+ std::string kv;
+
+ int options = CONSTS::WriterOptions() | CONSTS::ReaderOptions() | CONSTS::ASYNC_WRITER_MODE;
+ DB *db = new DB(mbdir.c_str(), options, 128LL*1024*1024, 128LL*1024*1024);
+ assert(db->is_open());
+ MBData mbd;
+
+ db->CollectResource(1, 1);
+
+ for(int i = 0; i < key_list_size; i++) {
+ assert(db->Add(key_list[i], key_list[i]) == MBError::SUCCESS);
+ }
+
+ db->Close();
+ delete db;
+}
+
+
+// Multiple threads performing DB insertion/deletion/updating
+int main(int argc, char *argv[])
+{
+ std::string cmd = std::string("rm -rf ") + mbdir;
+ if(system(cmd.c_str()) != 0) {}
+
+ cmd = std::string("mkdir -p ") + mbdir;
+ if(system(cmd.c_str()) != 0) {}
+
+ pthread_t pid[256];
+ int nthread = 4;
+ if(nthread > 256) {
+ abort();
+ }
+
+ if(argc > 1) {
+ mbdir = std::string(argv[1]);
+ std::cout << "Mabain test db directory " << mbdir << "\n";
+ }
+ if(argc > 2) {
+ max_key = atoi(argv[2]);
+ std::cout << "Setting number of keys to be " << max_key << "\n";
+ }
+
+ SetTestStatus(false);
+ mabain::DB::SetLogFile(mbdir + "/mabain.log");
+
+ write_index.store(0, std::memory_order_release);
+ // Writer needs to enable async writer mode.
+ int options = CONSTS::WriterOptions() | CONSTS::ASYNC_WRITER_MODE;
+ DB *db = new DB(mbdir.c_str(), options, 128LL*1024*1024, 128LL*1024*1024);
+ assert(db->is_open());
+ db->RemoveAll();
+
+ for(int i = 0; i < nthread; i++) {
+ if(pthread_create(&pid[i], NULL, insert_thread, db) != 0) {
+ std::cout << "failed to create thread\n";
+ abort();
+ }
+ }
+
+ while(!stop_processing) {
+ usleep(5);
+ }
+
+ for(int i = 0; i < nthread; i++) {
+ pthread_join(pid[i], NULL);
+ }
+
+ // Writer handle must be the last one to close if reader handles are used for DB update.
+ assert(db->Close() == MBError::SUCCESS);
+ delete db;
+
+ // at this timer we added all required keys, verify all those keys
+ Lookup();
+
+ // delete keys so that we can create some garbage data to be reclaimed
+ Deletekeys();
+
+ // kick start the garbage collection task
+ GarbageCollectResources();
+
+ // lookup few keys that we added during garbage collection
+ GarbageLookup();
+
+ mabain::DB::CloseLogFile();
+ SetTestStatus(true);
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <mabain/db.h>
+
+using namespace mabain;
+
+const char *db_dir = "./tmp_dir/";
+
+// Remove key-value pair to mabain db
+int main(int argc, char *argv[])
+{
+ if(argc == 2) {
+ db_dir = argv[1];
+ }
+
+ mabain::DB::SetLogFile("/var/tmp/mabain_test/mabain.log");
+ DB db(db_dir, CONSTS::WriterOptions());
+ if(!db.is_open()) {
+ std::cerr << "failed to open mabain db: " << db.StatusStr() << "\n";
+ exit(1);
+ }
+
+ std::string key[3], value[3];
+ int rval;
+
+ key[0] = "Apple";
+ key[1] = "Orange";
+ key[2] = "Grape";
+ value[0] = "Red";
+ value[1] = "Yellow";
+ value[2] = "Purple";
+
+ // Add
+ for(int i = 0; i < 3; i++) {
+ db.Add(key[i], value[i]);
+ }
+
+ // Remove
+ for(int i = 0; i < 3; i++) {
+ rval = db.Remove(key[i]);
+ if(rval != MBError::SUCCESS)
+ std::cerr << "failed to remove key " << key[i] << "\n";
+ else
+ std::cout << "Removed " << key[i] << ": " << value[i] << "\n";
+ }
+
+ // Query
+ MBData mb_data;
+ for(int i = 0; i < 3; i++) {
+ rval = db.Find(key[i], mb_data);
+ if(rval != MBError::SUCCESS)
+ std::cout << "key " << key[i] << " not found\n";
+ }
+
+ db.PrintStats(std::cout);
+ db.Close();
+ mabain::DB::CloseLogFile();
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <openssl/sha.h>
+
+#define MABAIN_TEST_KEY_TYPE_INT 0
+#define MABAIN_TEST_KEY_TYPE_SHA_128 1
+#define MABAIN_TEST_KEY_TYPE_SHA_256 2
+#define MABAIN_TEST_KEY_BUFF_SIZE 1024
+
+namespace {
+
+class TestKey
+{
+public:
+ TestKey(int ktype) {
+ key_type = ktype;
+ key_buff[0] = '\0';
+ }
+ ~TestKey() {
+ }
+ const char* get_key(int key) {
+ switch(key_type) {
+ case MABAIN_TEST_KEY_TYPE_INT:
+ snprintf(key_buff, MABAIN_TEST_KEY_BUFF_SIZE, "%d", key);
+ break;
+ case MABAIN_TEST_KEY_TYPE_SHA_128:
+ {
+ unsigned char hash[SHA_DIGEST_LENGTH];
+ SHA_CTX sha1;
+ SHA1_Init(&sha1);
+ SHA1_Update(&sha1, (unsigned char*)&key, 4);
+ SHA1_Final(hash, &sha1);
+ int i = 0;
+ for(i = 0; i < SHA_DIGEST_LENGTH; i++)
+ {
+ sprintf(key_buff + (i * 2), "%02x", hash[i]);
+ }
+ key_buff[32] = 0;
+ }
+ break;
+ case MABAIN_TEST_KEY_TYPE_SHA_256:
+ {
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ SHA256_CTX sha256;
+ SHA256_Init(&sha256);
+ SHA256_Update(&sha256, (unsigned char*)&key, 4);
+ SHA256_Final(hash, &sha256);
+ int i = 0;
+ for(i = 0; i < SHA256_DIGEST_LENGTH; i++)
+ {
+ sprintf(key_buff + (i * 2), "%02x", hash[i]);
+ }
+ key_buff[64] = 0;
+ }
+ break;
+ default:
+ abort();
+ }
+ return (const char *) key_buff;
+ }
+
+private:
+ int key_type;
+ char key_buff[MABAIN_TEST_KEY_BUFF_SIZE];
+};
+
+}
--- /dev/null
+#!/bin/bash
+cd examples
+LD_LIBRARY_PATH=/usr/local/lib/ ./mb_insert_test
+LD_LIBRARY_PATH=/usr/local/lib/ ./mb_longest_prefix_test
+LD_LIBRARY_PATH=/usr/local/lib/ ./mb_lookup_test
+LD_LIBRARY_PATH=/usr/local/lib/ ./mb_memory_only_test
+
+
--- /dev/null
+CPP=g++
+TARGET=libmabain.so
+
+all: $(TARGET)
+
+CFLAGS = -I. -I.. -Iutil -Wall -Werror -c -Wwrite-strings -Wsign-compare -Wcast-align -Wformat-security -fdiagnostics-show-option
+CFLAGS += -g -ggdb -fPIC -O2 -std=c++11
+CFLAGS += -D__SHM_LOCK__ -D__LOCK_FREE__ -D__SHM_QUEUE__
+LDFLAGS = -lpthread
+
+SOURCES = $(wildcard *.cpp) $(wildcard util/*.cpp)
+HEADERS = $(wildcard *.h) $(wildcard util/*.h)
+OBJECTS = $(SOURCES:.cpp=.o)
+
+$(TARGET):$(OBJECTS) $(HEADERS)
+ $(CPP) -shared -o $(TARGET) $(OBJECTS) $(LDFLAGS)
+
+.cpp.o: $(HEADERS) $(SOURCES)
+ $(CPP) $(CFLAGS) $< -o $@
+
+build: all
+
+uninstall:
+ -rm -rf $(INSTALLDIR)/include/mabain
+ -rm -f $(INSTALLDIR)/lib/$(TARGET)
+ -rm -f $(INSTALLDIR)/bin/mbc
+
+clean:
+ -rm -f *.o util/*.o $(TARGET)
+ -make -C unittest clean
+
+unit-test: all
+ make -C unittest
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <unistd.h>
+#include <string.h>
+
+#include "async_writer.h"
+#include "error.h"
+#include "logger.h"
+#include "mb_data.h"
+#include "mb_rc.h"
+#include "integer_4b_5b.h"
+#include "dict.h"
+#include "./util/shm_mutex.h"
+
+namespace mabain {
+
+static void free_async_node(AsyncNode *node_ptr)
+{
+#ifndef __SHM_QUEUE__
+ if(node_ptr->key != NULL)
+ {
+ free(node_ptr->key);
+ node_ptr->key = NULL;
+ node_ptr->key_len = 0;
+ }
+
+ if(node_ptr->data != NULL)
+ {
+ free(node_ptr->data);
+ node_ptr->data = NULL;
+ node_ptr->data_len = 0;
+ }
+#endif
+
+ node_ptr->type = MABAIN_ASYNC_TYPE_NONE;
+}
+
+AsyncWriter::AsyncWriter(DB *db_ptr)
+ : db(db_ptr),
+ tid(0),
+ stop_processing(false),
+ queue(NULL),
+#ifdef __SHM_QUEUE__
+ header(NULL)
+#else
+ num_users(0),
+ queue_index(0),
+ writer_index(0)
+#endif
+{
+ dict = NULL;
+ if(!(db_ptr->GetDBOptions() & CONSTS::ACCESS_MODE_WRITER))
+ throw (int) MBError::NOT_ALLOWED;
+ if(db == NULL)
+ throw (int) MBError::INVALID_ARG;
+ dict = db->GetDictPtr();
+ if(dict == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+#ifdef __SHM_QUEUE__
+ // initialize shared memory queue pointer
+ header = dict->GetHeaderPtr();
+ if(header == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+ char *hdr_ptr = (char *) header;
+ queue = reinterpret_cast<AsyncNode *>(hdr_ptr + RollableFile::page_size);
+ header->rc_flag.store(0, std::memory_order_release);
+#else
+ queue = new AsyncNode[MB_MAX_NUM_SHM_QUEUE_NODE];
+ memset(queue, 0, MB_MAX_NUM_SHM_QUEUE_NODE * sizeof(AsyncNode));
+ for(int i = 0; i < MB_MAX_NUM_SHM_QUEUE_NODE; i++)
+ {
+ queue[i].in_use.store(false, std::memory_order_release);
+ if(pthread_mutex_init(&queue[i].mutex, NULL) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to init mutex");
+ throw (int) MBError::MUTEX_ERROR;
+ }
+ if(pthread_cond_init(&queue[i].cond, NULL) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to init conditional variable");
+ throw (int) MBError::MUTEX_ERROR;
+ }
+ }
+ is_rc_running = false;
+#endif
+
+ rc_backup_dir = NULL;
+ // start the thread
+ if(pthread_create(&tid, NULL, async_thread_wrapper, this) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to create async thread");
+ tid = 0;
+ throw (int) MBError::THREAD_FAILED;
+ }
+
+}
+
+AsyncWriter::~AsyncWriter()
+{
+}
+
+#ifndef __SHM_QUEUE__
+void AsyncWriter::UpdateNumUsers(int delta)
+{
+ if(delta > 0)
+ num_users.fetch_add(1, std::memory_order_release);
+ else if(delta < 0)
+ num_users.fetch_sub(1, std::memory_order_release);
+}
+#endif
+
+int AsyncWriter::StopAsyncThread()
+{
+#ifndef __SHM_QUEUE__
+ if(num_users.load(std::memory_order_consume) > 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "still being used, cannot shutdown async thread");
+ }
+#endif
+
+ stop_processing = true;
+
+#ifndef __SHM_QUEUE__
+ for(int i = 0; i < MB_MAX_NUM_SHM_QUEUE_NODE; i++)
+ {
+ pthread_cond_signal(&queue[i].cond);
+ }
+#endif
+
+ if(tid != 0)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "joining async writer thread");
+ pthread_join(tid, NULL);
+ }
+
+#ifndef __SHM_QUEUE__
+ for(int i = 0; i < MB_MAX_NUM_SHM_QUEUE_NODE; i++)
+ {
+ pthread_mutex_destroy(&queue[i].mutex);
+ pthread_cond_destroy(&queue[i].cond);
+ }
+
+ if(queue != NULL)
+ delete [] queue;
+#endif
+
+ return MBError::SUCCESS;
+}
+
+#ifndef __SHM_QUEUE__
+// Check if async tasks are completed.
+bool AsyncWriter::Busy() const
+{
+ uint32_t index = queue_index.load(std::memory_order_consume);
+ return index != writer_index || is_rc_running;
+}
+#endif
+
+#ifndef __SHM_QUEUE__
+AsyncNode* AsyncWriter::AcquireSlot()
+{
+ uint32_t index = queue_index.fetch_add(1, std::memory_order_release);
+ AsyncNode *node_ptr = queue + (index % MB_MAX_NUM_SHM_QUEUE_NODE);
+
+ if(pthread_mutex_lock(&node_ptr->mutex) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to lock mutex");
+ return NULL;
+ }
+
+ while(node_ptr->in_use.load(std::memory_order_consume))
+ {
+ pthread_cond_wait(&node_ptr->cond, &node_ptr->mutex);
+ }
+
+ return node_ptr;
+}
+#endif
+
+#ifndef __SHM_QUEUE__
+int AsyncWriter::PrepareSlot(AsyncNode *node_ptr) const
+{
+ node_ptr->in_use.store(true, std::memory_order_release);
+ pthread_cond_signal(&node_ptr->cond);
+ if(pthread_mutex_unlock(&node_ptr->mutex) != 0)
+ return MBError::MUTEX_ERROR;
+ return MBError::SUCCESS;
+}
+#endif
+
+#ifndef __SHM_QUEUE__
+int AsyncWriter::Add(const char *key, int key_len, const char *data,
+ int data_len, bool overwrite)
+{
+ if(stop_processing)
+ return MBError::DB_CLOSED;
+
+ AsyncNode *node_ptr = AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ node_ptr->key = (char *) malloc(key_len);
+ node_ptr->data = (char *) malloc(data_len);
+ if(node_ptr->key == NULL || node_ptr->data == NULL)
+ {
+ pthread_mutex_unlock(&node_ptr->mutex);
+ free_async_node(node_ptr);
+ return MBError::NO_MEMORY;
+ }
+ memcpy(node_ptr->key, key, key_len);
+ memcpy(node_ptr->data, data, data_len);
+ node_ptr->key_len = key_len;
+ node_ptr->data_len = data_len;
+ node_ptr->overwrite = overwrite;
+
+ node_ptr->type = MABAIN_ASYNC_TYPE_ADD;
+
+
+ return PrepareSlot(node_ptr);
+}
+#endif
+
+#ifndef __SHM_QUEUE__
+int AsyncWriter::Remove(const char *key, int len)
+{
+ if(stop_processing)
+ return MBError::DB_CLOSED;
+
+ AsyncNode *node_ptr = AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ node_ptr->key = (char *) malloc(len);
+ if(node_ptr->key == NULL)
+ {
+ pthread_mutex_unlock(&node_ptr->mutex);
+ free_async_node(node_ptr);
+ return MBError::NO_MEMORY;
+ }
+ memcpy(node_ptr->key, key, len);
+ node_ptr->key_len = len;
+ node_ptr->type = MABAIN_ASYNC_TYPE_REMOVE;
+
+ return PrepareSlot(node_ptr);
+}
+#endif
+
+#ifndef __SHM_QUEUE__
+int AsyncWriter::Backup(const char *backup_dir)
+{
+ if(backup_dir == NULL)
+ return MBError::INVALID_ARG;
+
+ if(stop_processing)
+ return MBError::DB_CLOSED;
+
+ AsyncNode *node_ptr = AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ node_ptr->data = (char *) strdup(backup_dir);
+ if(node_ptr->data == NULL)
+ {
+ pthread_mutex_unlock(&node_ptr->mutex);
+ free_async_node(node_ptr);
+ return MBError::NO_MEMORY;
+ }
+ node_ptr->type = MABAIN_ASYNC_TYPE_BACKUP;
+ return PrepareSlot(node_ptr);
+}
+#endif
+
+#ifndef __SHM_QUEUE__
+int AsyncWriter::RemoveAll()
+{
+ if(stop_processing)
+ return MBError::DB_CLOSED;
+
+ AsyncNode *node_ptr = AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ node_ptr->type = MABAIN_ASYNC_TYPE_REMOVE_ALL;
+
+ return PrepareSlot(node_ptr);
+}
+#endif
+
+#ifndef __SHM_QUEUE__
+int AsyncWriter::CollectResource(int64_t m_index_rc_size, int64_t m_data_rc_size,
+ int64_t max_dbsz, int64_t max_dbcnt)
+{
+ if(stop_processing)
+ return MBError::DB_CLOSED;
+
+ AsyncNode *node_ptr = AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ int64_t *data_ptr = (int64_t *) calloc(4, sizeof(int64_t));
+ if (data_ptr == NULL)
+ return MBError::NO_MEMORY;
+
+ node_ptr->data = (char *) data_ptr;
+ node_ptr->data_len = sizeof(int64_t)*4;
+ data_ptr[0] = m_index_rc_size;
+ data_ptr[1] = m_data_rc_size;
+ data_ptr[2] = max_dbsz;
+ data_ptr[3] = max_dbcnt;
+ node_ptr->type = MABAIN_ASYNC_TYPE_RC;
+
+ return PrepareSlot(node_ptr);
+}
+#endif
+
+// Run a given number of tasks if they are available.
+// This function should only be called by rc or pruner.
+int AsyncWriter::ProcessTask(int ntasks, bool rc_mode)
+{
+ AsyncNode *node_ptr;
+ MBData mbd;
+ int rval = MBError::SUCCESS;
+ int count = 0;
+#ifdef __SHM_QUEUE__
+ struct timespec tm_exp;
+#endif
+
+ while(count < ntasks)
+ {
+#ifdef __SHM_QUEUE__
+ node_ptr = &queue[header->writer_index % header->async_queue_size];
+ tm_exp.tv_sec = time(NULL) + MB_ASYNC_SHM_LOCK_TMOUT;
+ tm_exp.tv_nsec = 0;
+ rval = pthread_mutex_timedlock(&node_ptr->mutex, &tm_exp);
+ if(rval == ETIMEDOUT)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "mutex lock timeout, need to re-intialize");
+ InitShmMutex(&node_ptr->mutex);
+ count = ntasks;
+ continue;
+ }
+ else if(rval != 0)
+#else
+ node_ptr = &queue[writer_index % MB_MAX_NUM_SHM_QUEUE_NODE];
+ if(pthread_mutex_lock(&node_ptr->mutex) != 0)
+#endif
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to lock mutex");
+ throw (int) MBError::MUTEX_ERROR;
+ }
+
+ if(node_ptr->in_use.load(std::memory_order_consume))
+ {
+ switch(node_ptr->type)
+ {
+ case MABAIN_ASYNC_TYPE_ADD:
+ if(rc_mode)
+ mbd.options = CONSTS::OPTION_RC_MODE;
+ mbd.buff = (uint8_t *) node_ptr->data;
+ mbd.data_len = node_ptr->data_len;
+ try {
+ rval = dict->Add((uint8_t *)node_ptr->key, node_ptr->key_len, mbd, node_ptr->overwrite);
+ } catch (int err) {
+ rval = err;
+ Logger::Log(LOG_LEVEL_ERROR, "dict->Add throws error %s",
+ MBError::get_error_str(err));
+ }
+ break;
+ case MABAIN_ASYNC_TYPE_REMOVE:
+ // FIXME
+ // Removing entries during rc is currently not supported.
+ // The index or data index could have been reset in fucntion ResourceColletion::Finish.
+ // However, the deletion may be run for some entry which still exist in the rc root tree.
+ // This requires modifying buffer in high end, where the offset for writing is greather than
+ // header->m_index_offset. This causes exception thrown from DictMem::WriteData.
+ // Note this is not a problem for Dict::Add since Add does not modify buffers in high end.
+ // This problem will be fixed when resolving issue: https://github.com/chxdeng/mabain/issues/21
+ rval = MBError::SUCCESS;
+ break;
+ case MABAIN_ASYNC_TYPE_REMOVE_ALL:
+ if(!rc_mode)
+ {
+ try {
+ rval = dict->RemoveAll();
+ } catch (int err) {
+ Logger::Log(LOG_LEVEL_ERROR, "dict->Add throws error %s",
+ MBError::get_error_str(err));
+ rval = err;
+ }
+ }
+ else
+ {
+ rval = MBError::SUCCESS;
+ }
+ break;
+ case MABAIN_ASYNC_TYPE_RC:
+ // ignore rc task since it is running already.
+ rval = MBError::RC_SKIPPED;
+ break;
+ case MABAIN_ASYNC_TYPE_NONE:
+ rval = MBError::SUCCESS;
+ break;
+ case MABAIN_ASYNC_TYPE_BACKUP:
+ // clean up existing backup dir varibale buffer.
+ if (rc_backup_dir != NULL)
+ free(rc_backup_dir);
+#ifdef __SHM_QUEUE__
+ rc_backup_dir = (char *) malloc(node_ptr->data_len+1);
+ memcpy(rc_backup_dir, node_ptr->data, node_ptr->data_len);
+ rc_backup_dir[node_ptr->data_len] = '\0';
+#else
+ rc_backup_dir = (char *) node_ptr->data;
+ node_ptr->data = NULL;
+#endif
+ rval = MBError::SUCCESS;
+ break;
+ default:
+ rval = MBError::INVALID_ARG;
+ break;
+ }
+
+#ifdef __SHM_QUEUE__
+ header->writer_index++;
+#else
+ writer_index++;
+#endif
+ free_async_node(node_ptr);
+ node_ptr->in_use.store(false, std::memory_order_release);
+ pthread_cond_signal(&node_ptr->cond);
+ mbd.Clear();
+ count++;
+ }
+ else
+ {
+ // done processing
+ count = ntasks;
+ }
+
+ if(pthread_mutex_unlock(&node_ptr->mutex) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to unlock mutex");
+ throw (int) MBError::MUTEX_ERROR;
+ }
+
+ if(rval != MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_DEBUG, "failed to run update %d: %s",
+ (int)node_ptr->type, MBError::get_error_str(rval));
+ }
+ }
+
+ if(stop_processing)
+ return MBError::RC_SKIPPED;
+ return MBError::SUCCESS;
+}
+
+#ifdef __SHM_QUEUE__
+uint32_t AsyncWriter::NextShmSlot(uint32_t windex, uint32_t qindex)
+{
+ int cnt = 0;
+ while(windex != qindex)
+ {
+ if(queue[windex % header->async_queue_size].in_use.load(std::memory_order_consume))
+ break;
+ if(++cnt > header->async_queue_size)
+ {
+ windex = qindex;
+ break;
+ }
+
+ windex++;
+ }
+
+ return windex;
+}
+#endif
+
+void* AsyncWriter::async_writer_thread()
+{
+ AsyncNode *node_ptr;
+ MBData mbd;
+ int rval;
+ int64_t min_index_size = 0;
+ int64_t min_data_size = 0;
+ int64_t max_dbsize = MAX_6B_OFFSET;
+ int64_t max_dbcount = MAX_6B_OFFSET;
+#ifdef __SHM_QUEUE__
+ struct timespec tm_exp;
+ bool skip;
+#endif
+
+ Logger::Log(LOG_LEVEL_INFO, "async writer started");
+ while(true)
+ {
+#ifdef __SHM_QUEUE__
+ node_ptr = &queue[header->writer_index % header->async_queue_size];
+ tm_exp.tv_sec = time(NULL) + MB_ASYNC_SHM_LOCK_TMOUT;
+ tm_exp.tv_nsec = 0;
+ rval = pthread_mutex_timedlock(&node_ptr->mutex, &tm_exp);
+ if(rval == ETIMEDOUT)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "async writer shared memory mutex lock timeout, need to re-intialize");
+ InitShmMutex(&node_ptr->mutex);
+ header->writer_index++;
+ continue;
+ }
+ else if(rval != 0)
+#else
+ node_ptr = &queue[writer_index % MB_MAX_NUM_SHM_QUEUE_NODE];
+ if(pthread_mutex_lock(&node_ptr->mutex) != 0)
+#endif
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "async writer failed to lock shared memory mutex");
+ throw (int) MBError::MUTEX_ERROR;
+ }
+
+#ifdef __SHM_QUEUE__
+ skip = false;
+#endif
+ while(!node_ptr->in_use.load(std::memory_order_consume))
+ {
+ if(stop_processing)
+ break;
+#ifdef __SHM_QUEUE__
+ tm_exp.tv_sec = time(NULL) + MB_ASYNC_SHM_LOCK_TMOUT;
+ tm_exp.tv_nsec = 0;
+ pthread_cond_timedwait(&node_ptr->cond, &node_ptr->mutex, &tm_exp);
+
+ uint32_t windex = header->writer_index;
+ uint32_t qindex = header->queue_index.load(std::memory_order_consume);
+ if(windex != qindex)
+ {
+ // Reader process may have exited unexpectedly. Recover index.
+ skip = true;
+ header->writer_index = NextShmSlot(windex, qindex);
+ break;
+ }
+ continue;
+#else
+ pthread_cond_wait(&node_ptr->cond, &node_ptr->mutex);
+#endif
+ }
+
+#ifdef __SHM_QUEUE__
+ if(skip || stop_processing)
+ {
+ if(pthread_mutex_unlock(&node_ptr->mutex) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "async writer failed to unlock shared memory mutex");
+ throw (int) MBError::MUTEX_ERROR;
+ }
+ if(stop_processing)
+ break;
+ continue;
+ }
+#else
+ if(stop_processing && !node_ptr->in_use.load(std::memory_order_consume))
+ {
+ pthread_mutex_unlock(&node_ptr->mutex);
+ break;
+ }
+#endif
+ // process the node
+ switch(node_ptr->type)
+ {
+ case MABAIN_ASYNC_TYPE_ADD:
+ mbd.buff = (uint8_t *) node_ptr->data;
+ mbd.data_len = node_ptr->data_len;
+ try {
+ rval = dict->Add((uint8_t *)node_ptr->key, node_ptr->key_len, mbd,
+ node_ptr->overwrite);
+ } catch (int err) {
+ Logger::Log(LOG_LEVEL_ERROR, "dict->Add throws error %s",
+ MBError::get_error_str(err));
+ rval = err;
+ }
+ break;
+ case MABAIN_ASYNC_TYPE_REMOVE:
+ mbd.options |= CONSTS::OPTION_FIND_AND_STORE_PARENT;
+ try {
+ rval = dict->Remove((uint8_t *)node_ptr->key, node_ptr->key_len, mbd);
+ } catch (int err) {
+ Logger::Log(LOG_LEVEL_ERROR, "dict->Remmove throws error %s",
+ MBError::get_error_str(err));
+ rval = err;
+ }
+ mbd.options &= ~CONSTS::OPTION_FIND_AND_STORE_PARENT;
+ break;
+ case MABAIN_ASYNC_TYPE_REMOVE_ALL:
+ try {
+ rval = dict->RemoveAll();
+ } catch (int err) {
+ Logger::Log(LOG_LEVEL_ERROR, "dict->RemoveAll throws error %s",
+ MBError::get_error_str(err));
+ rval = err;
+ }
+ break;
+ case MABAIN_ASYNC_TYPE_RC:
+ rval = MBError::SUCCESS;
+#ifdef __SHM_QUEUE__
+ header->rc_flag.store(1, std::memory_order_release);
+#else
+ is_rc_running = true;
+#endif
+ {
+ int64_t *data_ptr = reinterpret_cast<int64_t *>(node_ptr->data);
+ min_index_size = data_ptr[0];
+ min_data_size = data_ptr[1];
+ max_dbsize = data_ptr[2];
+ max_dbcount = data_ptr[3];
+ }
+ break;
+ case MABAIN_ASYNC_TYPE_NONE:
+ rval = MBError::SUCCESS;
+ break;
+ case MABAIN_ASYNC_TYPE_BACKUP:
+ try {
+ DBBackup mbbk(*db);
+ rval = mbbk.Backup((const char*) node_ptr->data);
+ } catch (int error) {
+ rval = error;
+ }
+ break;
+ default:
+ rval = MBError::INVALID_ARG;
+ break;
+ }
+
+#ifdef __SHM_QUEUE__
+ header->writer_index++;
+#else
+ writer_index++;
+#endif
+ free_async_node(node_ptr);
+ node_ptr->in_use.store(false, std::memory_order_release);
+ pthread_cond_signal(&node_ptr->cond);
+ if(pthread_mutex_unlock(&node_ptr->mutex) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to unlock mutex");
+ throw (int) MBError::MUTEX_ERROR;
+ }
+
+ if(rval != MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_DEBUG, "failed to run update %d: %s",
+ (int)node_ptr->type, MBError::get_error_str(rval));
+ }
+
+ mbd.Clear();
+
+#ifdef __SHM_QUEUE__
+ if (header->rc_flag.load(std::memory_order_consume) == 1)
+#else
+ if(is_rc_running)
+#endif
+ {
+ rval = MBError::SUCCESS;
+ try {
+ ResourceCollection rc = ResourceCollection(*db);
+ rc.ReclaimResource(min_index_size, min_data_size, max_dbsize, max_dbcount, this);
+ } catch (int error) {
+ if(error != MBError::RC_SKIPPED)
+ Logger::Log(LOG_LEVEL_WARN, "rc failed :%s", MBError::get_error_str(error));
+ else
+ rval = error;
+ }
+
+#ifdef __SHM_QUEUE__
+ header->rc_flag.store(0, std::memory_order_release);
+#else
+ is_rc_running = false;
+#endif
+ if(rc_backup_dir != NULL)
+ {
+ if(rval == MBError::SUCCESS)
+ {
+#ifdef __SHM_QUEUE__
+ dict->SHMQ_Backup(rc_backup_dir);
+#else
+ Backup(rc_backup_dir);
+#endif
+ }
+ free(rc_backup_dir);
+ rc_backup_dir = NULL;
+ }
+ }
+ }
+
+ mbd.buff = NULL;
+ Logger::Log(LOG_LEVEL_INFO, "async writer exiting");
+ return NULL;
+}
+
+void* AsyncWriter::async_thread_wrapper(void *context)
+{
+ AsyncWriter *instance_ptr = static_cast<AsyncWriter *>(context);
+ return instance_ptr->async_writer_thread();
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __ASYNC_WRITER_H__
+#define __ASYNC_WRITER_H__
+
+#include <pthread.h>
+
+#include "db.h"
+//#include "mb_rc.h"
+#include "dict.h"
+#include "mb_backup.h"
+
+namespace mabain {
+
+#define MABAIN_ASYNC_TYPE_NONE 0
+#define MABAIN_ASYNC_TYPE_ADD 1
+#define MABAIN_ASYNC_TYPE_REMOVE 2
+#define MABAIN_ASYNC_TYPE_REMOVE_ALL 3
+#define MABAIN_ASYNC_TYPE_RC 4
+#define MABAIN_ASYNC_TYPE_BACKUP 5
+
+#define MB_ASYNC_SHM_KEY_SIZE 256
+#define MB_ASYNC_SHM_DATA_SIZE 1024
+#define MB_ASYNC_SHM_LOCK_TMOUT 5
+
+
+typedef struct _AsyncNode
+{
+ std::atomic<bool> in_use;
+ pthread_mutex_t mutex;
+ pthread_cond_t cond;
+
+#ifdef __SHM_QUEUE__
+ char key[MB_ASYNC_SHM_KEY_SIZE];
+ char data[MB_ASYNC_SHM_DATA_SIZE];
+#else
+ char *key;
+ char *data;
+#endif
+ int key_len;
+ int data_len;
+ bool overwrite;
+ char type;
+} AsyncNode;
+
+class AsyncWriter
+{
+public:
+
+ AsyncWriter(DB *db_ptr);
+ ~AsyncWriter();
+
+#ifndef __SHM_QUEUE__
+ void UpdateNumUsers(int delta);
+ int Add(const char *key, int key_len, const char *data, int data_len, bool overwrite);
+ int Remove(const char *key, int len);
+ int RemoveAll();
+ int Backup(const char *backup_dir);
+ int CollectResource(int64_t m_index_rc_size, int64_t m_data_rc_size,
+ int64_t max_dbsz, int64_t max_dbcnt);
+ bool Busy() const;
+#endif
+
+ int StopAsyncThread();
+ int ProcessTask(int ntasks, bool rc_mode);
+
+private:
+ static void *async_thread_wrapper(void *context);
+ AsyncNode* AcquireSlot();
+ int PrepareSlot(AsyncNode *node_ptr) const;
+ void* async_writer_thread();
+#ifdef __SHM_QUEUE__
+ uint32_t NextShmSlot(uint32_t windex, uint32_t qindex);
+#endif
+
+ // db pointer
+ DB *db;
+ Dict *dict;
+
+ // thread id
+ pthread_t tid;
+ bool stop_processing;
+
+ AsyncNode *queue;
+#ifdef __SHM_QUEUE__
+ IndexHeader *header;
+#else
+ std::atomic<int> num_users;
+ std::atomic<uint32_t> queue_index;
+ uint32_t writer_index;
+#endif
+
+ bool is_rc_running;
+ char *rc_backup_dir;
+};
+
+}
+
+#endif
--- /dev/null
+/*
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <iostream>
+#include <unistd.h>
+#include <sys/syscall.h>
+
+#include <errno.h>
+
+#include "dict.h"
+#include "db.h"
+#include "error.h"
+#include "logger.h"
+#include "mb_lsq.h"
+#include "version.h"
+#include "mb_rc.h"
+#include "integer_4b_5b.h"
+#include "async_writer.h"
+#include "mb_backup.h"
+#include "resource_pool.h"
+#include "drm_base.h"
+#include "util/shm_mutex.h"
+#include "util/utils.h"
+
+namespace mabain {
+
+// Current mabain version 1.2.0
+uint16_t version[4] = {1, 2, 0, 0};
+
+DB::~DB()
+{
+ if(status != MBError::DB_CLOSED)
+ Close();
+}
+
+int DB::Close()
+{
+ int rval = MBError::SUCCESS;
+
+ if((options & CONSTS::ACCESS_MODE_WRITER) && async_writer != NULL)
+ {
+ rval = async_writer->StopAsyncThread();
+ if(rval != MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "failed to stop async writer thread: %s",
+ MBError::get_error_str(rval));
+ }
+
+ delete async_writer;
+ async_writer = NULL;
+ }
+
+ if(dict != NULL)
+ {
+ if(options & CONSTS::ACCESS_MODE_WRITER)
+ dict->PrintStats(Logger::GetLogStream());
+ UpdateNumHandlers(options, -1);
+
+ dict->Destroy();
+ delete dict;
+ dict = NULL;
+ }
+ else
+ {
+ rval = status;
+ }
+
+ status = MBError::DB_CLOSED;
+ if(options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ ResourcePool::getInstance().RemoveResourceByPath(mb_dir + "_lock");
+ release_writer_lock(writer_lock_fd);
+ }
+ Logger::Log(LOG_LEVEL_INFO, "connector %u disconnected from DB", identifier);
+ return rval;
+}
+
+int DB::UpdateNumHandlers(int mode, int delta)
+{
+ int rval = MBError::SUCCESS;
+
+ WrLock();
+
+ if(mode & CONSTS::ACCESS_MODE_WRITER)
+ rval = dict->UpdateNumWriter(delta);
+ else
+ dict->UpdateNumReader(delta);
+
+ UnLock();
+
+ return rval;
+}
+
+// Constructor for initializing DB handle
+DB::DB(const char *db_path,
+ int db_options,
+ size_t memcap_index,
+ size_t memcap_data,
+ uint32_t id,
+ uint32_t queue_size) : status(MBError::NOT_INITIALIZED),
+ writer_lock_fd(-1)
+{
+ MBConfig config;
+ memset(&config, 0, sizeof(config));
+ config.mbdir = db_path;
+ config.options = db_options;
+ config.memcap_index = memcap_index;
+ config.memcap_data = memcap_data;
+ config.connect_id = id;
+ config.queue_size = queue_size;
+
+ InitDB(config);
+}
+
+DB::DB(MBConfig &config) : status(MBError::NOT_INITIALIZED),
+ writer_lock_fd(-1)
+{
+ InitDB(config);
+}
+
+int DB::ValidateConfig(MBConfig &config)
+{
+ if(config.mbdir == NULL)
+ return MBError::INVALID_ARG;
+
+ if(config.memcap_index == 0)
+ config.memcap_index = 2*config.block_size_index;
+ if(config.memcap_data == 0)
+ config.memcap_data = 2*config.block_size_data;
+
+ if(config.options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ if(config.block_size_index == 0)
+ config.block_size_index = INDEX_BLOCK_SIZE_DEFAULT;
+ if(config.block_size_data == 0)
+ config.block_size_data = DATA_BLOCK_SIZE_DEFAULT;
+ if(config.num_entry_per_bucket <= 0)
+ config.num_entry_per_bucket = 1000;
+ if(config.num_entry_per_bucket < 8)
+ {
+ std::cerr << "count in eviction bucket must be greater than 7\n";
+ return MBError::INVALID_ARG;
+ }
+ }
+ if(config.options & CONSTS::USE_SLIDING_WINDOW)
+ {
+ std::cout << "sliding window support is deprecated\n";
+ config.options &= ~CONSTS::USE_SLIDING_WINDOW;
+ }
+
+ if(config.block_size_index != 0 && (config.block_size_index % BLOCK_SIZE_ALIGN != 0))
+ {
+ std::cerr << "block size must be multiple of " << BLOCK_SIZE_ALIGN << "\n";
+ return MBError::INVALID_ARG;
+ }
+ if(config.block_size_data != 0 && (config.block_size_data % BLOCK_SIZE_ALIGN != 0))
+ {
+ std::cerr << "block size must be multiple of " << BLOCK_SIZE_ALIGN << "\n";
+ return MBError::INVALID_ARG;
+ }
+
+ if(config.max_num_index_block == 0)
+ config.max_num_index_block = 1024;
+ if(config.max_num_data_block == 0)
+ config.max_num_data_block = 1024;
+ if (config.queue_size == 0)
+ config.queue_size = MB_MAX_NUM_SHM_QUEUE_NODE;
+
+ return MBError::SUCCESS;
+}
+
+void DB::PreCheckDB(const MBConfig &config, bool &init_header, bool &update_header)
+{
+ if(config.options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ std::string lock_file = mb_dir + "_lock";
+ // internal check first
+ int ret = ResourcePool::getInstance().AddResourceByPath(lock_file, NULL);
+ if(ret == MBError::SUCCESS)
+ {
+ if(!(config.options & CONSTS::MEMORY_ONLY_MODE))
+ {
+ // process check by file lock
+ writer_lock_fd = acquire_writer_lock(lock_file);
+ if(writer_lock_fd < 0)
+ status = MBError::WRITER_EXIST;
+ }
+ }
+ else
+ {
+ status = MBError::WRITER_EXIST;
+ }
+ if(status == MBError::WRITER_EXIST)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to initialize db: %s",
+ MBError::get_error_str(status));
+ return;
+ }
+ }
+
+ if(config.options & CONSTS::MEMORY_ONLY_MODE)
+ {
+ if(config.options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ init_header = true;
+ }
+ else
+ {
+ init_header = false;
+ if(!ResourcePool::getInstance().CheckExistence(mb_dir + "_mabain_h"))
+ status = MBError::NO_DB;
+ }
+ }
+ else
+ {
+ // Check if the DB directory exist with proper permission
+ if(access(mb_dir.c_str(), F_OK))
+ {
+ std::cerr << "database directory check for " + mb_dir + " failed errno: " +
+ std::to_string(errno) << std::endl;
+ status = MBError::NO_DB;
+ return;
+ }
+ Logger::Log(LOG_LEVEL_INFO, "connector %u DB options: %d",
+ config.connect_id, config.options);
+ // Check if DB exist. This can be done by check existence of the first index file.
+ // If this is the first time the DB is opened and it is in writer mode, then we
+ // need to update the header for the first time. If only reader access mode is
+ // required and the file does not exist, we should bail here and the DB open will
+ // not be successful.
+ std::string header_file = mb_dir + "_mabain_h";
+ if(access(header_file.c_str(), R_OK))
+ {
+ if(config.options & CONSTS::ACCESS_MODE_WRITER)
+ init_header = true;
+ else
+ status = MBError::NO_DB;
+ }
+ }
+
+ // Check Header version
+ if(!init_header && !(config.options & CONSTS::MEMORY_ONLY_MODE))
+ {
+ try {
+ DRMBase::ValidateHeaderFile(mb_dir + "_mabain_h", config.options,
+ config.queue_size * sizeof(AsyncNode), update_header);
+ } catch (int error) {
+ status = error;
+ return;
+ }
+ }
+}
+
+void DB::PostDBUpdate(const MBConfig &config, bool init_header, bool update_header)
+{
+ if((config.options & CONSTS::ACCESS_MODE_WRITER) && (init_header || update_header))
+ {
+ if(init_header)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "opened a new db %s", mb_dir.c_str());
+ }
+ else
+ {
+ Logger::Log(LOG_LEVEL_INFO, "converted %s to version %d.%d.%d", mb_dir.c_str(),
+ version[0], version[1], version[2]);
+ }
+ IndexHeader *header = dict->GetHeaderPtr();
+ if(header != NULL) header->async_queue_size = config.queue_size;
+ dict->Init(identifier);
+ dict->InitShmObjects();
+ }
+
+ if(dict->Status() != MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to iniitialize dict: %s ",
+ MBError::get_error_str(dict->Status()));
+ status = dict->Status();
+ return;
+ }
+
+ lock.Init(dict->GetShmLockPtrs());
+ UpdateNumHandlers(config.options, 1);
+
+ if(config.options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ if(config.options & CONSTS::ASYNC_WRITER_MODE)
+ async_writer = new AsyncWriter(this);
+ }
+
+#ifdef __SHM_QUEUE__
+ if(!(init_header || update_header))
+ {
+ IndexHeader *header = dict->GetHeaderPtr();
+ if(header != NULL && header->async_queue_size != (int) config.queue_size)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "async queue size not matching with header: %d %d",
+ header->async_queue_size, (int) config.queue_size);
+ status = MBError::INVALID_SIZE;
+ return;
+ }
+ }
+#endif
+
+ Logger::Log(LOG_LEVEL_INFO, "connector %u successfully opened DB %s for %s",
+ identifier, mb_dir.c_str(),
+ (config.options & CONSTS::ACCESS_MODE_WRITER) ? "writing":"reading");
+ status = MBError::SUCCESS;
+
+ if(config.options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ // Run rc exception recovery
+ ResourceCollection rc(*this);
+ rc.ExceptionRecovery();
+ }
+}
+
+void DB::InitDB(MBConfig &config)
+{
+ dict = NULL;
+ async_writer = NULL;
+
+ if(ValidateConfig(config) != MBError::SUCCESS)
+ return;
+
+ // save the configuration
+ memcpy(&dbConfig, &config, sizeof(MBConfig));
+ dbConfig.mbdir = NULL;
+
+ // If id not given, use thread ID
+ if(config.connect_id == 0)
+ {
+#ifdef __APPLE__
+ config.connect_id = reinterpret_cast<uint64_t>(pthread_self()) & 0x7FFFFFFF;
+#else
+ config.connect_id = static_cast<uint32_t>(syscall(SYS_gettid));
+#endif
+ }
+ identifier = config.connect_id;
+ mb_dir = std::string(config.mbdir);
+ if(mb_dir[mb_dir.length()-1] != '/')
+ mb_dir += "/";
+ options = config.options;
+
+ bool init_header = false;
+ bool update_header = false; // true when header version is different from lib version
+ PreCheckDB(config, init_header, update_header);
+ if(MBError::NOT_INITIALIZED != status)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "database %s check failed: %s", mb_dir.c_str(),
+ MBError::get_error_str(status));
+ return;
+ }
+
+ dict = new Dict(mb_dir, init_header, config.data_size, config.options,
+ config.memcap_index, config.memcap_data,
+ config.block_size_index, config.block_size_data,
+ config.max_num_index_block, config.max_num_data_block,
+ config.num_entry_per_bucket, config.queue_size);
+
+ PostDBUpdate(config, init_header, update_header);
+}
+
+int DB::Status() const
+{
+ return status;
+}
+
+DB::DB(const DB &db) : status(MBError::NOT_INITIALIZED),
+ writer_lock_fd(-1)
+{
+ MBConfig db_config = db.dbConfig;
+ db_config.mbdir = db.mb_dir.c_str();
+ InitDB(db_config);
+}
+
+const DB& DB::operator = (const DB &db)
+{
+ if(this == &db)
+ return *this; // no self-assignment
+
+ this->Close();
+
+ MBConfig db_config = db.dbConfig;
+ db_config.mbdir = db.mb_dir.c_str();
+ status = MBError::NOT_INITIALIZED;
+ writer_lock_fd = -1;
+ InitDB(db_config);
+
+ return *this;
+}
+
+bool DB::is_open() const
+{
+ return status == MBError::SUCCESS;
+}
+
+const char* DB::StatusStr() const
+{
+ return MBError::get_error_str(status);
+}
+
+// Find the exact key match
+int DB::Find(const char* key, int len, MBData &mdata) const
+{
+ if(key == NULL)
+ return MBError::INVALID_ARG;
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+ // Writer in async mode cannot be used for lookup
+ if(options & CONSTS::ASYNC_WRITER_MODE)
+ return MBError::NOT_ALLOWED;
+
+ return dict->Find(reinterpret_cast<const uint8_t*>(key), len, mdata);
+}
+
+int DB::Find(const std::string &key, MBData &mdata) const
+{
+ return Find(key.data(), key.size(), mdata);
+}
+
+// Find all possible prefix matches. The caller needs to call this function
+// repeatedly if data.next is true.
+int DB::FindPrefix(const char* key, int len, MBData &data) const
+{
+ if(key == NULL)
+ return MBError::INVALID_ARG;
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+ // Writer in async mode cannot be used for lookup
+ if(options & CONSTS::ASYNC_WRITER_MODE)
+ return MBError::NOT_ALLOWED;
+
+ if(data.match_len >= len)
+ return MBError::OUT_OF_BOUND;
+
+ int rval;
+ rval = dict->FindPrefix(reinterpret_cast<const uint8_t*>(key+data.match_len),
+ len-data.match_len, data);
+
+ return rval;
+}
+
+// Find the longest prefix match
+int DB::FindLongestPrefix(const char* key, int len, MBData &data) const
+{
+ if(key == NULL)
+ return MBError::INVALID_ARG;
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+ // Writer in async mode cannot be used for lookup
+ if(options & CONSTS::ASYNC_WRITER_MODE)
+ return MBError::NOT_ALLOWED;
+
+ data.match_len = 0;
+
+ return dict->FindPrefix(reinterpret_cast<const uint8_t*>(key), len, data);
+}
+
+int DB::FindLongestPrefix(const std::string &key, MBData &data) const
+{
+ return FindLongestPrefix(key.data(), key.size(), data);
+}
+
+// Add a key-value pair
+int DB::Add(const char* key, int len, MBData &mbdata, bool overwrite)
+{
+ int rval = MBError::SUCCESS;
+
+ if(key == NULL)
+ return MBError::INVALID_ARG;
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+
+#ifndef __SHM_QUEUE__
+ if(async_writer != NULL)
+ return async_writer->Add(key, len, reinterpret_cast<const char *>(mbdata.buff),
+ mbdata.data_len, overwrite);
+
+ rval = dict->Add(reinterpret_cast<const uint8_t*>(key), len, mbdata, overwrite);
+#else
+ if (async_writer == NULL && (options & CONSTS::ACCESS_MODE_WRITER))
+ {
+ rval = dict->Add(reinterpret_cast<const uint8_t*>(key), len, mbdata, overwrite);
+ }
+ else
+ {
+ rval = dict->SHMQ_Add(reinterpret_cast<const char*>(key), len,
+ reinterpret_cast<const char*>(mbdata.buff), mbdata.data_len, overwrite);
+ }
+#endif
+
+ return rval;
+}
+
+int DB::Add(const char* key, int len, const char* data, int data_len, bool overwrite)
+{
+ if(key == NULL || data == NULL)
+ return MBError::INVALID_ARG;
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+
+#ifndef __SHM_QUEUE__
+ if(async_writer != NULL)
+ return async_writer->Add(key, len, data, data_len, overwrite);
+#endif
+
+ MBData mbdata;
+ mbdata.data_len = data_len;
+ mbdata.buff = (uint8_t*) data;
+
+ int rval = MBError::SUCCESS;
+#ifndef __SHM_QUEUE__
+ rval = dict->Add(reinterpret_cast<const uint8_t*>(key), len, mbdata, overwrite);
+#else
+ if (async_writer == NULL && (options & CONSTS::ACCESS_MODE_WRITER))
+ {
+ rval = dict->Add(reinterpret_cast<const uint8_t*>(key), len, mbdata, overwrite);
+ }
+ else
+ {
+ rval = dict->SHMQ_Add(reinterpret_cast<const char*>(key), len,
+ reinterpret_cast<const char*>(mbdata.buff), mbdata.data_len, overwrite);
+ }
+#endif
+
+ mbdata.buff = NULL;
+ return rval;
+}
+
+int DB::Add(const std::string &key, const std::string &value, bool overwrite)
+{
+ return Add(key.data(), key.size(), value.data(), value.size(), overwrite);
+}
+
+int DB::Remove(const char *key, int len)
+{
+ int rval = MBError::SUCCESS;
+
+ if(key == NULL)
+ return MBError::INVALID_ARG;
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+
+#ifndef __SHM_QUEUE__
+ if(async_writer != NULL)
+ return async_writer->Remove(key, len);
+
+ rval = dict->Remove(reinterpret_cast<const uint8_t*>(key), len);
+#else
+ if (async_writer == NULL && (options & CONSTS::ACCESS_MODE_WRITER))
+ {
+ rval = dict->Remove(reinterpret_cast<const uint8_t*>(key), len);
+ }
+ else
+ {
+ rval = dict->SHMQ_Remove(reinterpret_cast<const char*>(key), len);
+ }
+#endif
+
+ return rval;
+}
+
+int DB::Remove(const std::string &key)
+{
+ return Remove(key.data(), key.size());
+}
+
+int DB::RemoveAll()
+{
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+
+#ifndef __SHM_QUEUE__
+ if(async_writer != NULL)
+ return async_writer->RemoveAll();
+#endif
+
+ int rval;
+ rval = dict->RemoveAll();
+ return rval;
+}
+
+int DB::Backup(const char *bk_dir)
+{
+ int rval = MBError::SUCCESS;
+
+ if(options & CONSTS::MEMORY_ONLY_MODE)
+ return MBError::NOT_ALLOWED;
+
+ if(bk_dir == NULL)
+ return MBError::INVALID_ARG;
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+ if(options & MMAP_ANONYMOUS_MODE)
+ return MBError::NOT_ALLOWED;
+
+#ifndef __SHM_QUEUE__
+ if(async_writer != NULL)
+ return async_writer->Backup(bk_dir);
+
+ try {
+ DBBackup bk(*this);
+ rval = bk.Backup(bk_dir);
+ } catch (int error) {
+ Logger::Log(LOG_LEVEL_WARN, "Backup failed :%s", MBError::get_error_str(error));
+ rval = error;
+ }
+#else
+ try {
+ if (async_writer == NULL && (options & CONSTS::ASYNC_WRITER_MODE))
+ {
+ DBBackup bk(*this);
+ rval = bk.Backup(bk_dir);
+ }
+ else
+ {
+ rval = dict->SHMQ_Backup(bk_dir);
+ }
+ } catch (int error) {
+ Logger::Log(LOG_LEVEL_WARN, "Backup failed :%s", MBError::get_error_str(error));
+ rval = error;
+ }
+#endif
+ return rval;
+}
+
+void DB::Flush() const
+{
+ if(options & CONSTS::MEMORY_ONLY_MODE)
+ return;
+
+ if(status != MBError::SUCCESS)
+ return;
+
+ dict->Flush();
+}
+
+int DB::CollectResource(int64_t min_index_rc_size, int64_t min_data_rc_size,
+ int64_t max_dbsz, int64_t max_dbcnt)
+{
+ if(status != MBError::SUCCESS)
+ return status;
+
+#ifndef __SHM_QUEUE__
+ if(async_writer != NULL)
+ return async_writer->CollectResource(min_index_rc_size, min_data_rc_size,
+ max_dbsz, max_dbcnt);
+
+ try {
+ ResourceCollection rc(*this);
+ rc.ReclaimResource(min_index_rc_size, min_data_rc_size, max_dbsz, max_dbcnt);
+ } catch (int error) {
+ if(error != MBError::RC_SKIPPED)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to run gc: %s",
+ MBError::get_error_str(error));
+ return error;
+ }
+ }
+#else
+ try {
+ if (async_writer == NULL && (options & CONSTS::ACCESS_MODE_WRITER))
+ {
+ ResourceCollection rc(*this);
+ rc.ReclaimResource(min_index_rc_size, min_data_rc_size, max_dbsz, max_dbcnt);
+ }
+ else
+ {
+ dict->SHMQ_CollectResource(min_index_rc_size, min_data_rc_size, max_dbsz, max_dbcnt);
+ }
+ } catch (int error) {
+ if(error != MBError::RC_SKIPPED)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to run gc: %s",
+ MBError::get_error_str(error));
+ return error;
+ }
+ }
+#endif
+ return MBError::SUCCESS;
+}
+
+int64_t DB::Count() const
+{
+ if(status != MBError::SUCCESS)
+ return -1;
+
+ return dict->Count();
+}
+
+void DB::PrintStats(std::ostream &out_stream) const
+{
+ if(status != MBError::SUCCESS)
+ return;
+
+ dict->PrintStats(out_stream);
+}
+
+void DB::PrintHeader(std::ostream &out_stream) const
+{
+ if(dict != NULL)
+ dict->PrintHeader(out_stream);
+}
+
+int DB::WrLock()
+{
+ return lock.WrLock();
+}
+
+int DB::RdLock()
+{
+ return lock.RdLock();
+}
+
+int DB::UnLock()
+{
+ return lock.UnLock();
+}
+
+int DB::TryWrLock()
+{
+ return lock.TryWrLock();
+}
+
+int DB::ClearLock() const
+{
+#ifdef __SHM_LOCK__
+ // No db handler should hold mutex when this is called.
+ if(status != MBError::SUCCESS)
+ return status;
+ IndexHeader *hdr = dict->GetHeaderPtr();
+ return InitShmRWLock(&hdr->mb_rw_lock);
+#else
+ // Nothing needs to be done if we don't use shared memory mutex.
+ return MBError::SUCCESS;
+#endif
+}
+
+int DB::SetLogLevel(int level)
+{
+ return Logger::SetLogLevel(level);
+}
+
+void DB::LogDebug()
+{
+ Logger::SetLogLevel(LOG_LEVEL_DEBUG);
+}
+
+Dict* DB::GetDictPtr() const
+{
+ if(options & CONSTS::ACCESS_MODE_WRITER)
+ return dict;
+ return NULL;
+}
+
+int DB::GetDBOptions() const
+{
+ return options;
+}
+
+const std::string& DB::GetDBDir() const
+{
+ return mb_dir;
+}
+
+void DB::GetDBConfig(MBConfig &config) const
+{
+ memcpy(&config, &dbConfig, sizeof(MBConfig));
+ config.mbdir = NULL;
+}
+
+int DB::SetAsyncWriterPtr(DB *db_writer)
+{
+#ifndef __SHM_QUEUE__
+ if(db_writer == NULL)
+ return MBError::INVALID_ARG;
+ if(options & CONSTS::ACCESS_MODE_WRITER)
+ return MBError::NOT_ALLOWED;
+ if(db_writer->mb_dir != mb_dir)
+ return MBError::INVALID_ARG;
+ if(!(db_writer->options & CONSTS::ACCESS_MODE_WRITER) ||
+ !(db_writer->options & CONSTS::ASYNC_WRITER_MODE) ||
+ db_writer->async_writer == NULL)
+ {
+ return MBError::INVALID_ARG;
+ }
+
+ db_writer->async_writer->UpdateNumUsers(1);
+ async_writer = db_writer->async_writer;
+#endif
+ return MBError::SUCCESS;
+}
+
+int DB::UnsetAsyncWriterPtr(DB *db_writer)
+{
+#ifndef __SHM_QUEUE__
+ if(db_writer == NULL)
+ return MBError::INVALID_ARG;
+ if(options & CONSTS::ACCESS_MODE_WRITER)
+ return MBError::NOT_ALLOWED;
+ if(db_writer->mb_dir != mb_dir)
+ return MBError::INVALID_ARG;
+ if(!(db_writer->options & CONSTS::ACCESS_MODE_WRITER) ||
+ !(db_writer->options & CONSTS::ASYNC_WRITER_MODE) ||
+ db_writer->async_writer == NULL)
+ {
+ return MBError::INVALID_ARG;
+ }
+
+ db_writer->async_writer->UpdateNumUsers(-1);
+ async_writer = NULL;
+#endif
+ return MBError::SUCCESS;
+}
+
+bool DB::AsyncWriterEnabled() const
+{
+#ifdef __SHM_QUEUE__
+ return true;
+#else
+ return (async_writer != NULL);
+#endif
+}
+
+bool DB::AsyncWriterBusy() const
+{
+#ifdef __SHM_QUEUE__
+ return dict->SHMQ_Busy();
+#else
+ if(async_writer != NULL)
+ return async_writer->Busy();
+ return true;
+#endif
+}
+
+void DB::SetLogFile(const std::string &log_file)
+{
+ Logger::InitLogFile(log_file);
+}
+
+void DB::CloseLogFile()
+{
+ Logger::Close();
+}
+
+void DB::ClearResources(const std::string &path)
+{
+ ResourcePool::getInstance().RemoveResourceByDB(path);
+}
+
+} // namespace mabain
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __DB_H__
+#define __DB_H__
+
+#include <iostream>
+#include <string>
+
+#include "mb_data.h"
+#include "error.h"
+#include "lock.h"
+#include "integer_4b_5b.h"
+
+namespace mabain {
+
+
+#define MB_MAX_NUM_SHM_QUEUE_NODE 64
+
+class Dict;
+class MBlsq;
+class LockFree;
+class AsyncWriter;
+struct _DBTraverseNode;
+
+typedef struct _MBConfig
+{
+ const char *mbdir;
+ int options;
+ size_t memcap_index;
+ size_t memcap_data;
+ int data_size;
+ uint32_t connect_id;
+ uint32_t block_size_index;
+ uint32_t block_size_data;
+ int max_num_data_block;
+ int max_num_index_block;
+
+ // For automatic eviction
+ // All entries in the oldest buckets will be pruned.
+ int num_entry_per_bucket;
+ uint32_t queue_size;
+} MBConfig;
+
+// Database handle class
+class DB
+{
+public:
+ // DB iterator class as an inner class
+ class iterator
+ {
+ friend class DBTraverseBase;
+
+ public:
+ std::string key;
+ MBData value;
+ int match;
+
+ iterator(const DB &db, int iter_state);
+ // Copy constructor
+ iterator(const iterator &rhs);
+ void init(bool check_async_mode = true);
+ int init_no_next();
+ ~iterator();
+
+ // operator overloading
+ bool operator!=(const iterator &rhs);
+ const iterator& operator++();
+
+ private:
+ int get_node_offset(const std::string &node_key, size_t &parent_edge_off,
+ size_t &node_offset);
+ int load_node(const std::string &curr_node_key, size_t &parent_edge_off);
+ int load_kv_for_node(const std::string &curr_node_key);
+ int load_kvs(const std::string &curr_node_key, MBlsq *chid_node_list);
+ void iter_obj_init();
+ bool next_dbt_buffer(struct _DBTraverseNode *dbt_n);
+ void add_node_offset(size_t node_offset);
+ iterator* next();
+
+ const DB &db_ref;
+ int state;
+ EdgePtrs edge_ptrs;
+ // temp buffer to hold the node
+ uint8_t node_buff[NUM_ALPHABET+NODE_EDGE_KEY_FIRST];
+ MBlsq *node_stack;
+ MBlsq *kv_per_node;
+ LockFree *lfree;
+ };
+
+ // db_path: database directory
+ // db_options: db access option (read/write)
+ // memcap_index: maximum memory size in bytes for key index
+ // memcap_data: maximum memory size for data file mapping
+ // data_size: the value size; if zero, the value size will be variable.
+ // id: the connector id
+ DB(const char *db_path, int db_options, size_t memcap_index = 64*1024*1024LL,
+ size_t memcap_data = 64*1024*1024LL, uint32_t id = 0, uint32_t queue_size = MB_MAX_NUM_SHM_QUEUE_NODE);
+ DB(MBConfig &config);
+ ~DB();
+
+ DB(const DB &db);
+ const DB& operator = (const DB &db);
+
+ // Add a key-value pair
+ int Add(const char* key, int len, const char* data, int data_len, bool overwrite = false);
+ int Add(const char* key, int len, MBData &data, bool overwrite = false);
+ int Add(const std::string &key, const std::string &value, bool overwrite = false);
+ // Find an entry by exact match using a key
+ int Find(const char* key, int len, MBData &mdata) const;
+ int Find(const std::string &key, MBData &mdata) const;
+ // Find all possible prefix matches using a key
+ // This is not fully implemented yet.
+ int FindPrefix(const char* key, int len, MBData &data) const;
+ // Find the longest prefix match using a key
+ int FindLongestPrefix(const char* key, int len, MBData &data) const;
+ int FindLongestPrefix(const std::string &key, MBData &data) const;
+ // Remove an entry using a key
+ int Remove(const char *key, int len);
+ int Remove(const std::string &key);
+ int RemoveAll();
+ // DB Backup
+ int Backup(const char *backup_dir);
+
+ // Close the DB handle
+ int Close();
+ void Flush() const;
+ static void ClearResources(const std::string &path);
+
+ // Garbage collection
+ // min_index_rc_size and min_data_rc_size are the threshold for trigering garbage
+ // collector. If the pending index buffer size is less than min_index_rc_size,
+ // rc will be ignored for index segment. If the pending data buffer size is less
+ // than min_data_rc_size, rc will be ignored for data segment.
+ // eviction will be ignored if db size is less than 0xFFFFFFFFFFFF and db count is
+ // less than 0xFFFFFFFFFFFF.
+ int CollectResource(int64_t min_index_rc_size = 33554432 , int64_t min_data_rc_size = 33554432,
+ int64_t max_dbsiz = MAX_6B_OFFSET, int64_t max_dbcnt = MAX_6B_OFFSET);
+
+ // Multi-thread update using async thread
+ // FOR THIS TO WORK, WRITER MUST BE THE LAST ONE TO CLOSE HANDLE.
+ int SetAsyncWriterPtr(DB *db_writer);
+ int UnsetAsyncWriterPtr(DB *db_writer);
+ bool AsyncWriterEnabled() const;
+ bool AsyncWriterBusy() const;
+
+ // multi-thread or multi-process locking for DB management
+ int WrLock();
+ int RdLock();
+ int UnLock();
+ int TryWrLock();
+ int ClearLock() const;
+
+ int UpdateNumHandlers(int mode, int delta);
+
+ // level 0: only error will be logged.
+ // level 1: error and warn will be logged.
+ // level 2: error, warn, and info will be logged. This is the default setting.
+ // level 3: error, warn, info and debug will be logged.
+ static void SetLogFile(const std::string &log_file);
+ static int SetLogLevel(int level);
+ static void LogDebug();
+ static void CloseLogFile();
+
+ // Print database stats
+ void PrintStats(std::ostream &out_stream = std::cout) const;
+ void PrintHeader(std::ostream &out_stream = std::cout) const;
+ // current count of key-value pair
+ int64_t Count() const;
+ // DB status
+ int Status() const;
+ // DB status string
+ const char *StatusStr() const;
+ // Check if DB is opened successfully
+ bool is_open() const;
+
+ Dict* GetDictPtr() const;
+ int GetDBOptions() const;
+ const std::string& GetDBDir() const;
+
+ void GetDBConfig(MBConfig &config) const;
+
+ //iterator
+ const iterator begin(bool check_async_mode = true, bool rc_mode = false) const;
+ const iterator end() const;
+
+private:
+ void InitDB(MBConfig &config);
+ void PreCheckDB(const MBConfig &config, bool &init_header, bool &update_header);
+ void PostDBUpdate(const MBConfig &config, bool init_header, bool update_header);
+ static int ValidateConfig(MBConfig &config);
+
+ // DB directory
+ std::string mb_dir;
+ int options;
+ Dict *dict;
+ int status;
+
+ // DB connector ID
+ uint32_t identifier;
+
+ // db lock
+ MBLock lock;
+ MBConfig dbConfig;
+
+ AsyncWriter *async_writer;
+
+ int writer_lock_fd;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <stdlib.h>
+#include <iostream>
+#include <errno.h>
+
+#include "mabain_consts.h"
+#include "db.h"
+#include "dict.h"
+#include "dict_mem.h"
+#include "error.h"
+#include "integer_4b_5b.h"
+#include "async_writer.h"
+#include "util/shm_mutex.h"
+
+#define MAX_DATA_BUFFER_RESERVE_SIZE 0xFFFF
+#define NUM_DATA_BUFFER_RESERVE MAX_DATA_BUFFER_RESERVE_SIZE/DATA_BUFFER_ALIGNMENT
+#define DATA_HEADER_SIZE 32
+
+#define READER_LOCK_FREE_START \
+ LockFreeData snapshot; \
+ int lf_ret; \
+ lfree.ReaderLockFreeStart(snapshot);
+#define READER_LOCK_FREE_STOP(edgeoff, data) \
+ lf_ret = lfree.ReaderLockFreeStop(snapshot, (edgeoff), (data)); \
+ if(lf_ret != MBError::SUCCESS) \
+ return lf_ret; \
+
+namespace mabain {
+
+Dict::Dict(const std::string &mbdir, bool init_header, int datasize,
+ int db_options, size_t memsize_index, size_t memsize_data,
+ uint32_t block_sz_idx, uint32_t block_sz_data,
+ int max_num_index_blk, int max_num_data_blk,
+ int64_t entry_per_bucket, uint32_t queue_size)
+ : options(db_options),
+#ifdef __SHM_QUEUE__
+ mm(mbdir, init_header, memsize_index, db_options, block_sz_idx, max_num_index_blk, queue_size),
+ queue(NULL)
+#else
+ mm(mbdir, init_header, memsize_index, db_options, block_sz_idx, max_num_index_blk, queue_size)
+#endif
+{
+ status = MBError::NOT_INITIALIZED;
+ reader_rc_off = 0;
+
+ header = mm.GetHeaderPtr();
+ if(header == NULL)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "header not mapped");
+ throw (int) MBError::MMAP_FAILED;
+ }
+
+ // confirm block size is the same
+ if(!init_header)
+ {
+ if(block_sz_data != 0 && header->data_block_size != block_sz_data)
+ {
+ std::cerr << "mabain data block size not match " << block_sz_data << ": "
+ << header->data_block_size << std::endl;
+ PrintHeader(std::cout);
+ Destroy();
+ throw (int) MBError::INVALID_SIZE;
+ }
+ }
+ else
+ {
+ header->data_block_size = block_sz_data;
+ }
+
+ lfree.LockFreeInit(&header->lock_free, header, db_options);
+ mm.InitLockFreePtr(&lfree);
+
+#ifdef __SHM_QUEUE__
+ // initialize shared memory queue
+ char *hdr_ptr = (char *) header;
+ queue = reinterpret_cast<AsyncNode *>(hdr_ptr + RollableFile::page_size);
+#endif
+
+ // Open data file
+ kv_file = new RollableFile(mbdir + "_mabain_d",
+ static_cast<size_t>(header->data_block_size),
+ memsize_data, db_options, max_num_data_blk);
+
+ kv_file->InitShmSlidingAddr(&header->shm_data_sliding_start);
+ // If init_header is false, we can set the dict status to SUCCESS.
+ // Otherwise, the status will be set in the Init.
+ if(init_header)
+ {
+ // Initialize header
+ header->entry_per_bucket = entry_per_bucket;
+ header->index_block_size = block_sz_idx;
+ header->data_block_size = block_sz_data;
+ header->data_size = datasize;
+ header->count = 0;
+ header->m_data_offset = GetStartDataOffset(); // start from a non-zero offset
+ // We known that only writers will set init_header to true.
+ free_lists = new FreeList(mbdir+"_dbfl", DATA_BUFFER_ALIGNMENT,
+ NUM_DATA_BUFFER_RESERVE);
+ }
+ else
+ {
+ if(options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ if(header->entry_per_bucket != entry_per_bucket)
+ {
+ Destroy();
+ std::cerr << "mabain count per bucket not match\n";
+ throw (int) MBError::INVALID_SIZE;
+ }
+ mm.ResetSlidingWindow();
+ ResetSlidingWindow();
+ free_lists = new FreeList(mbdir+"_dbfl", DATA_BUFFER_ALIGNMENT,
+ NUM_DATA_BUFFER_RESERVE);
+ if(mm.IsValid())
+ {
+ int rval = ExceptionRecovery();
+ if(rval == MBError::SUCCESS)
+ {
+ header->excep_lf_offset = 0;
+ header->excep_offset = 0;
+ status = MBError::SUCCESS;
+ }
+ }
+ }
+ else
+ {
+ if(mm.IsValid())
+ status = MBError::SUCCESS;
+ }
+ }
+}
+
+Dict::~Dict()
+{
+}
+
+// This function only needs to be called by writer.
+int Dict::Init(uint32_t id)
+{
+ if(!(options & CONSTS::ACCESS_MODE_WRITER))
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "dict initialization not allowed for non-writer");
+ return MBError::NOT_ALLOWED;
+ }
+
+ if(status != MBError::NOT_INITIALIZED)
+ {
+ // status can be NOT_INITIALIZED or SUCCESS.
+ Logger::Log(LOG_LEVEL_WARN, "connector %u dict already initialized", id);
+ return MBError::SUCCESS;
+ }
+
+ if(header == NULL)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "connector %u header not mapped", id);
+ return MBError::ALLOCATION_ERROR;
+ }
+
+ Logger::Log(LOG_LEVEL_INFO, "connector %u initializing DictMem", id);
+ mm.InitRootNode();
+
+ if(header->data_size > CONSTS::MAX_DATA_SIZE)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "data size %d is too large", header->data_size);
+ return MBError::INVALID_SIZE;
+ }
+
+ if(mm.IsValid())
+ status = MBError::SUCCESS;
+
+ return status;
+}
+
+void Dict::Destroy()
+{
+ if(options & CONSTS::ACCESS_MODE_WRITER)
+ {
+ mm.ResetSlidingWindow();
+ ResetSlidingWindow();
+ }
+
+ mm.Destroy();
+
+ if(free_lists != NULL)
+ delete free_lists;
+
+ if(kv_file != NULL)
+ delete kv_file;
+}
+
+int Dict::Status() const
+{
+ return status;
+}
+
+// Add a key-value pair
+// if overwrite is true and an entry with input key already exists, the old data will
+// be overwritten. Otherwise, IN_DICT will be returned.
+int Dict::Add(const uint8_t *key, int len, MBData &data, bool overwrite)
+{
+ if(!(options & CONSTS::ACCESS_MODE_WRITER))
+ {
+ return MBError::NOT_ALLOWED;
+ }
+ if(len > CONSTS::MAX_KEY_LENGHTH || data.data_len > CONSTS::MAX_DATA_SIZE ||
+ len <= 0 || data.data_len <= 0)
+ return MBError::OUT_OF_BOUND;
+
+ EdgePtrs edge_ptrs;
+ size_t data_offset = 0;
+ int rval;
+
+ rval = mm.GetRootEdge_Writer(data.options & CONSTS::OPTION_RC_MODE, key[0], edge_ptrs);
+ if(rval != MBError::SUCCESS)
+ return rval;
+
+ if(edge_ptrs.len_ptr[0] == 0)
+ {
+ ReserveData(data.buff, data.data_len, data_offset);
+ // Add the first edge along this edge
+ mm.AddRootEdge(edge_ptrs, key, len, data_offset);
+ if(data.options & CONSTS::OPTION_RC_MODE)
+ {
+ header->rc_count++;
+ }
+ else
+ {
+ header->count++;
+ header->num_update++;
+ }
+
+ return MBError::SUCCESS;
+ }
+
+ bool inc_count = true;
+ int i;
+ const uint8_t *key_buff;
+ uint8_t tmp_key_buff[NUM_ALPHABET];
+ const uint8_t *p = key;
+ int edge_len = edge_ptrs.len_ptr[0];
+ if(edge_len > LOCAL_EDGE_LEN)
+ {
+ if(mm.ReadData(tmp_key_buff, edge_len-1, Get5BInteger(edge_ptrs.ptr)) != edge_len-1)
+ return MBError::READ_ERROR;
+ key_buff = tmp_key_buff;
+ }
+ else
+ {
+ key_buff = edge_ptrs.ptr;
+ }
+ if(edge_len < len)
+ {
+ for(i = 1; i < edge_len; i++)
+ {
+ if(key_buff[i-1] != key[i])
+ break;
+ }
+ if(i >= edge_len)
+ {
+ int match_len;
+ bool next;
+ p += edge_len;
+ len -= edge_len;
+ while((next = mm.FindNext(p, len, match_len, edge_ptrs, tmp_key_buff)))
+ {
+ if(match_len < edge_ptrs.len_ptr[0])
+ break;
+
+ p += match_len;
+ len -= match_len;
+ if(len <= 0)
+ break;
+ }
+ if(!next)
+ {
+ ReserveData(data.buff, data.data_len, data_offset);
+ rval = mm.UpdateNode(edge_ptrs, p, len, data_offset);
+ }
+ else if(match_len < static_cast<int>(edge_ptrs.len_ptr[0]))
+ {
+ if(len > match_len)
+ {
+ ReserveData(data.buff, data.data_len, data_offset);
+ rval = mm.AddLink(edge_ptrs, match_len, p+match_len, len-match_len,
+ data_offset, data);
+ }
+ else if(len == match_len)
+ {
+ ReserveData(data.buff, data.data_len, data_offset);
+ rval = mm.InsertNode(edge_ptrs, match_len, data_offset, data);
+ }
+ }
+ else if(len == 0)
+ {
+ rval = UpdateDataBuffer(edge_ptrs, overwrite, data.buff, data.data_len, inc_count);
+ }
+ }
+ else
+ {
+ ReserveData(data.buff, data.data_len, data_offset);
+ rval = mm.AddLink(edge_ptrs, i, p+i, len-i, data_offset, data);
+ }
+ }
+ else
+ {
+ for(i = 1; i < len; i++)
+ {
+ if(key_buff[i-1] != key[i])
+ break;
+ }
+ if(i < len)
+ {
+ ReserveData(data.buff, data.data_len, data_offset);
+ rval = mm.AddLink(edge_ptrs, i, p+i, len-i, data_offset, data);
+ }
+ else
+ {
+ if(edge_ptrs.len_ptr[0] > len)
+ {
+ ReserveData(data.buff, data.data_len, data_offset);
+ rval = mm.InsertNode(edge_ptrs, i, data_offset, data);
+ }
+ else
+ {
+ rval = UpdateDataBuffer(edge_ptrs, overwrite, data.buff, data.data_len, inc_count);
+ }
+ }
+ }
+
+ if(data.options & CONSTS::OPTION_RC_MODE)
+ {
+ if(rval == MBError::SUCCESS)
+ header->rc_count++;
+ }
+ else
+ {
+ if(rval == MBError::SUCCESS)
+ header->num_update++;
+ if(inc_count)
+ header->count++;
+ }
+ return rval;
+}
+
+int Dict::ReadDataFromEdge(MBData &data, const EdgePtrs &edge_ptrs) const
+{
+ size_t data_off;
+ if(edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ data_off = Get6BInteger(edge_ptrs.offset_ptr);
+ }
+ else
+ {
+ uint8_t node_buff[NODE_EDGE_KEY_FIRST];
+ if(mm.ReadData(node_buff, NODE_EDGE_KEY_FIRST, Get6BInteger(edge_ptrs.offset_ptr))
+ != NODE_EDGE_KEY_FIRST)
+ return MBError::READ_ERROR;
+ if(!(node_buff[0] & FLAG_NODE_MATCH))
+ return MBError::NOT_EXIST;
+ data_off = Get6BInteger(node_buff+2);
+ }
+ data.data_offset = data_off;
+
+ uint16_t data_len[2];
+ // Read data length first
+ if(ReadData(reinterpret_cast<uint8_t*>(&data_len[0]), DATA_HDR_BYTE, data_off)
+ != DATA_HDR_BYTE)
+ return MBError::READ_ERROR;
+ data_off += DATA_HDR_BYTE;
+ if(data.buff_len < data_len[0] + 1)
+ {
+ if(data.Resize(data_len[0]) != MBError::SUCCESS)
+ return MBError::NO_MEMORY;
+ }
+ if(ReadData(data.buff, data_len[0], data_off) != data_len[0])
+ return MBError::READ_ERROR;
+
+ data.data_len = data_len[0];
+ data.bucket_index = data_len[1];
+ return MBError::SUCCESS;
+}
+
+// Delete operations:
+// If this is a leaf node, need to remove the edge. Otherwise, unset the match flag.
+// Also need to set the delete flag in the data block so that it can be reclaimed later.
+int Dict::DeleteDataFromEdge(MBData &data, EdgePtrs &edge_ptrs)
+{
+ int rval = MBError::SUCCESS;
+ size_t data_off;
+ uint16_t data_len;
+ int rel_size;
+
+ // Check if this is a leaf node first by using the EDGE_FLAG_DATA_OFF bit
+ if(edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ data_off = Get6BInteger(edge_ptrs.offset_ptr);
+ if(ReadData(reinterpret_cast<uint8_t*>(&data_len), DATA_SIZE_BYTE, data_off)
+ != DATA_SIZE_BYTE)
+ return MBError::READ_ERROR;
+
+ rel_size = free_lists->GetAlignmentSize(data_len + DATA_HDR_BYTE);
+ header->pending_data_buff_size += rel_size;
+ free_lists->ReleaseBuffer(data_off, rel_size);
+
+ rval = mm.RemoveEdgeByIndex(edge_ptrs, data);
+ }
+ else
+ {
+ // No exception handling in this case
+ header->excep_lf_offset = 0;
+ header->excep_offset = 0;
+
+ uint8_t node_buff[NODE_EDGE_KEY_FIRST];
+ size_t node_off = Get6BInteger(edge_ptrs.offset_ptr);
+
+ // Read node header
+ if(mm.ReadData(node_buff, NODE_EDGE_KEY_FIRST, node_off) != NODE_EDGE_KEY_FIRST)
+ return MBError::READ_ERROR;
+
+ if(node_buff[0] & FLAG_NODE_MATCH)
+ {
+ // Unset the match flag
+ node_buff[0] &= ~FLAG_NODE_MATCH;
+ mm.WriteData(&node_buff[0], 1, node_off);
+
+ // Release data buffer
+ data_off = Get6BInteger(node_buff+2);
+ if(ReadData(reinterpret_cast<uint8_t*>(&data_len), DATA_SIZE_BYTE, data_off)
+ != DATA_SIZE_BYTE)
+ return MBError::READ_ERROR;
+
+ rel_size = free_lists->GetAlignmentSize(data_len + DATA_HDR_BYTE);
+ header->pending_data_buff_size += rel_size;
+ free_lists->ReleaseBuffer(data_off, rel_size);
+ }
+ else
+ {
+ rval = MBError::NOT_EXIST;
+ }
+ }
+
+ return rval;
+}
+
+int Dict::ReadDataFromNode(MBData &data, const uint8_t *node_ptr) const
+{
+ size_t data_off = Get6BInteger(node_ptr+2);
+ if(data_off == 0)
+ return MBError::NOT_EXIST;
+
+ data.data_offset = data_off;
+
+ // Read data length first
+ uint16_t data_len[2];
+ if(ReadData(reinterpret_cast<uint8_t *>(&data_len[0]), DATA_HDR_BYTE, data_off)
+ != DATA_HDR_BYTE)
+ return MBError::READ_ERROR;
+ data_off += DATA_HDR_BYTE;
+
+ if(data.buff_len < data_len[0] + 1)
+ {
+ if(data.Resize(data_len[0]) != MBError::SUCCESS)
+ return MBError::NO_MEMORY;
+ }
+ if(ReadData(data.buff, data_len[0], data_off) != data_len[0])
+ return MBError::READ_ERROR;
+
+ data.data_len = data_len[0];
+ data.bucket_index = data_len[1];
+ return MBError::SUCCESS;
+}
+
+int Dict::FindPrefix(const uint8_t *key, int len, MBData &data)
+{
+ int rval;
+ MBData data_rc;
+ size_t rc_root_offset = header->rc_root_offset.load(MEMORY_ORDER_READER);
+ if(rc_root_offset != 0)
+ {
+ reader_rc_off = rc_root_offset;
+ rval = FindPrefix_Internal(rc_root_offset, key, len, data_rc);
+#ifdef __LOCK_FREE__
+ while(rval == MBError::TRY_AGAIN)
+ {
+ nanosleep((const struct timespec[]){{0, 10L}}, NULL);
+ data_rc.Clear();
+ rval = FindPrefix_Internal(rc_root_offset, key, len, data_rc);
+ }
+#endif
+ if(rval != MBError::NOT_EXIST && rval != MBError::SUCCESS)
+ return rval;
+ data.options &= ~(CONSTS::OPTION_RC_MODE | CONSTS::OPTION_READ_SAVED_EDGE);
+ }
+ else
+ {
+ if(reader_rc_off != 0)
+ {
+ reader_rc_off = 0;
+ RemoveUnused(0);
+ mm.RemoveUnused(0);
+ }
+ }
+
+ rval = FindPrefix_Internal(0, key, len, data);
+#ifdef __LOCK_FREE__
+ while(rval == MBError::TRY_AGAIN)
+ {
+ nanosleep((const struct timespec[]){{0, 10L}}, NULL);
+ data.Clear();
+ rval = FindPrefix_Internal(0, key, len, data);
+ }
+#endif
+
+ // The longer match wins.
+ if(data_rc.match_len > data.match_len)
+ {
+ data_rc.TransferValueTo(data.buff, data.data_len);
+ rval = MBError::SUCCESS;
+ }
+ return rval;
+
+}
+
+int Dict::FindPrefix_Internal(size_t root_off, const uint8_t *key, int len, MBData &data)
+{
+ int rval;
+ data.next = false;
+ EdgePtrs &edge_ptrs = data.edge_ptrs;
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_START
+#endif
+
+ if(data.match_len == 0)
+ {
+ rval = mm.GetRootEdge(data.options & CONSTS::OPTION_RC_MODE, key[0], edge_ptrs);
+ if(rval != MBError::SUCCESS)
+ return MBError::READ_ERROR;
+
+ if(edge_ptrs.len_ptr[0] == 0)
+ {
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return MBError::NOT_EXIST;
+ }
+ }
+
+ // Compare edge string
+ const uint8_t *key_buff;
+ uint8_t *node_buff = data.node_buff;
+ const uint8_t *p = key;
+ int edge_len = edge_ptrs.len_ptr[0];
+ int edge_len_m1 = edge_len - 1;
+ if(edge_len > LOCAL_EDGE_LEN)
+ {
+ if(mm.ReadData(node_buff, edge_len_m1, Get5BInteger(edge_ptrs.ptr))
+ != edge_len_m1)
+ {
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return MBError::READ_ERROR;
+ }
+ key_buff = node_buff;
+ }
+ else
+ {
+ key_buff = edge_ptrs.ptr;
+ }
+
+ rval = MBError::NOT_EXIST;
+ if(edge_len < len)
+ {
+ if(edge_len > 1 && memcmp(key_buff, key+1, edge_len_m1) != 0)
+ {
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return MBError::NOT_EXIST;
+ }
+
+ len -= edge_len;
+ p += edge_len;
+
+ if(edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ // prefix match for leaf node
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ data.match_len = p - key;
+ return ReadDataFromEdge(data, edge_ptrs);
+ }
+
+ uint8_t last_node_buffer[NODE_EDGE_KEY_FIRST];
+#ifdef __LOCK_FREE__
+ size_t edge_offset_prev = edge_ptrs.offset;
+#endif
+ int last_prefix_rval = MBError::NOT_EXIST;
+ while(true)
+ {
+ rval = mm.NextEdge(p, edge_ptrs, node_buff, data);
+ if(rval != MBError::READ_ERROR)
+ {
+ if(node_buff[0] & FLAG_NODE_MATCH)
+ {
+ data.match_len = p - key;
+ if(data.options & CONSTS::OPTION_ALL_PREFIX)
+ {
+ rval = ReadDataFromNode(data, node_buff);
+ data.next = true;
+ rval = last_prefix_rval;
+ break;
+ }
+ else
+ {
+ memcpy(last_node_buffer, node_buff, NODE_EDGE_KEY_FIRST);
+ last_prefix_rval = MBError::SUCCESS;
+ }
+ }
+ }
+
+ if(rval != MBError::SUCCESS)
+ break;
+
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_offset_prev, data)
+#endif
+ edge_len = edge_ptrs.len_ptr[0];
+ edge_len_m1 = edge_len - 1;
+ // match edge string
+ if(edge_len > LOCAL_EDGE_LEN)
+ {
+ if(mm.ReadData(node_buff, edge_len_m1, Get5BInteger(edge_ptrs.ptr))
+ != edge_len_m1)
+ {
+ rval = MBError::READ_ERROR;
+ break;
+ }
+ key_buff = node_buff;
+ }
+ else
+ {
+ key_buff = edge_ptrs.ptr;
+ }
+
+ if((edge_len > 1 && memcmp(key_buff, p+1, edge_len_m1) != 0) || edge_len == 0)
+ {
+ rval = MBError::NOT_EXIST;
+ break;
+ }
+
+ len -= edge_len;
+ p += edge_len;
+ if(len <= 0 || (edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF))
+ {
+ data.match_len = p - key;
+ rval = ReadDataFromEdge(data, edge_ptrs);
+ break;
+ }
+#ifdef __LOCK_FREE__
+ edge_offset_prev = edge_ptrs.offset;
+#endif
+ }
+
+ if(rval == MBError::NOT_EXIST && last_prefix_rval != rval)
+ rval = ReadDataFromNode(data, last_node_buffer);
+ }
+ else if(edge_len == len)
+ {
+ if(edge_len_m1 == 0 || memcmp(key_buff, key+1, edge_len_m1) == 0)
+ {
+ data.match_len = len;
+ rval = ReadDataFromEdge(data, edge_ptrs);
+ }
+ }
+
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return rval;
+}
+
+int Dict::Find(const uint8_t *key, int len, MBData &data)
+{
+ int rval;
+ size_t rc_root_offset = header->rc_root_offset.load(MEMORY_ORDER_READER);
+
+ if(rc_root_offset != 0)
+ {
+ reader_rc_off = rc_root_offset;
+ rval = Find_Internal(rc_root_offset, key, len, data);
+#ifdef __LOCK_FREE__
+ while(rval == MBError::TRY_AGAIN)
+ {
+ nanosleep((const struct timespec[]){{0, 10L}}, NULL);
+ rval = Find_Internal(rc_root_offset, key, len, data);
+ }
+#endif
+ if(rval == MBError::SUCCESS)
+ {
+ data.match_len = len;
+ return rval;
+ }
+ else if(rval != MBError::NOT_EXIST)
+ return rval;
+ data.options &= ~(CONSTS::OPTION_RC_MODE | CONSTS::OPTION_READ_SAVED_EDGE);
+ }
+ else
+ {
+ if(reader_rc_off != 0)
+ {
+ reader_rc_off = 0;
+ RemoveUnused(0);
+ mm.RemoveUnused(0);
+ }
+ }
+
+ rval = Find_Internal(0, key, len, data);
+#ifdef __LOCK_FREE__
+ while(rval == MBError::TRY_AGAIN)
+ {
+ nanosleep((const struct timespec[]){{0, 10L}}, NULL);
+ rval = Find_Internal(0, key, len, data);
+ }
+#endif
+ if(rval == MBError::SUCCESS)
+ data.match_len = len;
+
+ return rval;
+}
+
+int Dict::Find_Internal(size_t root_off, const uint8_t *key, int len, MBData &data)
+{
+ EdgePtrs &edge_ptrs = data.edge_ptrs;
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_START
+#endif
+ int rval;
+ rval = mm.GetRootEdge(root_off, key[0], edge_ptrs);
+
+ if(rval != MBError::SUCCESS)
+ return MBError::READ_ERROR;
+ if(edge_ptrs.len_ptr[0] == 0)
+ {
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return MBError::NOT_EXIST;
+ }
+
+ // Compare edge string
+ const uint8_t *key_buff;
+ uint8_t *node_buff = data.node_buff;
+ const uint8_t *p = key;
+ int edge_len = edge_ptrs.len_ptr[0];
+ int edge_len_m1 = edge_len - 1;
+
+ rval = MBError::NOT_EXIST;
+ if(edge_len > LOCAL_EDGE_LEN)
+ {
+ size_t edge_str_off_lf = Get5BInteger(edge_ptrs.ptr);
+ if(mm.ReadData(node_buff, edge_len_m1, edge_str_off_lf) != edge_len_m1)
+ {
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return MBError::READ_ERROR;
+ }
+ key_buff = node_buff;
+ }
+ else
+ {
+ key_buff = edge_ptrs.ptr;
+ }
+
+ if(edge_len < len)
+ {
+ if((edge_len > 1 && memcmp(key_buff, key+1, edge_len_m1) != 0) ||
+ (edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF))
+ {
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return MBError::NOT_EXIST;
+ }
+
+ len -= edge_len;
+ p += edge_len;
+
+#ifdef __LOCK_FREE__
+ size_t edge_offset_prev = edge_ptrs.offset;
+#endif
+ while(true)
+ {
+ rval = mm.NextEdge(p, edge_ptrs, node_buff, data);
+ if(rval != MBError::SUCCESS)
+ break;
+
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_offset_prev, data)
+#endif
+ edge_len = edge_ptrs.len_ptr[0];
+ edge_len_m1 = edge_len - 1;
+ // match edge string
+ if(edge_len > LOCAL_EDGE_LEN)
+ {
+ size_t edge_str_off_lf = Get5BInteger(edge_ptrs.ptr);
+ if(mm.ReadData(node_buff, edge_len_m1, edge_str_off_lf) != edge_len_m1)
+ {
+ rval = MBError::READ_ERROR;
+ break;
+ }
+ key_buff = node_buff;
+ }
+ else
+ {
+ key_buff = edge_ptrs.ptr;
+ }
+
+ if((edge_len_m1 > 0 && memcmp(key_buff, p+1, edge_len_m1) != 0) || edge_len_m1 < 0)
+ {
+ rval = MBError::NOT_EXIST;
+ break;
+ }
+
+ len -= edge_len;
+ if(len <= 0)
+ {
+ // If this is for remove operation, return IN_DICT to caller.
+ if(data.options & CONSTS::OPTION_FIND_AND_STORE_PARENT)
+ rval = MBError::IN_DICT;
+ else
+ rval = ReadDataFromEdge(data, edge_ptrs);
+ break;
+ }
+ else
+ {
+ if(edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ // Reach a leaf node and no match found
+ rval = MBError::NOT_EXIST;
+ break;
+ }
+ }
+ p += edge_len;
+#ifdef __LOCK_FREE__
+ edge_offset_prev = edge_ptrs.offset;
+#endif
+ }
+ }
+ else if(edge_len == len)
+ {
+ if(len > 1 && memcmp(key_buff, key+1, len-1) != 0)
+ {
+ rval = MBError::NOT_EXIST;
+ }
+ else
+ {
+ // If this is for remove operation, return IN_DICT to caller.
+ if(data.options & CONSTS::OPTION_FIND_AND_STORE_PARENT)
+ {
+ data.edge_ptrs.curr_node_offset = mm.GetRootOffset();
+ data.edge_ptrs.curr_nt = 1;
+ data.edge_ptrs.curr_edge_index = 0;
+ data.edge_ptrs.parent_offset = data.edge_ptrs.offset;
+ rval = MBError::IN_DICT;
+ }
+ else
+ {
+ rval = ReadDataFromEdge(data, edge_ptrs);
+ }
+ }
+ }
+
+#ifdef __LOCK_FREE__
+ READER_LOCK_FREE_STOP(edge_ptrs.offset, data)
+#endif
+ return rval;
+}
+
+void Dict::PrintStats(std::ostream *out_stream) const
+{
+ if(out_stream != NULL)
+ return PrintStats(*out_stream);
+ return PrintStats(std::cout);
+}
+
+void Dict::PrintStats(std::ostream &out_stream) const
+{
+ if(status != MBError::SUCCESS)
+ return;
+
+ out_stream << "DB stats:\n";
+ out_stream << "\tNumber of DB writer: " << header->num_writer << std::endl;
+ out_stream << "\tNumber of DB reader: " << header->num_reader << std::endl;
+ out_stream << "\tEntry count in DB: " << header->count << std::endl;
+ out_stream << "\tEntry count per bucket: " << header->entry_per_bucket << std::endl;
+ out_stream << "\tEviction bucket index: " << header->eviction_bucket_index << std::endl;
+ out_stream << "\tData block size: " << header->data_block_size << std::endl;
+ out_stream << "\tData size: " << header->m_data_offset << std::endl;
+ out_stream << "\tPending Buffer Size: " << header->pending_data_buff_size << std::endl;
+ if(free_lists)
+ out_stream << "\tTrackable Buffer Size: " << free_lists->GetTotSize() << std::endl;
+ mm.PrintStats(out_stream);
+
+ kv_file->PrintStats(out_stream);
+}
+
+int64_t Dict::Count() const
+{
+ if(header == NULL)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "db was not initialized successfully: %s",
+ MBError::get_error_str(status));
+ return 0;
+ }
+
+ return header->count;
+}
+
+// For DB iterator
+int Dict::ReadNextEdge(const uint8_t *node_buff, EdgePtrs &edge_ptrs,
+ int &match, MBData &data, std::string &match_str,
+ size_t &node_off, bool rd_kv) const
+{
+ if(edge_ptrs.curr_nt > static_cast<int>(node_buff[1]))
+ return MBError::OUT_OF_BOUND;
+
+ if(mm.ReadData(edge_ptrs.edge_buff, EDGE_SIZE, edge_ptrs.offset) != EDGE_SIZE)
+ return MBError::READ_ERROR;
+
+ node_off = 0;
+ match_str = "";
+
+ int rval = MBError::SUCCESS;
+ InitTempEdgePtrs(edge_ptrs);
+ if(edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ // match of leaf node
+ match = MATCH_EDGE;
+ if(rd_kv)
+ {
+ rval = ReadDataFromEdge(data, edge_ptrs);
+ if(rval != MBError::SUCCESS)
+ return rval;
+ }
+ }
+ else
+ {
+ match = MATCH_NONE;
+ if(edge_ptrs.len_ptr[0] > 0)
+ {
+ node_off = Get6BInteger(edge_ptrs.offset_ptr);
+ if(rd_kv)
+ rval = ReadNodeMatch(node_off, match, data);
+ }
+ }
+
+ if(edge_ptrs.len_ptr[0] > 0 && rd_kv)
+ {
+ int edge_len_m1 = edge_ptrs.len_ptr[0] - 1;
+ match_str = std::string(1, (const char)node_buff[NODE_EDGE_KEY_FIRST+edge_ptrs.curr_nt]);
+ if(edge_len_m1 > LOCAL_EDGE_LEN_M1)
+ {
+ if(mm.ReadData(data.node_buff, edge_len_m1, Get5BInteger(edge_ptrs.ptr)) != edge_len_m1)
+ return MBError::READ_ERROR;
+ match_str += std::string(reinterpret_cast<char*>(data.node_buff), edge_len_m1);
+ }
+ else if(edge_len_m1 > 0)
+ {
+ match_str += std::string(reinterpret_cast<char*>(edge_ptrs.ptr), edge_len_m1);
+ }
+ }
+
+ edge_ptrs.curr_nt++;
+ edge_ptrs.offset += EDGE_SIZE;
+ return rval;
+}
+
+// For DB iterator
+int Dict::ReadNode(size_t node_off, uint8_t *node_buff, EdgePtrs &edge_ptrs,
+ int &match, MBData &data, bool rd_kv) const
+{
+ if(mm.ReadData(node_buff, NODE_EDGE_KEY_FIRST, node_off) != NODE_EDGE_KEY_FIRST)
+ return MBError::READ_ERROR;
+
+ edge_ptrs.curr_nt = 0;
+ int nt = node_buff[1] + 1;
+ node_off += NODE_EDGE_KEY_FIRST;
+ if(mm.ReadData(node_buff + NODE_EDGE_KEY_FIRST, nt, node_off) != nt)
+ return MBError::READ_ERROR;
+
+ int rval = MBError::SUCCESS;
+ edge_ptrs.offset = node_off + nt;
+ if(node_buff[0] & FLAG_NODE_MATCH)
+ {
+ // match of non-leaf node
+ match = MATCH_NODE;
+ if(rd_kv)
+ rval = ReadDataFromNode(data, node_buff);
+ }
+ else
+ {
+ // no match at the non-leaf node
+ match = MATCH_NONE;
+ }
+
+ return rval;
+}
+
+void Dict::ReadNodeHeader(size_t node_off, int &node_size, int &match,
+ size_t &data_offset, size_t &data_link_offset)
+{
+ uint8_t node_buff[NODE_EDGE_KEY_FIRST];
+ if(mm.ReadData(node_buff, NODE_EDGE_KEY_FIRST, node_off) != NODE_EDGE_KEY_FIRST)
+ throw (int) MBError::READ_ERROR;
+
+ node_size = mm.GetNodeSizePtr()[ node_buff[1] ];
+ if(node_buff[0] & FLAG_NODE_MATCH)
+ {
+ match = MATCH_NODE;
+ data_offset = Get6BInteger(node_buff + 2);
+ data_link_offset = node_off + 2;
+ }
+}
+
+int Dict::ReadNodeMatch(size_t node_off, int &match, MBData &data) const
+{
+ uint8_t node_buff[NODE_EDGE_KEY_FIRST];
+ if(mm.ReadData(node_buff, NODE_EDGE_KEY_FIRST, node_off) != NODE_EDGE_KEY_FIRST)
+ return MBError::READ_ERROR;
+
+ int rval = MBError::SUCCESS;
+ if(node_buff[0] & FLAG_NODE_MATCH)
+ {
+ match = MATCH_NODE;
+ rval = ReadDataFromNode(data, node_buff);
+ if(rval != MBError::SUCCESS)
+ return rval;
+ }
+
+ return MBError::SUCCESS;
+}
+
+size_t Dict::GetRootOffset() const
+{
+ return mm.GetRootOffset();
+}
+
+// For DB iterator
+int Dict::ReadRootNode(uint8_t *node_buff, EdgePtrs &edge_ptrs, int &match,
+ MBData &data) const
+{
+ size_t root_off;
+ if(data.options & CONSTS::OPTION_RC_MODE)
+ root_off = header->rc_root_offset;
+ else
+ root_off = mm.GetRootOffset();
+ return ReadNode(root_off, node_buff, edge_ptrs, match, data);
+}
+
+int Dict::Remove(const uint8_t *key, int len)
+{
+ MBData data(0, CONSTS::OPTION_FIND_AND_STORE_PARENT);
+ return Remove(key, len, data);
+}
+
+int Dict::Remove(const uint8_t *key, int len, MBData &data)
+{
+ if(!(options & CONSTS::ACCESS_MODE_WRITER))
+ {
+ return MBError::NOT_ALLOWED;
+ }
+ if(data.options & CONSTS::OPTION_RC_MODE)
+ {
+ // FIXME Remove in RC mode not implemented yet!!!
+ return MBError::INVALID_ARG;
+ }
+
+ // The DELETE flag must be set
+ if(!(data.options & CONSTS::OPTION_FIND_AND_STORE_PARENT))
+ return MBError::INVALID_ARG;
+
+ int rval;
+ rval = Find(key, len, data);
+ if(rval == MBError::IN_DICT)
+ {
+ rval = DeleteDataFromEdge(data, data.edge_ptrs);
+ while(rval == MBError::TRY_AGAIN)
+ {
+ data.Clear();
+ len -= data.edge_ptrs.len_ptr[0];
+#ifdef __DEBUG__
+ assert(len > 0);
+#endif
+ rval = Find(key, len, data);
+ if(MBError::IN_DICT == rval)
+ {
+ rval = mm.RemoveEdgeByIndex(data.edge_ptrs, data);
+ }
+ }
+ }
+
+ if(rval == MBError::SUCCESS)
+ {
+ header->count--;
+ if(header->count == 0)
+ {
+ RemoveAll();
+ }
+ }
+
+ return rval;
+}
+
+int Dict::RemoveAll()
+{
+ int rval = MBError::SUCCESS;;
+ for(int c = 0; c < NUM_ALPHABET; c++)
+ {
+ rval = mm.ClearRootEdge(c);
+ if(rval != MBError::SUCCESS)
+ break;
+ }
+
+ mm.ClearMem();
+ mm.ResetSlidingWindow();
+
+ header->count = 0;
+ header->m_data_offset = GetStartDataOffset();
+ free_lists->Empty();
+ header->pending_data_buff_size = 0;
+ ResetSlidingWindow();
+
+ header->eviction_bucket_index = 0;
+ header->num_update = 0;
+ return rval;
+}
+
+pthread_rwlock_t* Dict::GetShmLockPtrs() const
+{
+ return &header->mb_rw_lock;
+}
+
+int Dict::InitShmObjects()
+{
+ if(status != MBError::SUCCESS)
+ return MBError::NOT_INITIALIZED;
+
+ Logger::Log(LOG_LEVEL_INFO, "initializing shared memory objects");
+
+#ifdef __SHM_LOCK__
+ status = InitShmRWLock(&header->mb_rw_lock);
+ if(status != MBError::SUCCESS)
+ return status;
+#endif
+
+#ifdef __SHM_QUEUE__
+ for(int i = 0; i < header->async_queue_size; i++)
+ {
+ status = InitShmMutex(&queue[i].mutex);
+ if(status != MBError::SUCCESS)
+ break;
+ status = InitShmCond(&queue[i].cond);
+ if(status != MBError::SUCCESS)
+ break;
+ }
+#endif
+
+ return status;
+}
+
+// Reserve buffer and write to it
+void Dict::ReserveData(const uint8_t* buff, int size, size_t &offset)
+{
+#ifdef __DEBUG__
+ assert(size <= CONSTS::MAX_DATA_SIZE);
+#endif
+
+ int buf_size = free_lists->GetAlignmentSize(size + DATA_HDR_BYTE);
+ int buf_index = free_lists->GetBufferIndex(buf_size);
+ uint16_t dsize[2];
+ dsize[0] = static_cast<uint16_t>(size);
+ // store bucket index for LRU eviction
+ dsize[1] = (header->num_update / header->entry_per_bucket) % 0xFFFF;
+ if(dsize[1] == header->eviction_bucket_index &&
+ header->num_update > header->entry_per_bucket)
+ {
+ header->eviction_bucket_index++;
+ }
+
+ if(free_lists->GetBufferCountByIndex(buf_index) > 0)
+ {
+ offset = free_lists->RemoveBufferByIndex(buf_index);
+ WriteData(reinterpret_cast<const uint8_t*>(&dsize[0]), DATA_HDR_BYTE, offset);
+ WriteData(buff, size, offset+DATA_HDR_BYTE);
+ header->pending_data_buff_size -= buf_size;
+ }
+ else
+ {
+ size_t old_off = header->m_data_offset;
+ uint8_t *ptr;
+
+ int rval = kv_file->Reserve(header->m_data_offset, buf_size, ptr);
+ if(rval != MBError::SUCCESS)
+ throw rval;
+
+ //Checking missing buffer due to alignment
+ if(old_off < header->m_data_offset)
+ {
+ free_lists->ReleaseAlignmentBuffer(old_off, header->m_data_offset);
+ header->pending_data_buff_size += header->m_data_offset - old_off;
+ }
+
+ offset = header->m_data_offset;
+ header->m_data_offset += buf_size;
+ if(ptr != NULL)
+ {
+ memcpy(ptr, &dsize[0], DATA_HDR_BYTE);
+ memcpy(ptr+DATA_HDR_BYTE, buff, size);
+ }
+ else
+ {
+ WriteData(reinterpret_cast<const uint8_t*>(&dsize[0]), DATA_HDR_BYTE, offset);
+ WriteData(buff, size, offset+DATA_HDR_BYTE);
+ }
+ }
+}
+
+int Dict::ReleaseBuffer(size_t offset)
+{
+ uint16_t data_size;
+
+ if(ReadData(reinterpret_cast<uint8_t*>(&data_size), DATA_SIZE_BYTE, offset)
+ != DATA_SIZE_BYTE)
+ return MBError::READ_ERROR;
+
+ int rel_size = free_lists->GetAlignmentSize(data_size + DATA_HDR_BYTE);
+ header->pending_data_buff_size += rel_size;
+ return free_lists->ReleaseBuffer(offset, rel_size);
+}
+
+int Dict::UpdateDataBuffer(EdgePtrs &edge_ptrs, bool overwrite, const uint8_t *buff,
+ int len, bool &inc_count)
+{
+ size_t data_off;
+
+ if(edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ inc_count = false;
+ // leaf node
+ if(!overwrite)
+ return MBError::IN_DICT;
+
+ data_off = Get6BInteger(edge_ptrs.offset_ptr);
+ if(ReleaseBuffer(data_off) != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_WARN, "failed to release data buffer: %llu", data_off);
+ ReserveData(buff, len, data_off);
+ Write6BInteger(edge_ptrs.offset_ptr, data_off);
+
+ header->excep_lf_offset = edge_ptrs.offset;
+ memcpy(header->excep_buff, edge_ptrs.offset_ptr, OFFSET_SIZE);
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ header->excep_updating_status = EXCEP_STATUS_ADD_DATA_OFF;
+ mm.WriteData(edge_ptrs.offset_ptr, OFFSET_SIZE, edge_ptrs.offset+EDGE_NODE_LEADING_POS);
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStop();
+#endif
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+ }
+ else
+ {
+ uint8_t *node_buff = header->excep_buff;
+ size_t node_off = Get6BInteger(edge_ptrs.offset_ptr);
+
+ if(mm.ReadData(node_buff, NODE_EDGE_KEY_FIRST, node_off) != NODE_EDGE_KEY_FIRST)
+ return MBError::READ_ERROR;
+
+ if(node_buff[0] & FLAG_NODE_MATCH)
+ {
+ inc_count = false;
+ if(!overwrite)
+ return MBError::IN_DICT;
+
+ data_off = Get6BInteger(node_buff+2);
+ if(ReleaseBuffer(data_off) != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_WARN, "failed to release data buffer %llu", data_off);
+
+ node_buff[NODE_EDGE_KEY_FIRST] = 0;
+ }
+ else
+ {
+ // set the match flag
+ node_buff[0] |= FLAG_NODE_MATCH;
+
+ node_buff[NODE_EDGE_KEY_FIRST] = 1;
+ }
+
+ ReserveData(buff, len, data_off);
+ Write6BInteger(node_buff+2, data_off);
+
+ header->excep_offset = node_off;
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = edge_ptrs.offset;
+ lfree.WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ header->excep_updating_status = EXCEP_STATUS_ADD_NODE;
+ mm.WriteData(node_buff, NODE_EDGE_KEY_FIRST, node_off);
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStop();
+#endif
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+ }
+
+ return MBError::SUCCESS;
+}
+
+// delta should be either +1 or -1.
+void Dict::UpdateNumReader(int delta) const
+{
+ header->num_reader += delta;
+ if(header->num_reader < 0)
+ header->num_reader = 0;
+
+ Logger::Log(LOG_LEVEL_INFO, "number of reader is set to: %d",
+ header->num_reader);
+}
+
+// delta should be either +1 or -1.
+int Dict::UpdateNumWriter(int delta) const
+{
+ if(delta > 0)
+ {
+ // Only one writer allowed
+ if(header->num_writer > 0)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "writer was not shutdown cleanly previously");
+ header->num_writer = 1;
+ // Reset number of reader too.
+ header->num_reader = 0;
+ return MBError::WRITER_EXIST;
+ }
+
+ header->num_writer = 1;
+ }
+ else if(delta < 0)
+ {
+ header->num_writer = 0;
+ header->lock_free.offset = MAX_6B_OFFSET;
+ }
+
+ Logger::Log(LOG_LEVEL_INFO, "number of writer is set to: %d", header->num_writer);
+ return MBError::SUCCESS;
+}
+
+DictMem* Dict::GetMM() const
+{
+ return (DictMem*) &mm;
+}
+
+size_t Dict::GetStartDataOffset() const
+{
+ return DATA_HEADER_SIZE;
+}
+
+void Dict::ResetSlidingWindow() const
+{
+ kv_file->ResetSlidingWindow();
+ if(header != NULL)
+ header->shm_data_sliding_start.store(0, std::memory_order_relaxed);
+}
+
+LockFree* Dict::GetLockFreePtr()
+{
+ return &lfree;
+}
+
+void Dict::Flush() const
+{
+ if(!(options & CONSTS::ACCESS_MODE_WRITER))
+ return;
+
+ if(kv_file != NULL)
+ kv_file->Flush();
+ mm.Flush();
+}
+
+// Recovery from abnormal writer terminations (segfault, kill -9 etc)
+// during DB updates (insertion, replacing and deletion).
+int Dict::ExceptionRecovery()
+{
+ if(header == NULL)
+ return MBError::NOT_INITIALIZED;
+
+ int rval = MBError::SUCCESS;
+ if(header->excep_updating_status == EXCEP_STATUS_NONE)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "writer was shutdown successfully previously");
+ return rval;
+ }
+
+ Logger::Log(LOG_LEVEL_INFO, "writer was not shutdown gracefully with exception status %d",
+ header->excep_updating_status);
+ // Dumper header before running recover
+ std::ofstream *logstream = Logger::GetLogStream();
+ if(logstream != NULL)
+ {
+ PrintHeader(*logstream);
+ }
+ else
+ {
+ PrintHeader(std::cout);
+ }
+
+ switch(header->excep_updating_status)
+ {
+ case EXCEP_STATUS_ADD_EDGE:
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(header->excep_lf_offset);
+#endif
+ mm.WriteData(header->excep_buff, EDGE_SIZE, header->excep_lf_offset);
+ header->count++;
+ break;
+ case EXCEP_STATUS_ADD_DATA_OFF:
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(header->excep_lf_offset);
+#endif
+ mm.WriteData(header->excep_buff, OFFSET_SIZE,
+ header->excep_lf_offset+EDGE_NODE_LEADING_POS);
+ break;
+ case EXCEP_STATUS_ADD_NODE:
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(header->excep_lf_offset);
+#endif
+ mm.WriteData(header->excep_buff, NODE_EDGE_KEY_FIRST,
+ header->excep_offset);
+ if(header->excep_buff[NODE_EDGE_KEY_FIRST]) header->count++;
+ break;
+ case EXCEP_STATUS_REMOVE_EDGE:
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(header->excep_lf_offset);
+#endif
+ Write6BInteger(header->excep_buff, header->excep_offset);
+ mm.WriteData(header->excep_buff, OFFSET_SIZE,
+ header->excep_lf_offset+EDGE_NODE_LEADING_POS);
+ break;
+ case EXCEP_STATUS_CLEAR_EDGE:
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(header->excep_lf_offset);
+#endif
+ mm.WriteData(DictMem::empty_edge, EDGE_SIZE, header->excep_lf_offset);
+ header->count--;
+ break;
+ case EXCEP_STATUS_RC_NODE:
+ case EXCEP_STATUS_RC_DATA:
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(header->excep_lf_offset);
+#endif
+ mm.WriteData(header->excep_buff, OFFSET_SIZE, header->excep_offset);
+ break;
+ case EXCEP_STATUS_RC_EDGE_STR:
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStart(header->excep_lf_offset);
+#endif
+ mm.WriteData(header->excep_buff, OFFSET_SIZE-1, header->excep_offset);
+ break;
+ default:
+ Logger::Log(LOG_LEVEL_ERROR, "unknown exception status: %d",
+ header->excep_updating_status);
+ rval = MBError::INVALID_ARG;
+ }
+#ifdef __LOCK_FREE__
+ lfree.WriterLockFreeStop();
+#endif
+
+ if(rval == MBError::SUCCESS)
+ {
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+ Logger::Log(LOG_LEVEL_INFO, "successfully recovered from abnormal termination");
+ }
+ else
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to recover from abnormal termination");
+ }
+
+ return rval;
+}
+
+void Dict::WriteData(const uint8_t *buff, unsigned len, size_t offset) const
+{
+ if(offset + len > header->m_data_offset)
+ {
+ std::cerr << "invalid dict write: " << offset << " " << len << " "
+ << header->m_data_offset << "\n";
+ throw (int) MBError::OUT_OF_BOUND;
+ }
+
+ if(kv_file->RandomWrite(buff, len, offset) != len)
+ throw (int) MBError::WRITE_ERROR;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __DICT_H__
+#define __DICT_H__
+
+#include <stdint.h>
+#include <string>
+
+#include "drm_base.h"
+#include "dict_mem.h"
+#include "rollable_file.h"
+#include "mb_data.h"
+#include "lock_free.h"
+
+namespace mabain {
+
+#ifdef __SHM_QUEUE__
+struct _AsyncNode;
+typedef struct _AsyncNode AsyncNode;
+#endif
+
+// dictionary class
+// This is the work horse class for basic db operations (add, find and remove).
+class Dict : public DRMBase
+{
+public:
+ Dict(const std::string &mbdir, bool init_header, int datasize,
+ int db_options, size_t memsize_index, size_t memsize_data,
+ uint32_t block_sz_index, uint32_t block_sz_data,
+ int max_num_index_blk, int max_num_data_blk,
+ int64_t entry_per_bucket, uint32_t queue_size);
+ virtual ~Dict();
+ void Destroy();
+
+ // Called by writer only
+ int Init(uint32_t id);
+ // Add key-value pair
+ int Add(const uint8_t *key, int len, MBData &data, bool overwrite);
+ // Find value by key
+ int Find(const uint8_t *key, int len, MBData &data);
+ // Find value by key using prefix match
+ int FindPrefix(const uint8_t *key, int len, MBData &data);
+ // Delete entry by key
+ int Remove(const uint8_t *key, int len);
+ // Delete entry by key
+ int Remove(const uint8_t *key, int len, MBData &data);
+
+ // Delete all entries
+ int RemoveAll();
+
+#ifdef __SHM_QUEUE__
+ // multiple-process updates using shared memory queue
+ int SHMQ_Add(const char *key, int key_len, const char *data, int data_len,
+ bool overwrite);
+ int SHMQ_Remove(const char *key, int len);
+ int SHMQ_RemoveAll();
+ int SHMQ_Backup(const char *backup_dir);
+ int SHMQ_CollectResource(int64_t m_index_rc_size, int64_t m_data_rc_size,
+ int64_t max_dbsz, int64_t max_dbcnt);
+ bool SHMQ_Busy() const;
+#endif
+
+ void ReserveData(const uint8_t* buff, int size, size_t &offset);
+ void WriteData(const uint8_t *buff, unsigned len, size_t offset) const;
+
+ // Print dictinary stats
+ void PrintStats(std::ostream *out_stream) const;
+ void PrintStats(std::ostream &out_stream) const;
+ int Status() const;
+ int64_t Count() const;
+ size_t GetRootOffset() const;
+ size_t GetStartDataOffset() const;
+
+ DictMem *GetMM() const;
+
+ LockFree* GetLockFreePtr();
+
+ // Used for DB iterator
+ int ReadNextEdge(const uint8_t *node_buff, EdgePtrs &edge_ptrs, int &match,
+ MBData &data, std::string &match_str, size_t &node_off,
+ bool rd_kv = true) const;
+ int ReadNode(size_t node_off, uint8_t *node_buff, EdgePtrs &edge_ptrs,
+ int &match, MBData &data, bool rd_kv = true) const;
+ void ReadNodeHeader(size_t node_off, int &node_size, int &match,
+ size_t &data_offset, size_t &data_link_offset);
+ int ReadRootNode(uint8_t *node_buff, EdgePtrs &edge_ptrs, int &match,
+ MBData &data) const;
+
+ // Shared memory objects
+ int InitShmObjects();
+ pthread_rwlock_t* GetShmLockPtrs() const;
+
+ void UpdateNumReader(int delta) const;
+ int UpdateNumWriter(int delta) const;
+
+ void ResetSlidingWindow() const;
+ void Flush() const;
+ int ExceptionRecovery();
+
+private:
+ int Find_Internal(size_t root_off, const uint8_t *key, int len, MBData &data);
+ int FindPrefix_Internal(size_t root_off, const uint8_t *key, int len, MBData &data);
+ int ReleaseBuffer(size_t offset);
+ int UpdateDataBuffer(EdgePtrs &edge_ptrs, bool overwrite, const uint8_t *buff,
+ int len, bool &inc_count);
+ int ReadDataFromEdge(MBData &data, const EdgePtrs &edge_ptrs) const;
+ int ReadDataFromNode(MBData &data, const uint8_t *node_ptr) const;
+ int DeleteDataFromEdge(MBData &data, EdgePtrs &edge_ptrs);
+ int ReadNodeMatch(size_t node_off, int &match, MBData &data) const;
+#ifdef __SHM_QUEUE__
+ int SHMQ_PrepareSlot(AsyncNode *node_ptr) const;
+ AsyncNode* SHMQ_AcquireSlot() const;
+#endif
+
+ // DB access permission
+ int options;
+ // Memory management
+ DictMem mm;
+
+ // dict status
+ int status;
+
+ LockFree lfree;
+
+ size_t reader_rc_off;
+#ifdef __SHM_QUEUE__
+ AsyncNode *queue;
+#endif
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string>
+#include <iostream>
+#include <limits.h>
+
+#include "db.h"
+#include "dict_mem.h"
+#include "logger.h"
+#include "error.h"
+#include "integer_4b_5b.h"
+#include "version.h"
+#include "resource_pool.h"
+#include "async_writer.h"
+
+#define OFFSET_SIZE_P1 7
+
+#define MAX_BUFFER_RESERVE_SIZE 8192
+#define NUM_BUFFER_RESERVE MAX_BUFFER_RESERVE_SIZE/BUFFER_ALIGNMENT
+
+namespace mabain {
+
+/////////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////
+// EDGE MEMORY LAYOUT
+// Edge size is 13 bytes.
+// X************ leading byte of edge key offset
+// *XXXX******** edge key string or edge key offset
+// *****X******* edge key length
+// ******X****** flag (0x01) indicating the data offset; This bit can used to
+// determine if the node if a leaf-node.
+// flag (0x02) indicating this edge owns the allocated bufifer
+// *******X***** leading byte of next node offset or data offset
+// ********xxxxX next node offset of data offset
+/////////////////////////////////////////////////////////////////////////////////////
+// NODE MEMORY LAYOUT
+// Node size is 1 + 1 + 6 + NT + NT*13
+// X************ flags (0x01 bit indicating match found)
+// *X*********** nt-1, nt is the number of edges for this node.
+// **XXXXXX***** data offset
+// NT bytes of first characters of each edge
+// NT edges NT*13 bytes
+// Since we use 6-byte to store both the index and data offset, the maximum size for
+// data and index is 281474976710655 bytes (or 255T).
+/////////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////
+
+
+DictMem::DictMem(const std::string &mbdir, bool init_header, size_t memsize,
+ int mode, uint32_t block_size, int max_num_blk, uint32_t queue_size)
+ : is_valid(false)
+{
+ root_offset = 0;
+ root_offset_rc = 0;
+ node_ptr = NULL;
+ kv_file = NULL;
+ node_size = NULL;
+
+ assert(sizeof(IndexHeader) <= (unsigned) RollableFile::page_size);
+ bool map_hdr = true;
+ bool create_hdr = false;
+ size_t hdr_size = RollableFile::page_size;
+ if(mode & CONSTS::ACCESS_MODE_WRITER)
+ create_hdr = true;
+#ifdef __SHM_QUEUE__
+ hdr_size += sizeof(AsyncNode) * queue_size;
+#endif
+ header_file = ResourcePool::getInstance().OpenFile(mbdir + "_mabain_h",
+ mode,
+ hdr_size,
+ map_hdr,
+ create_hdr);
+ if(header_file == NULL)
+ {
+ std::cerr << "failed top open header file: " << mbdir + "_mabain_h\n";
+ return;
+ }
+ header = reinterpret_cast<IndexHeader *>(header_file->GetMapAddr());
+ if(header == NULL)
+ return;
+
+ // Both reader and writer need to open the mmapped file.
+ if(!init_header)
+ {
+ if(block_size != 0 && header->index_block_size != block_size)
+ {
+ std::cerr << "mabain index block size not match " << block_size << ": "
+ << header->index_block_size << std::endl;
+ Destroy();
+ throw (int) MBError::INVALID_SIZE;
+ }
+ }
+ else
+ {
+ memset(header, 0, sizeof(IndexHeader));
+ header->index_block_size = block_size;
+ }
+ kv_file = new RollableFile(mbdir + "_mabain_i",
+ static_cast<size_t>(header->index_block_size),
+ memsize, mode, max_num_blk);
+
+ kv_file->InitShmSlidingAddr(&header->shm_index_sliding_start);
+
+ if(!(mode & CONSTS::ACCESS_MODE_WRITER))
+ {
+ node_size = NULL;
+ if(header->async_queue_size == 0)
+ {
+ std::cerr << "async_queue_size not set yet in header\n";
+ return;
+ }
+ is_valid = true;
+ return;
+ }
+
+ ////////////////////////////////////
+ // Writor only initialization below
+ ////////////////////////////////////
+
+ node_size = new int[NUM_ALPHABET];
+
+ for(int i = 0; i < NUM_ALPHABET; i++)
+ {
+ int nt = i + 1;
+ node_size[i] = 1 + 1 + OFFSET_SIZE + nt + nt*EDGE_SIZE;
+ }
+
+ node_ptr = new uint8_t[ node_size[NUM_ALPHABET-1] ];
+ free_lists = new FreeList(mbdir+"_ibfl", BUFFER_ALIGNMENT, NUM_BUFFER_RESERVE);
+
+ if(init_header)
+ {
+ // Set up DB version
+ header->version[0] = version[0];
+ header->version[1] = version[1];
+ header->version[2] = version[2];
+ header->version[3] = 0;
+ // Cannot set is_valid to true.
+ // More init to be dobe in InitRootNode.
+ }
+ else
+ {
+ is_valid = true;
+ }
+ Logger::Log(LOG_LEVEL_INFO, "set up mabain db version to %u.%u.%u",
+ header->version[0], header->version[1], header->version[2]);
+}
+
+// The whole edge is initizlized to zero.
+const uint8_t DictMem::empty_edge[] = {0};
+
+void DictMem::InitRootNode()
+{
+#ifdef __DEBUG__
+ assert(header != NULL);
+#endif
+ header->m_index_offset = 0;
+
+ bool node_move;
+ uint8_t *root_node;
+
+ node_move = ReserveNode(NUM_ALPHABET-1, root_offset, root_node);
+#ifdef __DEBUG__
+ assert(root_offset == 0);
+#endif
+
+ root_node[0] = FLAG_NODE_NONE;
+ root_node[1] = NUM_ALPHABET-1;
+ for(int i = 0; i < NUM_ALPHABET; i++)
+ {
+ root_node[NODE_EDGE_KEY_FIRST+i] = static_cast<uint8_t>(i);
+ }
+
+ if(node_move)
+ WriteData(root_node, node_size[NUM_ALPHABET-1], root_offset);
+
+ // Everything is running fine if reaching this point.
+ is_valid = true;
+}
+
+DictMem::~DictMem()
+{
+}
+
+void DictMem::Destroy()
+{
+ if(kv_file != NULL)
+ delete kv_file;
+
+ if(free_lists != NULL)
+ delete free_lists;
+
+ if(node_size != NULL)
+ delete [] node_size;
+ if(node_ptr != NULL)
+ delete [] node_ptr;
+}
+
+bool DictMem::IsValid() const
+{
+ return is_valid;
+}
+
+// Add root edge
+void DictMem::AddRootEdge(EdgePtrs &edge_ptrs, const uint8_t *key,
+ int len, size_t data_offset)
+{
+ edge_ptrs.len_ptr[0] = len;
+ if(len > LOCAL_EDGE_LEN)
+ {
+ size_t edge_str_off;
+ ReserveData(key+1, len-1, edge_str_off);
+ Write5BInteger(edge_ptrs.ptr, edge_str_off);
+ }
+ else
+ {
+ memcpy(edge_ptrs.ptr, key+1, len-1);
+ }
+
+ edge_ptrs.flag_ptr[0] = EDGE_FLAG_DATA_OFF;
+ Write6BInteger(edge_ptrs.offset_ptr, data_offset);
+
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = edge_ptrs.offset;
+ header->excep_updating_status = EXCEP_STATUS_ADD_EDGE;
+ lfree->WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ WriteEdge(edge_ptrs);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+}
+
+void DictMem::UpdateTailEdge(EdgePtrs &edge_ptrs, int match_len, MBData &data,
+ EdgePtrs &tail_edge, uint8_t &new_key_first,
+ bool &map_new_sliding)
+{
+ int edge_len = edge_ptrs.len_ptr[0] - match_len;
+ tail_edge.len_ptr[0] = edge_len;
+ if(edge_len > LOCAL_EDGE_LEN)
+ {
+ // Old key len must be greater than LOCAL_EDGE_LEN too.
+ // Load the string with length edge_len.
+ size_t new_key_off;
+ size_t edge_str_off = Get5BInteger(edge_ptrs.ptr);
+ if(ReadData(data.node_buff, edge_len, edge_str_off + match_len - 1)
+ != edge_len)
+ throw (int) MBError::READ_ERROR;
+ new_key_first = data.node_buff[0];
+
+ // Reserve the key buffer
+ ReserveData(data.node_buff+1, edge_len-1, new_key_off, map_new_sliding);
+ map_new_sliding = false;
+ Write5BInteger(tail_edge.ptr, new_key_off);
+ }
+ else
+ {
+ if(edge_ptrs.len_ptr[0] > LOCAL_EDGE_LEN)
+ {
+ if(ReadData(data.node_buff, edge_len, Get5BInteger(edge_ptrs.ptr)+match_len-1)
+ != edge_len)
+ throw (int) MBError::READ_ERROR;
+ new_key_first = data.node_buff[0];
+ if(edge_len > 1)
+ memcpy(tail_edge.ptr, data.node_buff+1, edge_len-1);
+ }
+ else
+ {
+ // Both new and old keys are local
+ new_key_first = edge_ptrs.ptr[match_len - 1];
+ if(edge_len > 1)
+ memcpy(tail_edge.ptr, edge_ptrs.ptr+match_len, edge_len-1);
+ }
+ }
+
+ // 7 = 1 + OFFSET_SIZE
+ memcpy(tail_edge.flag_ptr, edge_ptrs.flag_ptr, OFFSET_SIZE_P1);
+}
+
+//The old edge becomes head edge.
+void DictMem::UpdateHeadEdge(EdgePtrs &edge_ptrs, int match_len,
+ MBData &data, int &release_buffer_size,
+ size_t &edge_str_off, bool &map_new_sliding)
+{
+ int match_len_m1 = match_len - 1;
+ if(edge_ptrs.len_ptr[0] > LOCAL_EDGE_LEN)
+ {
+ if(match_len <= LOCAL_EDGE_LEN)
+ {
+ edge_str_off = Get5BInteger(edge_ptrs.ptr);
+ release_buffer_size = edge_ptrs.len_ptr[0] - 1;
+ // Old key is remote but new key is local. Need to read the old key.
+ if(match_len_m1 > 0)
+ {
+ if(ReadData(edge_ptrs.ptr, match_len_m1, edge_str_off) != match_len_m1)
+ throw (int) MBError::READ_ERROR;
+ }
+ }
+ else
+ {
+ edge_str_off = Get5BInteger(edge_ptrs.ptr);
+ release_buffer_size = edge_ptrs.len_ptr[0] - 1;
+ // Load the string with length edge_len - 1
+ if(ReadData(data.node_buff, match_len_m1, edge_str_off) != match_len_m1)
+ throw (int) MBError::READ_ERROR;
+ // Reserve the key buffer
+ size_t new_key_off;
+ ReserveData(data.node_buff, match_len_m1, new_key_off, map_new_sliding);
+ map_new_sliding = false;
+ Write5BInteger(edge_ptrs.ptr, new_key_off);
+ }
+ }
+
+ edge_ptrs.len_ptr[0] = match_len;
+ edge_ptrs.flag_ptr[0] = 0;
+}
+
+// Insert a new node on the current edge
+// The old edge becomes two edges.
+int DictMem::InsertNode(EdgePtrs &edge_ptrs, int match_len,
+ size_t data_offset, MBData &data)
+{
+ NodePtrs node_ptrs;
+ EdgePtrs new_edge_ptrs;
+ bool node_move;
+ uint8_t *node;
+ bool map_new_sliding = false;
+
+ // The new node has one edge. nt1 = nt - 1 = 0
+ node_move = ReserveNode(0, node_ptrs.offset, node);
+ if(node_move)
+ map_new_sliding = true;
+
+ InitNodePtrs(node, 0, node_ptrs);
+ node[1] = 0;
+ InitEdgePtrs(node_ptrs, 0, new_edge_ptrs);
+
+ uint8_t new_key_first;
+ UpdateTailEdge(edge_ptrs, match_len, data, new_edge_ptrs, new_key_first,
+ map_new_sliding);
+
+ int release_buffer_size = 0;
+ size_t edge_str_off = 0;
+ UpdateHeadEdge(edge_ptrs, match_len, data, release_buffer_size, edge_str_off,
+ map_new_sliding);
+ Write6BInteger(edge_ptrs.offset_ptr, node_ptrs.offset);
+
+ // Update the new node
+ // match found for the new node
+ node[0] = FLAG_NODE_NONE | FLAG_NODE_MATCH;
+ // Update data offset in the node
+ Write6BInteger(node_ptrs.ptr+2, data_offset);
+ // Update the first character in edge key
+ node_ptrs.edge_key_ptr[0] = new_key_first;
+
+ if(node_move)
+ WriteData(node, node_size[0], node_ptrs.offset);
+
+ if(release_buffer_size > 0)
+ ReleaseBuffer(edge_str_off, release_buffer_size);
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = edge_ptrs.offset;
+ header->excep_updating_status = EXCEP_STATUS_ADD_EDGE;
+ lfree->WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ WriteEdge(edge_ptrs);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+
+ header->n_edges++;
+ return MBError::SUCCESS;
+}
+
+// Insert a new node on the current edge.
+// Add a new edge to the new node. The new node will have two edges.
+// The old edge becomes two edges.
+int DictMem::AddLink(EdgePtrs &edge_ptrs, int match_len, const uint8_t *key,
+ int key_len, size_t data_off, MBData &data)
+{
+ NodePtrs node_ptrs;
+ EdgePtrs new_edge_ptrs[2];
+ bool node_move;
+ uint8_t* node;
+ bool map_new_sliding = false;
+
+ // The new node has two edge. nt1 = nt - 1 = 1
+ node_move = ReserveNode(1, node_ptrs.offset, node);
+ if(node_move)
+ map_new_sliding = true;
+ InitNodePtrs(node, 1, node_ptrs);
+ node[0] = FLAG_NODE_NONE;
+ node[1] = 1;
+ InitEdgePtrs(node_ptrs, 0, new_edge_ptrs[0]);
+ InitEdgePtrs(node_ptrs, 1, new_edge_ptrs[1]);
+
+ uint8_t new_key_first;
+ UpdateTailEdge(edge_ptrs, match_len, data, new_edge_ptrs[0], new_key_first,
+ map_new_sliding);
+
+ int release_buffer_size = 0;
+ size_t edge_str_off;
+ UpdateHeadEdge(edge_ptrs, match_len, data, release_buffer_size, edge_str_off,
+ map_new_sliding);
+ Write6BInteger(edge_ptrs.offset_ptr, node_ptrs.offset);
+
+ // Update the new node
+ // match not found for the new node, should not set node[1] and data offset
+ node_ptrs.edge_key_ptr[0] = new_key_first;
+ node_ptrs.edge_key_ptr[1] = key[0];
+
+ // Update the new edge
+ new_edge_ptrs[1].len_ptr[0] = key_len;
+ if(key_len > LOCAL_EDGE_LEN)
+ {
+ size_t new_key_off;
+ ReserveData(key+1, key_len-1, new_key_off, map_new_sliding);
+ Write5BInteger(new_edge_ptrs[1].ptr, new_key_off);
+ }
+ else
+ {
+ // edge key is local
+ if(key_len > 1)
+ memcpy(new_edge_ptrs[1].ptr, key+1, key_len-1);
+ }
+ // Indicate this new edge holds a data offset
+ new_edge_ptrs[1].flag_ptr[0] = EDGE_FLAG_DATA_OFF;
+ Write6BInteger(new_edge_ptrs[1].offset_ptr, data_off);
+
+ if(node_move)
+ WriteData(node, node_size[1], node_ptrs.offset);
+
+ // Update the parent edge
+ if(release_buffer_size > 0)
+ ReleaseBuffer(edge_str_off, release_buffer_size);
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = edge_ptrs.offset;
+ header->excep_updating_status = EXCEP_STATUS_ADD_EDGE;
+ lfree->WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ WriteEdge(edge_ptrs);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+
+ header->n_edges += 2;
+ return MBError::SUCCESS;
+}
+
+// Add a new edge in current node
+// This invloves creating a new node and copying data from old node to the new node
+// and updating the child node offset in edge_ptrs (parent edge).
+int DictMem::UpdateNode(EdgePtrs &edge_ptrs, const uint8_t *key, int key_len,
+ size_t data_off)
+{
+ int nt = edge_ptrs.curr_nt + 1;
+ bool node_move;
+ NodePtrs node_ptrs;
+ uint8_t* node;
+ bool map_new_sliding = false;
+
+ node_move = ReserveNode(nt, node_ptrs.offset, node);
+ if(node_move)
+ map_new_sliding = true;
+ InitNodePtrs(node, nt, node_ptrs);
+
+ // Load the old node
+ size_t old_node_off = Get6BInteger(edge_ptrs.offset_ptr);
+ int release_node_index = -1;
+ if(nt == 0)
+ {
+ // Change from empty node to node with one edge
+ // The old empty node stored the data offset instead of child node off.
+ if(edge_ptrs.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ Write6BInteger(node_ptrs.ptr+2, old_node_off);
+ edge_ptrs.flag_ptr[0] &= ~EDGE_FLAG_DATA_OFF;
+ node[0] = FLAG_NODE_MATCH | FLAG_NODE_NONE;
+ }
+ }
+ else
+ {
+#ifdef __DEBUG__
+ assert(nt > 0);
+#endif
+
+ // Copy old node
+ int copy_size = NODE_EDGE_KEY_FIRST + nt;
+ if(ReadData(node_ptrs.ptr, copy_size, old_node_off) != copy_size)
+ return MBError::READ_ERROR;
+ if(ReadData(node_ptrs.ptr+copy_size+1, EDGE_SIZE*nt, old_node_off+copy_size) !=
+ EDGE_SIZE*nt)
+ return MBError::READ_ERROR;
+
+ release_node_index = nt - 1;
+ }
+
+ node[1] = static_cast<uint8_t>(nt);
+
+ // Update the first edge key character for the new edge
+ node_ptrs.edge_key_ptr[nt] = key[0];
+
+ Write6BInteger(edge_ptrs.offset_ptr, node_ptrs.offset);
+
+ // Create the new edge
+ EdgePtrs new_edge_ptrs;
+ InitEdgePtrs(node_ptrs, nt, new_edge_ptrs);
+ new_edge_ptrs.len_ptr[0] = key_len;
+ if(key_len > LOCAL_EDGE_LEN)
+ {
+ size_t new_key_off;
+ ReserveData(key+1, key_len-1, new_key_off, map_new_sliding);
+ Write5BInteger(new_edge_ptrs.ptr, new_key_off);
+ }
+ else
+ {
+ // edge key is local
+ if(key_len > 1)
+ memcpy(new_edge_ptrs.ptr, key+1, key_len-1);
+ }
+
+ // Indicate this new edge holds a data offset
+ new_edge_ptrs.flag_ptr[0] = EDGE_FLAG_DATA_OFF;
+ Write6BInteger(new_edge_ptrs.offset_ptr, data_off);
+
+ if(node_move)
+ WriteData(node, node_size[nt], node_ptrs.offset);
+
+ if(release_node_index >= 0)
+ ReleaseNode(old_node_off, release_node_index);
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = edge_ptrs.offset;
+ header->excep_updating_status = EXCEP_STATUS_ADD_EDGE;
+ lfree->WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ WriteEdge(edge_ptrs);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+
+ header->n_edges++;
+ return MBError::SUCCESS;
+}
+
+bool DictMem::FindNext(const unsigned char *key, int keylen, int &match_len,
+ EdgePtrs &edge_ptr, uint8_t *key_tmp) const
+{
+ if(edge_ptr.flag_ptr[0] & EDGE_FLAG_DATA_OFF)
+ {
+ edge_ptr.curr_nt = -1;
+ return false;
+ }
+
+ size_t node_off = Get6BInteger(edge_ptr.offset_ptr);
+#ifdef __DEBUG__
+ assert(node_off != 0);
+#endif
+ if(ReadData(key_tmp, 1, node_off+1) != 1)
+ return false;
+ int nt = key_tmp[0];
+ edge_ptr.curr_nt = nt;
+ nt++;
+ // Load edge key first
+ node_off += NODE_EDGE_KEY_FIRST;
+ if(ReadData(key_tmp, nt, node_off) != nt)
+ return false;
+ int i;
+ for(i = 0; i < nt; i++)
+ {
+ if(key_tmp[i] == key[0])
+ break;
+ }
+
+ if(i >= nt)
+ return false;
+
+ match_len = 1;
+
+ // Load the new edge
+ edge_ptr.offset = node_off + nt + i*EDGE_SIZE;
+ if(ReadData(header->excep_buff, EDGE_SIZE, edge_ptr.offset) != EDGE_SIZE)
+ return false;
+ uint8_t *key_string_ptr;
+ int len = edge_ptr.len_ptr[0] - 1;
+ if(len > LOCAL_EDGE_LEN_M1)
+ {
+ if(ReadData(key_tmp, len, Get5BInteger(edge_ptr.ptr)) != len)
+ return false;
+ key_string_ptr = key_tmp;
+ }
+ else if(len > 0)
+ {
+ key_string_ptr = header->excep_buff;
+ }
+ else
+ {
+ return true;
+ }
+
+ for(i = 1; i < keylen; i++)
+ {
+ if(key_string_ptr[i-1] != key[i] || i > len)
+ break;
+ match_len++;
+ }
+
+ return true;
+}
+
+// Reserve buffer for a new node.
+// The allocated in-memory buffer must be initialized to zero.
+bool DictMem::ReserveNode(int nt, size_t &offset, uint8_t* &ptr)
+{
+#ifdef __DEBUG__
+ assert(nt >= 0 && nt < 256);
+#endif
+
+ int buf_size = free_lists->GetAlignmentSize(node_size[nt]);
+ int buf_index = free_lists->GetBufferIndex(buf_size);
+
+ header->n_states++;
+#ifdef __LOCK_FREE__
+ if(free_lists->GetBufferByIndex(buf_index, offset))
+ {
+ ptr = node_ptr;
+ memset(ptr, 0, buf_size);
+ header->pending_index_buff_size -= buf_size;
+ return true;
+ }
+#else
+ if(free_lists->GetBufferCountByIndex(buf_index) > 0)
+ {
+ offset = free_lists->RemoveBufferByIndex(buf_index);
+ ptr = node_ptr;
+ memset(ptr, 0, buf_size);
+ header->pending_index_buff_size -= buf_size;
+ return true;
+ }
+#endif
+
+ ptr = NULL;
+ size_t old_off = header->m_index_offset;
+ bool node_move = false;
+ int rval = kv_file->Reserve(header->m_index_offset, buf_size, ptr);
+ if(rval != MBError::SUCCESS)
+ throw rval;
+ if(ptr == NULL)
+ {
+ node_move = true;
+ ptr = node_ptr;
+ }
+
+ //Checking missing buffer due to alignment
+ if(old_off < header->m_index_offset)
+ {
+ free_lists->ReleaseAlignmentBuffer(old_off, header->m_index_offset);
+ header->pending_index_buff_size += header->m_index_offset - old_off;
+ }
+
+ memset(ptr, 0, buf_size);
+ offset = header->m_index_offset;
+ header->m_index_offset += buf_size;
+ return node_move;
+}
+
+// Reserve buffer for a new key
+void DictMem::ReserveData(const uint8_t* key, int size, size_t &offset,
+ bool map_new_sliding)
+{
+ int buf_index = free_lists->GetBufferIndex(size);
+ int buf_size = free_lists->GetAlignmentSize(size);
+
+#ifdef __LOCK_FREE__
+ if(free_lists->GetBufferByIndex(buf_index, offset))
+ {
+ WriteData(key, size, offset);
+ header->pending_index_buff_size -= buf_size;
+ }
+#else
+ if(free_lists->GetBufferCountByIndex(buf_index) > 0)
+ {
+ offset = free_lists->RemoveBufferByIndex(buf_index);
+ WriteData(key, size, offset);
+ header->pending_index_buff_size -= buf_size;
+ }
+#endif
+ else
+ {
+ size_t old_off = header->m_index_offset;
+ uint8_t *ptr;
+
+ int rval = kv_file->Reserve(header->m_index_offset, buf_size, ptr,
+ map_new_sliding);
+ if(rval != MBError::SUCCESS)
+ throw rval;
+
+ //Checking missing buffer due to alignment
+ if(old_off < header->m_index_offset)
+ {
+ free_lists->ReleaseAlignmentBuffer(old_off, header->m_index_offset);
+ header->pending_index_buff_size += header->m_index_offset - old_off;
+ }
+
+ offset = header->m_index_offset;
+ header->m_index_offset += buf_size;
+ if(ptr != NULL)
+ memcpy(ptr, key, size);
+ else
+ WriteData(key, size, offset);
+ }
+
+ header->edge_str_size += buf_size;
+}
+
+// Release node buffer
+void DictMem::ReleaseNode(size_t offset, int nt)
+{
+ if(nt < 0)
+ return;
+
+ int buf_index = free_lists->GetBufferIndex(node_size[nt]);
+ int rval = free_lists->AddBufferByIndex(buf_index, offset);
+ if(rval == MBError::SUCCESS)
+ header->n_states--;
+ else
+ Logger::Log(LOG_LEVEL_ERROR, "failed to release node buffer");
+ header->pending_index_buff_size += free_lists->GetAlignmentSize(node_size[nt]);
+}
+
+// Release edge string buffer
+void DictMem::ReleaseBuffer(size_t offset, int size)
+{
+ int rval = free_lists->ReleaseBuffer(offset, size);
+ if(rval != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_ERROR, "failed to release buffer");
+ else
+ header->edge_str_size -= free_lists->GetAlignmentSize(size);
+ header->pending_index_buff_size += free_lists->GetAlignmentSize(size);
+}
+
+int DictMem::GetRootEdge(size_t rc_off, int nt, EdgePtrs &edge_ptrs) const
+{
+ if(rc_off != 0)
+ edge_ptrs.offset = rc_off + NODE_EDGE_KEY_FIRST + NUM_ALPHABET + nt*EDGE_SIZE;
+ else
+ edge_ptrs.offset = root_offset + NODE_EDGE_KEY_FIRST + NUM_ALPHABET + nt*EDGE_SIZE;
+ if(ReadData(edge_ptrs.edge_buff, EDGE_SIZE, edge_ptrs.offset) != EDGE_SIZE)
+ return MBError::READ_ERROR;
+
+ InitTempEdgePtrs(edge_ptrs);
+ return MBError::SUCCESS;
+}
+
+// The temp edge is written to shared memory for handling segfault situations.
+// When writer restarts from segfault, it will retry WriteEdge so that the DB is
+// maintained consistently.
+int DictMem::GetRootEdge_Writer(bool rc_mode, int nt, EdgePtrs &edge_ptrs) const
+{
+ if(rc_mode)
+ {
+ if(root_offset_rc == 0)
+ throw (int) MBError::UNKNOWN_ERROR;
+ edge_ptrs.offset = root_offset_rc + NODE_EDGE_KEY_FIRST + NUM_ALPHABET + nt*EDGE_SIZE;
+ }
+ else
+ {
+ edge_ptrs.offset = root_offset + NODE_EDGE_KEY_FIRST + NUM_ALPHABET + nt*EDGE_SIZE;
+ }
+ if(ReadData(header->excep_buff, EDGE_SIZE, edge_ptrs.offset) != EDGE_SIZE)
+ return MBError::READ_ERROR;
+
+ edge_ptrs.ptr = header->excep_buff;
+ edge_ptrs.len_ptr = edge_ptrs.ptr + EDGE_LEN_POS;
+ edge_ptrs.flag_ptr = edge_ptrs.ptr + EDGE_FLAG_POS;
+ edge_ptrs.offset_ptr = edge_ptrs.flag_ptr + 1;
+ return MBError::SUCCESS;
+}
+
+/////////////////////////////////////////////
+// Init root node in resource collection mode
+/////////////////////////////////////////////
+size_t DictMem::InitRootNode_RC()
+{
+ bool node_move;
+ uint8_t *root_node;
+
+ node_move = ReserveNode(NUM_ALPHABET-1, root_offset_rc, root_node);
+ root_node[0] = FLAG_NODE_NONE;
+ root_node[1] = NUM_ALPHABET-1;
+ for(int i = 0; i < NUM_ALPHABET; i++)
+ {
+ root_node[NODE_EDGE_KEY_FIRST+i] = static_cast<uint8_t>(i);
+ }
+
+ if(node_move)
+ WriteData(root_node, node_size[NUM_ALPHABET-1], root_offset_rc);
+
+ return root_offset_rc;
+}
+
+// No need to call LokcFree for removing all DB entries.
+// Note readers may get READ_ERROR as return, which should be expected
+// considering the full DB is deleted.
+int DictMem::ClearRootEdge(int nt) const
+{
+ size_t offset = root_offset + NODE_EDGE_KEY_FIRST + NUM_ALPHABET + nt*EDGE_SIZE;
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = offset;
+ header->excep_updating_status = EXCEP_STATUS_CLEAR_EDGE;
+ lfree->WriterLockFreeStart(offset);
+#endif
+ WriteData(DictMem::empty_edge, EDGE_SIZE, offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+
+ return MBError::SUCCESS;
+}
+
+int DictMem::ClearRootEdges_RC() const
+{
+ if(root_offset_rc == 0)
+ return MBError::INVALID_ARG;
+
+ size_t offset;
+ for(int i = 0; i < NUM_ALPHABET; i++)
+ {
+ offset = root_offset_rc + NODE_EDGE_KEY_FIRST + NUM_ALPHABET + i*EDGE_SIZE;
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = offset;
+ header->excep_updating_status = EXCEP_STATUS_CLEAR_EDGE;
+ lfree->WriterLockFreeStart(offset);
+#endif
+ DRMBase::WriteData(DictMem::empty_edge, EDGE_SIZE, offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+ }
+
+ return MBError::SUCCESS;
+}
+
+void DictMem::ClearMem() const
+{
+ int root_node_size = free_lists->GetAlignmentSize(node_size[NUM_ALPHABET-1]);
+ header->m_index_offset = root_offset + root_node_size;
+ header->n_states = 1; // Keep the root node
+ header->n_edges = 0;
+ header->edge_str_size = 0;
+ free_lists->Empty();
+ header->pending_index_buff_size = 0;
+}
+
+int DictMem::NextEdge(const uint8_t *key, EdgePtrs &edge_ptrs, uint8_t *node_buff,
+ MBData &mbdata) const
+{
+ size_t node_off;
+ // Check if need to read saved edge
+ if((mbdata.options & CONSTS::OPTION_READ_SAVED_EDGE) && edge_ptrs.offset == mbdata.edge_ptrs.offset)
+ node_off = Get6BInteger(mbdata.edge_ptrs.offset_ptr);
+ else
+ node_off = Get6BInteger(edge_ptrs.offset_ptr);
+
+ int byte_read;
+ byte_read = ReadData(node_buff, NODE_EDGE_KEY_FIRST, node_off);
+ if(byte_read != NODE_EDGE_KEY_FIRST)
+ return MBError::READ_ERROR;
+
+ int nt = node_buff[1] + 1;
+ byte_read = ReadData(node_buff+NODE_EDGE_KEY_FIRST, nt, node_off+NODE_EDGE_KEY_FIRST);
+ if(byte_read != nt)
+ return MBError::READ_ERROR;
+
+ int ret = MBError::NOT_EXIST;
+ for(int i = 0; i < nt; i++)
+ {
+ if(node_buff[i+NODE_EDGE_KEY_FIRST] == key[0])
+ {
+ if(mbdata.options & CONSTS::OPTION_FIND_AND_STORE_PARENT)
+ {
+ // update parent node/edge info for deletion
+ edge_ptrs.curr_nt = nt;
+ edge_ptrs.curr_edge_index = i;
+ edge_ptrs.parent_offset = edge_ptrs.offset;
+ edge_ptrs.curr_node_offset = node_off;
+ }
+ size_t offset_new = node_off + NODE_EDGE_KEY_FIRST + nt + i*EDGE_SIZE;
+ byte_read = ReadData(edge_ptrs.edge_buff, EDGE_SIZE, offset_new);
+ if(byte_read != EDGE_SIZE)
+ {
+ ret = MBError::READ_ERROR;
+ break;
+ }
+
+ edge_ptrs.offset = offset_new;
+ ret = MBError::SUCCESS;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+void DictMem::RemoveRootEdge(const EdgePtrs &edge_ptrs)
+{
+ // Clear the edge
+ // Root node needs special handling.
+ if(edge_ptrs.len_ptr[0] > LOCAL_EDGE_LEN)
+ ReleaseBuffer(Get5BInteger(edge_ptrs.ptr), edge_ptrs.len_ptr[0]-1);
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = edge_ptrs.offset;
+ header->excep_updating_status = EXCEP_STATUS_CLEAR_EDGE;
+ lfree->WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ WriteData(DictMem::empty_edge, EDGE_SIZE, edge_ptrs.offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+}
+
+int DictMem::RemoveEdgeSizeN(const EdgePtrs &edge_ptrs,
+ int nt,
+ size_t node_offset,
+ uint8_t *old_node_buffer,
+ size_t &str_off_rel,
+ int &str_size_rel,
+ size_t parent_edge_offset)
+{
+ // Reserve a new node with nt-1
+ bool node_move;
+ size_t new_node_offset;
+ uint8_t *node;
+
+ // Reserve for the new node
+ node_move = ReserveNode(nt-2, new_node_offset, node);
+
+ // Copy data from old node
+ uint8_t *first_key_ptr = node + NODE_EDGE_KEY_FIRST;
+ uint8_t *edge_ptr = first_key_ptr + nt - 1;
+ uint8_t old_edge_buff[16];
+ size_t old_edge_offset = node_offset + NODE_EDGE_KEY_FIRST + nt;
+ memcpy(node, old_node_buffer, NODE_EDGE_KEY_FIRST);
+ node[1] = nt - 2;
+ for(int i = 0; i < nt; i++)
+ {
+ // load the edge
+ if(ReadData(old_edge_buff, EDGE_SIZE, old_edge_offset) != EDGE_SIZE)
+ return MBError::READ_ERROR;
+
+ if(i == edge_ptrs.curr_edge_index)
+ {
+ // Need to release this edge string buffer
+ if(old_edge_buff[EDGE_LEN_POS] > LOCAL_EDGE_LEN)
+ {
+ str_off_rel = Get5BInteger(old_edge_buff);
+ str_size_rel = old_edge_buff[EDGE_LEN_POS]-1;
+ }
+ }
+ else
+ {
+ first_key_ptr[0] = old_node_buffer[NODE_EDGE_KEY_FIRST+i];
+ memcpy(edge_ptr, old_edge_buff, EDGE_SIZE);
+
+ first_key_ptr++;
+ edge_ptr += EDGE_SIZE;
+ }
+ old_edge_offset += EDGE_SIZE;
+ }
+
+ // Write the new node before free
+ if(node_move)
+ WriteData(node, node_size[nt-2], new_node_offset);
+
+ // Update the link from parent edge to the new node offset
+ Write6BInteger(header->excep_buff, new_node_offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStart(parent_edge_offset);
+#endif
+ WriteData(header->excep_buff, OFFSET_SIZE, parent_edge_offset+EDGE_NODE_LEADING_POS);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+#endif
+
+ return MBError::SUCCESS;
+}
+
+int DictMem::RemoveEdgeSizeOne(uint8_t *old_node_buffer,
+ size_t parent_edge_offset,
+ size_t node_offset,
+ int nt,
+ size_t &str_off_rel,
+ int &str_size_rel)
+{
+ int rval = MBError::SUCCESS;
+
+ if(old_node_buffer[0] & FLAG_NODE_MATCH)
+ {
+ uint8_t *parent_edge_buff = header->excep_buff;
+ size_t data_offset = Get6BInteger(old_node_buffer+2);
+ parent_edge_buff[0] = EDGE_FLAG_DATA_OFF;
+ Write6BInteger(parent_edge_buff+1, data_offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStart(parent_edge_offset);
+#endif
+ // Write the one-byte flag and 6-byte offset
+ WriteData(parent_edge_buff, OFFSET_SIZE_P1, parent_edge_offset+EDGE_FLAG_POS);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+#endif
+ }
+ else
+ {
+ // This is an internal node.
+ rval = MBError::TRY_AGAIN;
+ }
+
+ uint8_t old_edge_buff[16];
+ size_t old_edge_offset = node_offset + NODE_EDGE_KEY_FIRST + nt;
+ if(ReadData(old_edge_buff, EDGE_SIZE, old_edge_offset) != EDGE_SIZE)
+ return MBError::READ_ERROR;
+ if(old_edge_buff[EDGE_LEN_POS] > LOCAL_EDGE_LEN)
+ {
+ str_off_rel = Get5BInteger(old_edge_buff);
+ str_size_rel = old_edge_buff[EDGE_LEN_POS]-1;
+ }
+
+ return rval;
+}
+
+int DictMem::RemoveEdgeByIndex(const EdgePtrs &edge_ptrs, MBData &data)
+{
+ header->excep_offset = edge_ptrs.curr_node_offset;
+
+ if(header->excep_offset == root_offset)
+ {
+ RemoveRootEdge(edge_ptrs);
+ return MBError::SUCCESS;
+ }
+
+ int nt = edge_ptrs.curr_nt;
+ if(nt < 1) return MBError::INVALID_ARG;
+
+ uint8_t *old_node_buffer = data.node_buff;
+ // load the current node
+ if(ReadData(old_node_buffer, NODE_EDGE_KEY_FIRST + nt, edge_ptrs.curr_node_offset)
+ != NODE_EDGE_KEY_FIRST + nt)
+ return MBError::READ_ERROR;
+
+ int rval = MBError::SUCCESS;
+ size_t str_off_rel;
+ int str_size_rel = 0;
+
+ header->excep_lf_offset = edge_ptrs.parent_offset;
+ header->excep_updating_status = EXCEP_STATUS_REMOVE_EDGE;
+ if(nt > 1)
+ {
+ rval = RemoveEdgeSizeN(edge_ptrs, nt, header->excep_offset, old_node_buffer,
+ str_off_rel, str_size_rel, header->excep_lf_offset);
+ }
+ else
+ {
+ rval = DictMem::RemoveEdgeSizeOne(old_node_buffer, header->excep_lf_offset,
+ header->excep_offset, nt, str_off_rel, str_size_rel);
+ }
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+
+ header->n_edges--;
+ ReleaseNode(header->excep_offset, nt-1);
+ if(str_size_rel > 0)
+ ReleaseBuffer(str_off_rel, str_size_rel);
+
+ // Clear the edge
+#ifdef __LOCK_FREE__
+ header->excep_lf_offset = edge_ptrs.offset;
+ header->excep_updating_status = EXCEP_STATUS_CLEAR_EDGE;
+ lfree->WriterLockFreeStart(edge_ptrs.offset);
+#endif
+ WriteData(DictMem::empty_edge, EDGE_SIZE, edge_ptrs.offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+ header->excep_updating_status = EXCEP_STATUS_NONE;
+#endif
+
+ return rval;
+}
+
+// Should only be called by writer
+void DictMem::PrintStats(std::ostream &out_stream) const
+{
+ if(!is_valid)
+ return;
+
+ out_stream << "Dict Memory Stats:" << std::endl;
+ out_stream << "\tIndex size: " << header->m_index_offset << std::endl;
+ out_stream << "\tIndex block size: " << header->index_block_size << std::endl;
+ out_stream << "\tNumber of edges: " << header->n_edges << std::endl;
+ out_stream << "\tNumber of nodes: " << header->n_states << std::endl;
+ out_stream << "\tEdge string size: " << header->edge_str_size << std::endl;
+ out_stream << "\tEdge size: " << header->n_edges*EDGE_SIZE << std::endl;
+ out_stream << "\tException flag: " << header->excep_updating_status << std::endl;
+ out_stream << "\tPending Buffer Size: " << header->pending_index_buff_size << std::endl;
+ if(free_lists != NULL)
+ out_stream << "\tTrackable Buffer Size: " << free_lists->GetTotSize() << std::endl;
+ kv_file->PrintStats(out_stream);
+}
+
+const int* DictMem::GetNodeSizePtr() const
+{
+ return node_size;
+}
+
+void DictMem::ResetSlidingWindow() const
+{
+ kv_file->ResetSlidingWindow();
+ if(header != NULL)
+ header->shm_index_sliding_start.store(0, std::memory_order_relaxed);
+}
+
+void DictMem::InitLockFreePtr(LockFree *lf)
+{
+ lfree = lf;
+}
+
+void DictMem::Flush() const
+{
+ if(kv_file != NULL)
+ kv_file->Flush();
+ if(header_file != NULL)
+ header_file->Flush();
+}
+
+void DictMem::WriteData(const uint8_t *buff, unsigned len, size_t offset) const
+{
+ if(offset + len > header->m_index_offset)
+ {
+ std::cerr << "invalid dmm write: " << offset << " " << len << " "
+ << header->m_index_offset << "\n";
+ throw (int) MBError::OUT_OF_BOUND;
+ }
+
+ if(kv_file->RandomWrite(buff, len, offset) != len)
+ throw (int) MBError::WRITE_ERROR;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __DICTMEM_H__
+#define __DICTMEM_H__
+
+#include <string>
+#include <memory>
+#include <stdint.h>
+#include <string.h>
+#include <pthread.h>
+#include <assert.h>
+
+#include "drm_base.h"
+#include "db.h"
+#include "mabain_consts.h"
+#include "mb_data.h"
+#include "rollable_file.h"
+#include "mb_lsq.h"
+#include "free_list.h"
+#include "lock_free.h"
+#include "error.h"
+
+namespace mabain {
+
+typedef struct _NodePtrs
+{
+ size_t offset;
+ uint8_t *ptr;
+
+ uint8_t *edge_key_ptr;
+ uint8_t *edge_ptr;
+} NodePtrs;
+
+// Memory management class for the dictionary
+class DictMem : public DRMBase
+{
+public:
+ DictMem(const std::string &mbdir, bool init_header, size_t memsize,
+ int mode, uint32_t block_size, int max_num_blk, uint32_t queue_size);
+ void Destroy();
+ virtual ~DictMem();
+
+ bool IsValid() const;
+ void PrintStats(std::ostream &out_stream) const;
+
+ void InitNodePtrs(uint8_t *ptr, int nt, NodePtrs &node_ptrs);
+ void InitEdgePtrs(const NodePtrs &node_ptrs, int index,
+ EdgePtrs &edge_ptrs);
+ void AddRootEdge(EdgePtrs &edge_ptrs, const uint8_t *key, int len,
+ size_t data_offset);
+ int InsertNode(EdgePtrs &edge_ptrs, int match_len, size_t data_offset,
+ MBData &data);
+ int AddLink(EdgePtrs &edge_ptrs, int match_len, const uint8_t *key,
+ int key_len, size_t data_off, MBData &data);
+ int UpdateNode(EdgePtrs &edge_ptrs, const uint8_t *key, int key_len,
+ size_t data_off);
+ bool FindNext(const unsigned char *key, int keylen, int &match_len,
+ EdgePtrs &edge_ptr, uint8_t *key_tmp) const;
+ int GetRootEdge(size_t rc_off, int nt, EdgePtrs &edge_ptrs) const;
+ int GetRootEdge_Writer(bool rc_mode, int nt, EdgePtrs &edge_ptrs) const;
+ int ClearRootEdge(int nt) const;
+ void ReserveData(const uint8_t* key, int size, size_t &offset,
+ bool map_new_sliding=true);
+ int NextEdge(const uint8_t *key, EdgePtrs &edge_ptrs,
+ uint8_t *tmp_buff, MBData &mbdata) const;
+ int RemoveEdgeByIndex(const EdgePtrs &edge_ptrs, MBData &data);
+ void InitRootNode();
+ inline void WriteEdge(const EdgePtrs &edge_ptrs) const;
+ void WriteData(const uint8_t *buff, unsigned len, size_t offset) const;
+ inline size_t GetRootOffset() const;
+ void ClearMem() const;
+ const int* GetNodeSizePtr() const;
+ void ResetSlidingWindow() const;
+
+ void InitLockFreePtr(LockFree *lf);
+
+ void Flush() const;
+
+ // Updates in RC mode
+ size_t InitRootNode_RC();
+ int ClearRootEdges_RC() const;
+
+ // empty edge, used for clearing edges
+ static const uint8_t empty_edge[EDGE_SIZE];
+
+private:
+ bool ReserveNode(int nt, size_t &offset, uint8_t* &ptr);
+ void ReleaseNode(size_t offset, int nt);
+ void ReleaseBuffer(size_t offset, int size);
+ void UpdateTailEdge(EdgePtrs &edge_ptrs, int match_len, MBData &data,
+ EdgePtrs &tail_edge, uint8_t &new_key_first,
+ bool &map_new_sliding);
+ void UpdateHeadEdge(EdgePtrs &edge_ptrs, int match_len,
+ MBData &data, int &release_buffer_size,
+ size_t &edge_str_off, bool &map_new_sliding);
+ void RemoveRootEdge(const EdgePtrs &edge_ptrs);
+ int RemoveEdgeSizeN(const EdgePtrs &edge_ptrs, int nt, size_t node_offset,
+ uint8_t *old_node_buffer, size_t &str_off_rel,
+ int &str_size_rel, size_t parent_edge_offset);
+ int RemoveEdgeSizeOne(uint8_t *old_node_buffer, size_t parent_edge_offset,
+ size_t node_offset, int nt, size_t &str_off_rel,
+ int &str_size_rel);
+
+ int *node_size;
+ bool is_valid;
+
+ size_t root_offset;
+ uint8_t *node_ptr;
+
+ // lock free pointer
+ LockFree *lfree;
+
+ // header file
+ std::shared_ptr<MmapFileIO> header_file;
+
+ size_t root_offset_rc;
+};
+
+inline void DictMem::WriteEdge(const EdgePtrs &edge_ptrs) const
+{
+ if(edge_ptrs.offset + EDGE_SIZE > header->m_index_offset)
+ {
+ std::cerr << "invalid edge write: " << edge_ptrs.offset << " " << EDGE_SIZE
+ << " " << header->m_index_offset << "\n";
+ throw (int) MBError::OUT_OF_BOUND;
+ }
+
+ if(kv_file->RandomWrite(edge_ptrs.ptr, EDGE_SIZE, edge_ptrs.offset) != EDGE_SIZE)
+ throw (int) MBError::WRITE_ERROR;
+}
+
+inline size_t DictMem::GetRootOffset() const
+{
+ return root_offset;
+}
+
+// update the edge pointers for fast access
+// node_ptrs.offset and node_ptrs.ptr[1] must already be populated before calling this function
+inline void DictMem::InitEdgePtrs(const NodePtrs &node_ptrs, int index, EdgePtrs &edge_ptrs)
+{
+ int edge_off = NODE_EDGE_KEY_FIRST + node_ptrs.ptr[1] + 1 + index*EDGE_SIZE;
+ edge_ptrs.offset = node_ptrs.offset + edge_off;
+ edge_ptrs.ptr = node_ptrs.ptr + edge_off;
+ edge_ptrs.len_ptr = edge_ptrs.ptr + EDGE_LEN_POS;
+ edge_ptrs.flag_ptr = edge_ptrs.ptr + EDGE_FLAG_POS;
+ edge_ptrs.offset_ptr = edge_ptrs.flag_ptr + 1;
+}
+
+inline void InitTempEdgePtrs(EdgePtrs &edge_ptrs)
+{
+ edge_ptrs.ptr = edge_ptrs.edge_buff;
+ edge_ptrs.len_ptr = edge_ptrs.ptr + EDGE_LEN_POS;
+ edge_ptrs.flag_ptr = edge_ptrs.ptr + EDGE_FLAG_POS;
+ edge_ptrs.offset_ptr = edge_ptrs.flag_ptr + 1;
+}
+
+// node_ptrs.offset must be populated before caling this function
+inline void DictMem::InitNodePtrs(uint8_t *ptr, int nt, NodePtrs &node_ptrs)
+{
+ node_ptrs.ptr = ptr;
+ nt++;
+ node_ptrs.edge_key_ptr = ptr + NODE_EDGE_KEY_FIRST;
+ node_ptrs.edge_ptr = node_ptrs.edge_key_ptr + nt;
+}
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include "error.h"
+#include "drm_base.h"
+#include "version.h"
+
+namespace mabain {
+
+void DRMBase::ReadHeaderVersion(const std::string &header_path, uint16_t ver[4])
+{
+ memset(ver, 0, sizeof(uint16_t)*4);
+ FILE *hdr_file = fopen(header_path.c_str(), "rb");
+ if(hdr_file == NULL)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to open header file %s", header_path.c_str());
+ throw (int) MBError::OPEN_FAILURE;
+ }
+ if(fread(ver, sizeof(uint16_t), 4, hdr_file) != 4)
+ {
+ fclose(hdr_file);
+ throw (int) MBError::READ_ERROR;
+ }
+ fclose(hdr_file);
+}
+
+void DRMBase::ReadHeader(const std::string &header_path, uint8_t *buff, int buf_size)
+{
+ FILE *hdr_file = fopen(header_path.c_str(), "rb");
+ if(hdr_file == NULL)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to open header file %s", header_path.c_str());
+ throw (int) MBError::OPEN_FAILURE;
+ }
+ if(fread(buff, buf_size, 1, hdr_file) != 1)
+ {
+ fclose(hdr_file);
+ throw (int) MBError::READ_ERROR;
+ }
+ fclose(hdr_file);
+}
+
+void DRMBase::WriteHeader(const std::string &header_path, uint8_t *buff, int queue_buff_size)
+{
+ FILE *hdr_file = fopen(header_path.c_str(), "wb");
+ if(hdr_file == NULL)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to open header file %s", header_path.c_str());
+ throw (int) MBError::OPEN_FAILURE;
+ }
+ if(fwrite(buff, RollableFile::page_size, 1, hdr_file) != 1)
+ {
+ fclose(hdr_file);
+ throw (int) MBError::WRITE_ERROR;
+ }
+ memset(buff, 0, RollableFile::page_size);
+ for(int i = 0; i < queue_buff_size / RollableFile::page_size; i++)
+ {
+ if(fwrite(buff, RollableFile::page_size, 1, hdr_file) != 1)
+ {
+ fclose(hdr_file);
+ throw (int) MBError::WRITE_ERROR;
+ }
+ }
+ size_t size_left = queue_buff_size % RollableFile::page_size;
+ if(size_left > 0)
+ {
+ if(fwrite(buff, size_left, 1, hdr_file) != 1)
+ {
+ fclose(hdr_file);
+ throw (int) MBError::WRITE_ERROR;
+ }
+ }
+ fclose(hdr_file);
+}
+
+void DRMBase::ValidateHeaderFile(const std::string &header_path, int mode,
+ int queue_buff_size, bool &update_header)
+{
+ uint16_t hdr_ver[4];
+ ReadHeaderVersion(header_path, hdr_ver);
+ if(hdr_ver[0] >= 1 && hdr_ver[1] >= 2) return;
+ if(!(mode & CONSTS::ACCESS_MODE_WRITER))
+ throw (int) MBError::VERSION_MISMATCH;
+
+ Logger::Log(LOG_LEVEL_INFO, "header version: %u.%u.%u does not match "
+ "library version: %u.%u.%u",
+ hdr_ver[0], hdr_ver[1], hdr_ver[2],
+ version[0], version[1], version[2]);
+ // Load the old header first
+ uint8_t buff[RollableFile::page_size];
+ ReadHeader(header_path, buff, RollableFile::page_size);
+ // Update version field
+ uint16_t *curr_version = reinterpret_cast<uint16_t*>(buff);
+ memcpy(curr_version, version, 4*sizeof(uint16_t));
+
+ // Write the new header file
+ std::string tmp_header_path = header_path + ".tmp";
+ Logger::Log(LOG_LEVEL_INFO, "updating header to newer version %s", tmp_header_path.c_str());
+ WriteHeader(tmp_header_path, buff, queue_buff_size);
+
+ // Swap header files
+ std::string old_header_path = header_path + "_" + std::to_string(hdr_ver[0]) + "_"
+ + std::to_string(hdr_ver[1]) + "_" + std::to_string(hdr_ver[2]);
+ if(rename(header_path.c_str(), old_header_path.c_str()) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to move file %s to %s", header_path.c_str(),
+ old_header_path.c_str());
+ throw (int) MBError::OPEN_FAILURE;
+ }
+ if(rename(tmp_header_path.c_str(), header_path.c_str()) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to move file %s to %s", tmp_header_path.c_str(),
+ header_path.c_str());
+ throw (int) MBError::OPEN_FAILURE;
+ }
+
+ update_header = true;
+}
+
+void DRMBase::PrintHeader(std::ostream &out_stream) const
+{
+ if(header == NULL)
+ return;
+
+ out_stream << "---------------- START OF HEADER ----------------" << std::endl;
+ out_stream << "version: " << header->version[0] << "." <<
+ header->version[1] << "." <<
+ header->version[2] << std::endl;
+ out_stream << "data size: " << header->data_size << std::endl;
+ out_stream << "db count: " << header->count << std::endl;
+ out_stream << "max data offset: " << header->m_data_offset << std::endl;
+ out_stream << "max index offset: " << header->m_index_offset << std::endl;
+ out_stream << "pending data buffer size: " << header->pending_data_buff_size << std::endl;
+ out_stream << "pending index buffer size: " << header->pending_index_buff_size << std::endl;
+ out_stream << "node count: " << header->n_states << std::endl;
+ out_stream << "edge count: " << header->n_edges << std::endl;
+ out_stream << "edge string size: " << header->edge_str_size << std::endl;
+ out_stream << "writer count: " << header->num_writer << std::endl;
+ out_stream << "reader count: " << header->num_reader << std::endl;
+ out_stream << "data sliding start: " << header->shm_data_sliding_start << std::endl;
+ out_stream << "index sliding start: " << header->shm_index_sliding_start << std::endl;
+ out_stream << "data block size: " << header->data_block_size << std::endl;
+ out_stream << "index block size: " << header->index_block_size << std::endl;
+ out_stream << "lock free data: " << std::endl;
+ out_stream << "\tcounter: " << header->lock_free.counter << std::endl;
+ out_stream << "\toffset: " << header->lock_free.offset << std::endl;
+ out_stream << "number of updates: " << header->num_update << std::endl;
+ out_stream << "entry count per bucket: " << header->entry_per_bucket << std::endl;
+ out_stream << "eviction bucket index: " << header->eviction_bucket_index << std::endl;
+ out_stream << "exception data: " << std::endl;
+ out_stream << "\tupdating status: " << header->excep_updating_status << std::endl;
+ out_stream << "\texception data buffer: ";
+ char data_str_buff[MB_EXCEPTION_BUFF_SIZE*3 + 1];
+ for(int i = 0; i < MB_EXCEPTION_BUFF_SIZE; i++)
+ {
+ sprintf(data_str_buff + 3*i, "%2x ", header->excep_buff[i]);
+ }
+ data_str_buff[MB_EXCEPTION_BUFF_SIZE*3] = '\0';
+ out_stream << data_str_buff << std::endl;
+ out_stream << "\toffset: " << header->excep_offset << std::endl;
+ out_stream << "\tlock free offset: " << header->excep_lf_offset << std::endl;
+ out_stream << "max index offset before rc: " << header->rc_m_index_off_pre << std::endl;
+ out_stream << "max data offset before rc: " << header->rc_m_data_off_pre << std::endl;
+ out_stream << "rc root offset: " << header->rc_root_offset << std::endl;
+ out_stream << "rc count: " << header->rc_count << std::endl;
+ out_stream << "shared memory queue size: " << header->async_queue_size << std::endl;
+ out_stream << "shared memory queue index: " << header->queue_index << std::endl;
+ out_stream << "shared memory writer index: " << header->writer_index << std::endl;
+ out_stream << "resource flag: " << header->rc_flag << std::endl;
+ out_stream << "---------------- END OF HEADER ----------------" << std::endl;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __DRM_BASE_H__
+#define __DRM_BASE_H__
+
+#include "rollable_file.h"
+#include "free_list.h"
+
+#define DATA_BUFFER_ALIGNMENT 1
+#define DATA_SIZE_BYTE 2
+#define DATA_HDR_BYTE 4
+#define OFFSET_SIZE 6
+#define EDGE_SIZE 13
+#define EDGE_LEN_POS 5
+#define EDGE_FLAG_POS 6
+#define EDGE_FLAG_DATA_OFF 0x01
+#define FLAG_NODE_MATCH 0x01
+#define FLAG_NODE_NONE 0x0
+#define BUFFER_ALIGNMENT 1
+#define LOCAL_EDGE_LEN 6
+#define LOCAL_EDGE_LEN_M1 5
+#define EDGE_NODE_LEADING_POS 7
+#define EXCEP_STATUS_NONE 0
+#define EXCEP_STATUS_ADD_EDGE 1
+#define EXCEP_STATUS_ADD_DATA_OFF 2
+#define EXCEP_STATUS_ADD_NODE 3
+#define EXCEP_STATUS_REMOVE_EDGE 4
+#define EXCEP_STATUS_CLEAR_EDGE 5
+#define EXCEP_STATUS_RC_NODE 6
+#define EXCEP_STATUS_RC_EDGE_STR 7
+#define EXCEP_STATUS_RC_DATA 8
+#define EXCEP_STATUS_RC_TREE 9
+#define MB_EXCEPTION_BUFF_SIZE 16
+
+namespace mabain {
+
+// Mabain DB header
+typedef struct _IndexHeader
+{
+ uint16_t version[4];
+ int data_size;
+ int64_t count;
+ size_t m_data_offset;
+ size_t m_index_offset;
+ int64_t pending_data_buff_size;
+ int64_t pending_index_buff_size;
+ int64_t n_states;
+ int64_t n_edges;
+ int64_t edge_str_size;
+ int num_writer;
+ int num_reader;
+ std::atomic<size_t> shm_index_sliding_start;
+ std::atomic<size_t> shm_data_sliding_start;
+
+ // Lock-free data structure
+ LockFreeShmData lock_free;
+
+ // read/write lock
+ pthread_rwlock_t mb_rw_lock;
+
+ // block size
+ uint32_t index_block_size;
+ uint32_t data_block_size;
+ // number of entry per bucket for eviction
+ int64_t entry_per_bucket;
+ // number of DB insertions and updates
+ // used for assigning bucket
+ int64_t num_update;
+ uint16_t eviction_bucket_index;
+
+ // temp variables used for abnormal writer terminations
+ int excep_updating_status;
+ uint8_t excep_buff[MB_EXCEPTION_BUFF_SIZE];
+ size_t excep_offset;
+ size_t excep_lf_offset;
+
+ // index root offset for insertions during rc
+ size_t rc_m_index_off_pre;
+ size_t rc_m_data_off_pre;
+ std::atomic<size_t> rc_root_offset;
+ int64_t rc_count;
+
+ // multi-process async queue
+ int async_queue_size;
+ std::atomic<uint32_t> queue_index;
+ uint32_t writer_index;
+ std::atomic<uint32_t> rc_flag;
+} IndexHeader;
+
+// An abstract interface class for Dict and DictMem
+class DRMBase
+{
+public:
+ DRMBase()
+ {
+ // Derived classes will initialize these objects.
+ kv_file = NULL;
+ free_lists = NULL;
+ }
+
+ ~DRMBase()
+ {
+ }
+
+ inline virtual void WriteData(const uint8_t *buff, unsigned len, size_t offset) const = 0;
+ inline int Reserve(size_t &offset, int size, uint8_t* &ptr);
+ inline uint8_t* GetShmPtr(size_t offset, int size) const;
+ inline size_t CheckAlignment(size_t offset, int size) const;
+ inline int ReadData(uint8_t *buff, unsigned len, size_t offset) const;
+ inline size_t GetResourceCollectionOffset() const;
+ inline void RemoveUnused(size_t max_size, bool writer_mode = false);
+
+ FreeList *GetFreeList() const
+ {
+ return free_lists;
+ }
+
+ IndexHeader *GetHeaderPtr() const
+ {
+ return header;
+ }
+
+ void PrintHeader(std::ostream &out_stream) const;
+
+ static void ValidateHeaderFile(const std::string &header_path, int mode, int queue_buff_size,
+ bool &update_header);
+
+protected:
+ static void ReadHeaderVersion(const std::string &header_path, uint16_t ver[4]);
+ static void ReadHeader(const std::string &header_path, uint8_t *buff, int buf_size);
+ static void WriteHeader(const std::string &header_path, uint8_t *buff, int queue_buff_size);
+
+ IndexHeader *header;
+ RollableFile *kv_file;
+ FreeList *free_lists;
+};
+
+inline void DRMBase::WriteData(const uint8_t *buff, unsigned len, size_t offset) const
+{
+ if(kv_file->RandomWrite(buff, len, offset) != len)
+ throw (int) MBError::WRITE_ERROR;
+}
+
+inline int DRMBase::Reserve(size_t &offset, int size, uint8_t* &ptr)
+{
+ return kv_file->Reserve(offset, size, ptr);
+}
+
+inline uint8_t* DRMBase::GetShmPtr(size_t offset, int size) const
+{
+ return kv_file->GetShmPtr(offset, size);
+}
+
+inline size_t DRMBase::CheckAlignment(size_t offset, int size) const
+{
+ return kv_file->CheckAlignment(offset, size);
+}
+
+inline int DRMBase::ReadData(uint8_t *buff, unsigned len, size_t offset) const
+{
+ return kv_file->RandomRead(buff, len, offset);
+}
+
+inline size_t DRMBase::GetResourceCollectionOffset() const
+{
+ return kv_file->GetResourceCollectionOffset();
+}
+
+inline void DRMBase::RemoveUnused(size_t max_size, bool writer_mode)
+{
+ return kv_file->RemoveUnused(max_size, writer_mode);
+}
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include "error.h"
+
+namespace mabain {
+
+const int MBError::MAX_ERROR_CODE = NO_DB;
+const char* MBError::error_str[] = {
+ "success",
+ "no memory",
+ "out of bound",
+ "invalid argument",
+ "not initialized",
+ "no existence",
+ "found in DB",
+ "mmap failed",
+ "no permission",
+ "file open failure",
+ "file write error",
+ "file read error",
+ "size not right",
+ "try again",
+ "resource allocation error",
+ "mutex error",
+ "unknown error",
+ "writer already running",
+ "no resource available",
+ "database closed",
+ "buffer discarded", // buffer will be reclaimed by shrink
+ "failed to create thread",
+ "rc skipped",
+ "version mismatch",
+
+ ///////////////////////////////////
+ "DB not exist",
+};
+
+const char* MBError::get_error_str(int err)
+{
+ if(err < 0)
+ return "db error";
+ else if(err > MAX_ERROR_CODE)
+ return "invalid error code";
+
+ return error_str[err];
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __MBERROR_H__
+#define __MBERROR_H__
+
+#undef TRY_AGAIN
+
+namespace mabain {
+
+// mabain errors
+class MBError
+{
+public:
+ enum mb_error
+ {
+ SUCCESS = 0,
+ NO_MEMORY = 1,
+ OUT_OF_BOUND = 2,
+ INVALID_ARG = 3,
+ NOT_INITIALIZED = 4,
+ NOT_EXIST = 5,
+ IN_DICT = 6,
+ MMAP_FAILED = 7,
+ NOT_ALLOWED = 8,
+ OPEN_FAILURE = 9,
+ WRITE_ERROR = 10,
+ READ_ERROR = 11,
+ INVALID_SIZE = 12,
+ TRY_AGAIN = 13,
+ ALLOCATION_ERROR = 14,
+ MUTEX_ERROR = 15,
+ UNKNOWN_ERROR = 16,
+ WRITER_EXIST = 17,
+ NO_RESOURCE = 18,
+ DB_CLOSED = 19,
+ BUFFER_LOST = 20,
+ THREAD_FAILED = 21,
+ RC_SKIPPED = 22,
+ VERSION_MISMATCH = 23,
+
+ // NO_DB should be the last enum.
+ NO_DB
+ };
+
+ static const int MAX_ERROR_CODE;
+ static const char* error_str[];
+ static const char* get_error_str(int err);
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <sys/mman.h>
+#include <errno.h>
+
+#include "file_io.h"
+
+namespace mabain {
+
+FileIO::FileIO(const std::string &fpath, int oflags, int fmode, bool sync)
+ : path(fpath),
+ options(oflags),
+ sync_on_write(sync),
+ mode(fmode)
+{
+ fd = -1;
+}
+
+FileIO::~FileIO()
+{
+ if(fd > 0)
+ close(fd);
+}
+
+int FileIO::Open()
+{
+ mode_t prev_mask = umask(0);
+ fd = open(path.c_str(), options, mode);
+ umask(prev_mask);
+
+ return fd;
+}
+
+size_t FileIO::Write(const void *data, size_t size)
+{
+ if(options & MMAP_ANONYMOUS_MODE)
+ return 0;
+
+ size_t bytes_written;
+
+ if(fd > 0)
+ {
+ bytes_written = write(fd, data, size);
+ if(sync_on_write) fsync(fd);
+ }
+ else
+ {
+ bytes_written = 0;
+ }
+
+ return bytes_written;
+}
+
+size_t FileIO::Read(void *buff, size_t size)
+{
+ if(options & MMAP_ANONYMOUS_MODE)
+ return 0;
+
+ size_t bytes_read;
+
+ if(fd > 0)
+ {
+ bytes_read = read(fd, buff, size);
+ }
+ else
+ {
+ bytes_read = 0;
+ }
+
+ return bytes_read;
+}
+
+size_t FileIO::RandomWrite(const void *data, size_t size, off_t offset)
+{
+ if(options & MMAP_ANONYMOUS_MODE)
+ return 0;
+
+ size_t bytes_written;
+
+ if(fd > 0)
+ {
+ bytes_written = pwrite(fd, data, size, offset);
+ if(sync_on_write) fsync(fd);
+ }
+ else
+ {
+ bytes_written = 0;
+ }
+
+ return bytes_written;
+}
+
+size_t FileIO::RandomRead(void *buff, size_t size, off_t offset)
+{
+ if(options & MMAP_ANONYMOUS_MODE)
+ return 0;
+
+ size_t bytes_read;
+
+ if(fd > 0)
+ {
+ bytes_read = pread(fd, buff, size, offset);
+ }
+ else
+ {
+ bytes_read = 0;
+ }
+
+ return bytes_read;
+}
+
+void* FileIO::MapFile(size_t size, int prot, int flags, off_t offset)
+{
+ return mmap(NULL, size, prot, flags, fd, offset);
+}
+
+off_t FileIO::SetOffset(off_t offset)
+{
+ return lseek(fd, offset, SEEK_SET);
+}
+
+void FileIO::Close()
+{
+ if(fd > 0)
+ {
+ close(fd);
+ fd = -1;
+ }
+}
+
+bool FileIO::IsOpen() const
+{
+ return fd > 0;
+}
+
+int FileIO::TruncateFile(off_t filesize)
+{
+ if(fd > 0)
+ return ftruncate(fd, filesize);
+
+ return 1;
+}
+
+void FileIO::Flush()
+{
+ if(fd > 0)
+ fsync(fd);
+}
+
+const std::string& FileIO::GetFilePath() const
+{
+ return path;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __FILE_IO_H__
+#define __FILE_IO_H__
+
+#include <string>
+#include <iostream>
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+namespace mabain {
+
+#define MMAP_ANONYMOUS_MODE 0x80000000 // This bit should not be used in fcntl.h.
+
+// This is the basic file io class
+class FileIO
+{
+public:
+ FileIO(const std::string &fpath, int oflags, int fmode, bool sync);
+ virtual ~FileIO();
+
+ int Open();
+ int TruncateFile(off_t filesize);
+ bool IsOpen() const;
+ void Close();
+
+ size_t Write(const void *data, size_t bytes);
+ size_t Read(void *buff, size_t bytes);
+ off_t SetOffset(off_t offset);
+
+ virtual size_t RandomWrite(const void *data, size_t size, off_t offset);
+ virtual size_t RandomRead(void *buff, size_t size, off_t offset);
+ virtual void Flush();
+
+ const std::string& GetFilePath() const;
+
+protected:
+ std::string path;
+ int options;
+ bool sync_on_write;
+
+ void* MapFile(size_t size, int prot, int flags, off_t offset);
+
+private:
+ int mode;
+
+ int fd;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <iostream>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <assert.h>
+
+#include "free_list.h"
+#include "error.h"
+#include "logger.h"
+#include "lock_free.h"
+
+namespace mabain {
+
+FreeList::FreeList(const std::string &file_path, size_t buff_alignment,
+ size_t max_n_buff, size_t max_buff_per_list)
+ : list_path(file_path),
+ alignment(buff_alignment),
+ max_num_buffer(max_n_buff),
+ max_buffer_per_list(max_buff_per_list),
+ buffer_free_list(NULL),
+ count(0),
+ tot_size(0)
+{
+ // rel_parent_off in ResourceCollection is defined as 2-byte signed integer.
+ // The maximal buffer size cannot be greather than 32767.
+ assert(GetBufferSizeByIndex(max_n_buff-1) <= 65535);
+
+ Logger::Log(LOG_LEVEL_INFO, "%s maximum number of buffers: %d", file_path.c_str(),
+ max_num_buffer);
+ buffer_free_list = new MBlsq*[max_num_buffer];
+ for(size_t i = 0; i < max_num_buffer; i++)
+ {
+ buffer_free_list[i] = new MBlsq(NULL);
+ }
+}
+
+FreeList::~FreeList()
+{
+ if(buffer_free_list)
+ {
+ for(size_t i = 0; i < max_num_buffer; i++)
+ {
+ if(buffer_free_list[i])
+ {
+ while(buffer_free_list[i]->Count())
+ buffer_free_list[i]->RemoveIntFromHead();
+ delete buffer_free_list[i];
+ }
+ }
+ delete [] buffer_free_list;
+ }
+}
+
+int FreeList::ReuseBuffer(size_t buf_index, size_t offset)
+{
+ int rval = MBError::BUFFER_LOST;
+ for(size_t i = buf_index - 1; i > 0; i--)
+ {
+ if(buffer_free_list[i]->Count() > max_buffer_per_list)
+ continue;
+
+ if(buffer_free_list[i]->AddIntToTail(offset) == MBError::SUCCESS)
+ {
+ count++;
+ tot_size += (i + 1) * alignment;
+ rval = MBError::SUCCESS;
+ }
+ break;
+ }
+
+ return rval;
+}
+
+int FreeList::AddBuffer(size_t offset, size_t size)
+{
+ int rval = MBError::SUCCESS;
+
+ size_t buf_index = GetBufferIndex(size);
+#ifdef __DEBUG__
+ assert(buf_index < max_num_buffer);
+#endif
+
+ if(buffer_free_list[buf_index]->Count() > (unsigned)max_buffer_per_list)
+ {
+ ReuseBuffer(buf_index, offset);
+ return rval;
+ }
+
+ buffer_free_list[buf_index]->AddIntToTail(offset);
+ count++;
+ tot_size += (buf_index + 1) * alignment;
+ return rval;
+}
+
+int FreeList::RemoveBuffer(size_t &offset, size_t size)
+{
+ int rval = MBError::NO_MEMORY;
+
+ size_t buf_index = GetBufferIndex(size);
+ if(buffer_free_list[buf_index]->Count() > 0)
+ {
+ offset = buffer_free_list[buf_index]->RemoveIntFromHead();
+ rval = MBError::SUCCESS;
+ count--;
+ tot_size -= (buf_index + 1) * alignment;
+ }
+
+ return rval;
+}
+
+size_t FreeList::GetTotSize() const
+{
+ return tot_size;
+}
+
+int64_t FreeList::Count() const
+{
+ return count;
+}
+
+int FreeList::StoreListOnDisk()
+{
+ if(buffer_free_list == NULL)
+ return MBError::NOT_ALLOWED;
+
+ int rval = MBError::SUCCESS;
+
+ if(count == 0)
+ return rval;
+
+ std::ofstream freelist_f(list_path.c_str(), std::fstream::out | std::fstream::binary);
+ if(!freelist_f.is_open())
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "cannot open file " + list_path);
+ return MBError::OPEN_FAILURE;
+ }
+
+ Logger::Log(LOG_LEVEL_INFO, "%s write %lld buffers to list disk: %llu", list_path.c_str(),
+ count, tot_size);
+ for(size_t buf_index = 0; buf_index < max_num_buffer; buf_index++)
+ {
+ int64_t buf_count = buffer_free_list[buf_index]->Count();
+ if(buf_count > 0)
+ {
+ // write list header (buffer index, buffer count)
+ freelist_f.write((char *)&buf_index, sizeof(size_t));
+ freelist_f.write((char *)&buf_count, sizeof(int64_t));
+ for(int64_t i = 0; i < buf_count; i++)
+ {
+ size_t offset = buffer_free_list[buf_index]->RemoveIntFromHead();
+ freelist_f.write((char *) &offset, sizeof(size_t));
+ count--;
+ tot_size -= (buf_index + 1) * alignment;
+ }
+ }
+ }
+
+ freelist_f.close();
+
+ return rval;
+}
+
+int FreeList::LoadListFromDisk()
+{
+ if(buffer_free_list == NULL)
+ return MBError::NOT_ALLOWED;
+
+ if(access(list_path.c_str(), F_OK) != 0)
+ {
+ if(errno == ENOENT)
+ {
+ Logger::Log(LOG_LEVEL_INFO, list_path + " does not exist");
+ return MBError::SUCCESS;
+ }
+
+ Logger::Log(LOG_LEVEL_ERROR, "cannot access %s with full permission: %d",
+ list_path.c_str(), errno);
+ return MBError::NOT_ALLOWED;
+ }
+
+ // Read the file
+ std::ifstream freelist_f(list_path.c_str(), std::fstream::in | std::fstream::binary);
+ if(!freelist_f.is_open())
+ return MBError::OPEN_FAILURE;
+
+ while(!freelist_f.eof())
+ {
+ size_t buf_index;
+ int64_t buf_count;
+ // Read header
+ freelist_f.read((char *) &buf_index, sizeof(size_t));
+ freelist_f.read((char *) &buf_count, sizeof(int64_t));
+ if(freelist_f.eof())
+ break;
+ for(int64_t i = 0; i < buf_count; i++)
+ {
+ size_t offset;
+ freelist_f.read((char *) &offset, sizeof(size_t));
+ buffer_free_list[buf_index]->AddIntToTail(offset);
+ count++;
+ tot_size += (buf_index + 1) * alignment;
+ }
+ }
+
+ freelist_f.close();
+
+ // Remove the file
+ if(unlink(list_path.c_str()) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to delete file %s: %d ",
+ list_path.c_str(), errno);
+ return MBError::WRITE_ERROR;
+ }
+
+ Logger::Log(LOG_LEVEL_INFO, "%s read %lld buffers to free list: %llu",
+ list_path.c_str(), count, tot_size);
+
+ return MBError::SUCCESS;
+}
+
+void FreeList::ReleaseAlignmentBuffer(size_t old_offset, size_t alignment_offset)
+{
+ if(alignment_offset <= old_offset)
+ return;
+
+ int rval = AddBuffer(old_offset, alignment_offset - old_offset);
+ if(rval != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_ERROR, "failed to release alignment buffer");
+}
+
+void FreeList::Empty()
+{
+ for(size_t i = 0; i < max_num_buffer; i++)
+ {
+ if(buffer_free_list[i])
+ {
+ buffer_free_list[i]->Clear();
+ }
+ }
+ count = 0;
+ tot_size = 0;
+}
+
+bool FreeList::GetBufferByIndex(size_t buf_index, size_t &offset)
+{
+#ifdef __DEBUG__
+ assert(buf_index < max_num_buffer);
+#endif
+ MBlsq *flist = buffer_free_list[buf_index];
+ if(flist->Count() > 0)
+ {
+ offset = flist->RemoveIntFromHead();
+ count--;
+ tot_size -= (buf_index + 1) * alignment;
+ return true;
+ }
+
+ return false;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __FREE_LIST_H__
+#define __FREE_LIST_H__
+
+#include <cstdlib>
+#include <string>
+
+#include "mb_lsq.h"
+#include "error.h"
+#include "lock_free.h"
+
+#define MAX_BUFFER_PER_LIST 256
+
+// Manage resource allocation/free using linked list
+namespace mabain {
+
+typedef struct _BufferCache
+{
+ size_t buf_index;
+ size_t buf_offset;
+} BufferCache;
+
+class FreeList
+{
+public:
+ FreeList(const std::string &file_path, size_t buff_alignment, size_t max_n_buff,
+ size_t max_buff_per_list = MAX_BUFFER_PER_LIST);
+ ~FreeList();
+
+ // Free a buffer by adding it to the free list
+ int AddBuffer(size_t offset, size_t size);
+ // Reserve a buffer by removing it from the free list
+ int RemoveBuffer(size_t &offset, size_t size);
+ // Release alignment buffer
+ void ReleaseAlignmentBuffer(size_t old_offset, size_t alignment_offset);
+
+ bool GetBufferByIndex(size_t buf_index, size_t &offset);
+
+ void Empty();
+
+ // Read buffer list from disk
+ int LoadListFromDisk();
+ // Save buffer list to disk
+ int StoreListOnDisk();
+ // Get buffer count
+ int64_t Count() const;
+ // Get total freed buffer size in the list
+ size_t GetTotSize() const;
+
+ inline int AddBufferByIndex(size_t buf_index, size_t offset);
+ inline size_t RemoveBufferByIndex(size_t buf_index);
+ inline size_t GetAlignmentSize(size_t size) const;
+ inline size_t GetBufferIndex(size_t size) const;
+ inline uint64_t GetBufferCountByIndex(size_t buf_index) const;
+ inline size_t GetBufferSizeByIndex(size_t buf_index) const;
+ inline int ReleaseBuffer(size_t offset, size_t size);
+
+private:
+ int ReuseBuffer(size_t buf_index, size_t offset);
+
+ // file path where the list will be serialized and stored
+ std::string list_path;
+ // buffer/memory alignment
+ size_t alignment;
+ // maximum number of buffers
+ size_t max_num_buffer;
+ // maximum buffer per list
+ // This restriction is to limit memory usage.
+ size_t max_buffer_per_list;
+ // number of separated buffers
+ MBlsq **buffer_free_list;
+ // total count of freed buffers
+ int64_t count;
+ // totol size allocted for all the buffers
+ size_t tot_size;
+};
+
+inline size_t FreeList::GetAlignmentSize(size_t size) const
+{
+#ifdef __DEBUG__
+ assert(size > 0 && size < max_num_buffer*alignment);
+#endif
+ size_t alignment_mod = size % alignment;
+ if(alignment_mod == 0)
+ return size;
+ return (size + alignment - alignment_mod);
+}
+
+inline size_t FreeList::GetBufferIndex(size_t size) const
+{
+#ifdef __DEBUG__
+ assert(size > 0 && size < max_num_buffer*alignment);
+#endif
+ return ((size - 1) / alignment);
+}
+
+inline uint64_t FreeList::GetBufferCountByIndex(size_t buf_index) const
+{
+#ifdef __DEBUG__
+ assert(buf_index < max_num_buffer);
+#endif
+ return buffer_free_list[buf_index]->Count();
+}
+
+inline size_t FreeList::GetBufferSizeByIndex(size_t buf_index) const
+{
+#ifdef __DEBUG__
+ assert(buf_index < max_num_buffer);
+#endif
+ return (buf_index + 1) * alignment;
+}
+
+inline int FreeList::AddBufferByIndex(size_t buf_index, size_t offset)
+{
+#ifdef __DEBUG__
+ assert(buf_index < max_num_buffer);
+#endif
+
+ if(buffer_free_list[buf_index]->Count() > (unsigned)max_buffer_per_list)
+ {
+ ReuseBuffer(buf_index, offset);
+ return MBError::SUCCESS;
+ }
+
+ count++;
+ tot_size += (buf_index + 1) * alignment;
+ return buffer_free_list[buf_index]->AddIntToTail(offset);
+}
+
+inline size_t FreeList::RemoveBufferByIndex(size_t buf_index)
+{
+#ifdef __DEBUG__
+ assert(buf_index < max_num_buffer);
+#endif
+ count--;
+ tot_size -= (buf_index + 1) * alignment;
+ return buffer_free_list[buf_index]->RemoveIntFromHead();
+}
+
+inline int FreeList::ReleaseBuffer(size_t offset, size_t size)
+{
+#ifdef __DEBUG__
+ assert(size > 0 && size < max_num_buffer*alignment);
+#endif
+ size_t buf_index = GetBufferIndex(size);
+ return AddBufferByIndex(buf_index, offset);
+}
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __INTEGER_4B_5B_H__
+#define __INTEGER_4B_5B_H__
+
+#include <stdint.h>
+#include <string.h>
+
+namespace mabain {
+
+#define MAX_4B_OFFSET 0xFFFFFFFF
+#define MAX_5B_OFFSET 0xFFFFFFFFFF
+#define MAX_6B_OFFSET 0xFFFFFFFFFFFF
+
+// write and read 5-byte 6-byte unsigned integer
+// Note this is based on engianness.
+
+inline void Write5BInteger(uint8_t *buffer, size_t offset)
+{
+#ifdef __DEBUG__
+ if(offset > MAX_5B_OFFSET)
+ {
+ std::cerr << "OFFSET " << offset << " TOO LARGE FOR 5 BYTES\n";
+ abort();
+ }
+#endif
+
+ uint8_t *src = reinterpret_cast<uint8_t*>(&offset);
+#ifndef __BIG__ENDIAN__
+ memcpy(buffer, src, 5);
+#else
+ memcpy(buffer, src+3, 5);
+#endif
+}
+
+inline size_t Get5BInteger(const uint8_t *buffer)
+{
+ size_t offset = 0;
+ uint8_t *target = reinterpret_cast<uint8_t*>(&offset);
+#ifndef __BIG__ENDIAN__
+ memcpy(target, buffer, 5);
+#else
+ memcpy(target+3, buffer, 5);
+#endif
+ return offset;
+}
+
+inline void Write6BInteger(uint8_t *buffer, size_t offset)
+{
+#ifdef __DEBUG__
+ if(offset > MAX_6B_OFFSET)
+ {
+ std::cerr << "OFFSET " << offset << "TOO LARGE FOR SIX BYTES\n";
+ abort();
+ }
+#endif
+
+ uint8_t *src = reinterpret_cast<uint8_t*>(&offset);
+#ifndef __BIG__ENDIAN__
+ memcpy(buffer, src, 6);
+#else
+ memcpy(buffer, src+2, 6);
+#endif
+}
+
+inline size_t Get6BInteger(const uint8_t *buffer)
+{
+ size_t offset = 0;
+ uint8_t *target = reinterpret_cast<uint8_t*>(&offset);
+#ifndef __BIG__ENDIAN__
+ memcpy(target, buffer, 6);
+#else
+ memcpy(target+2, buffer, 6);
+#endif
+ return offset;
+}
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include "db.h"
+#include "dict.h"
+#include "integer_4b_5b.h"
+#include "mbt_base.h"
+
+namespace mabain {
+
+typedef struct _iterator_node
+{
+ std::string *key;
+ uint8_t *data;
+ int data_len;
+ uint16_t bucket_index;
+} iterator_node;
+
+static void free_iterator_node(void *n)
+{
+ if(n == NULL)
+ return;
+
+ iterator_node *inode = (iterator_node *) n;
+ if(inode->key != NULL)
+ delete inode->key;
+ if(inode->data != NULL)
+ free(inode->data);
+
+ free(inode);
+}
+
+static iterator_node* new_iterator_node(const std::string &key, MBData *mbdata)
+{
+ iterator_node *inode = (iterator_node *) malloc(sizeof(*inode));
+ if(inode == NULL)
+ throw (int) MBError::NO_MEMORY;
+
+ inode->key = new std::string(key);
+ if(mbdata != NULL)
+ {
+ inode->bucket_index = mbdata->bucket_index;
+ mbdata->TransferValueTo(inode->data, inode->data_len);
+ if(inode->data == NULL || inode->data_len <= 0)
+ {
+ free_iterator_node(inode);
+ inode = NULL;
+ }
+ }
+ else
+ {
+ inode->data = NULL;
+ inode->data_len = 0;
+ }
+
+ return inode;
+}
+
+/////////////////////////////////////////////////////////////////////
+// DB iterator
+// Example to use DB iterator
+// for(DB::iterator iter = db.begin(); iter != db.end(); ++iter) {
+// std::cout << iter.key << "\n";
+// }
+/////////////////////////////////////////////////////////////////////
+
+const DB::iterator DB::begin(bool check_async_mode, bool rc_mode) const
+{
+ DB::iterator iter = iterator(*this, DB_ITER_STATE_INIT);
+ if(rc_mode) iter.value.options |= CONSTS::OPTION_RC_MODE;
+ iter.init(check_async_mode);
+
+ return iter;
+}
+
+const DB::iterator DB::end() const
+{
+ return iterator(*this, DB_ITER_STATE_DONE);
+}
+
+void DB::iterator::iter_obj_init()
+{
+ node_stack = NULL;
+ kv_per_node = NULL;
+ lfree = NULL;
+
+ if(!(db_ref.GetDBOptions() & CONSTS::ACCESS_MODE_WRITER))
+ {
+#ifdef __LOCK_FREE__
+ lfree = db_ref.dict->GetLockFreePtr();
+#endif
+ }
+
+ if(state == DB_ITER_STATE_INIT)
+ state = DB_ITER_STATE_MORE;
+}
+
+DB::iterator::iterator(const DB &db, int iter_state)
+ : db_ref(db), state(iter_state)
+{
+ iter_obj_init();
+}
+
+DB::iterator::iterator(const iterator &rhs)
+ : db_ref(rhs.db_ref), state(rhs.state)
+{
+ iter_obj_init();
+}
+
+DB::iterator::~iterator()
+{
+ if(node_stack != NULL)
+ delete node_stack;
+ if(kv_per_node != NULL)
+ delete kv_per_node;
+}
+
+// Initialize the iterator, get the very first key-value pair.
+void DB::iterator::init(bool check_async_mode)
+{
+ // Writer in async mode cannot be used for lookup
+ if(check_async_mode && (db_ref.options & CONSTS::ASYNC_WRITER_MODE))
+ {
+ state = DB_ITER_STATE_DONE;
+ return;
+ }
+
+ node_stack = new MBlsq(free_iterator_node);
+ kv_per_node = new MBlsq(free_iterator_node);
+
+ load_kv_for_node("");
+ if(next() == NULL)
+ state = DB_ITER_STATE_DONE;
+}
+
+// Initialize the iterator, but do not get the first key-value pair.
+// This is used for resource collection.
+int DB::iterator::init_no_next()
+{
+ node_stack = new MBlsq(NULL);
+ kv_per_node = NULL;
+
+ int rval = db_ref.dict->ReadRootNode(node_buff, edge_ptrs, match, value);
+ if(rval != MBError::SUCCESS)
+ state = DB_ITER_STATE_DONE;
+ return rval;
+}
+
+const DB::iterator& DB::iterator::operator++()
+{
+ if(next() == NULL)
+ state = DB_ITER_STATE_DONE;
+
+ return *this;
+}
+
+// This overloaded operator should only be used for iterator state check.
+bool DB::iterator::operator!=(const iterator &rhs)
+{
+ return state != rhs.state;
+}
+
+int DB::iterator::get_node_offset(const std::string &node_key,
+ size_t &parent_edge_off,
+ size_t &node_offset)
+{
+ int rval;
+
+ node_offset = 0;
+ value.options |= CONSTS::OPTION_FIND_AND_STORE_PARENT;
+ while(true)
+ {
+ rval = db_ref.dict->Find((const uint8_t *)node_key.data(),
+ node_key.size(), value);
+ if(rval != MBError::TRY_AGAIN)
+ break;
+ nanosleep((const struct timespec[]){{0, 10L}}, NULL);
+ }
+
+ if(rval == MBError::IN_DICT)
+ {
+ parent_edge_off = edge_ptrs.parent_offset;
+ node_offset = Get6BInteger(value.edge_ptrs.offset_ptr);
+ rval = MBError::SUCCESS;
+ }
+ return rval;
+}
+
+int DB::iterator::load_kvs(const std::string &curr_node_key,
+ MBlsq *child_node_list)
+{
+ int rval;
+ size_t child_node_off;
+ std::string match_str;
+ iterator_node *inode;
+
+ while(true)
+ {
+ if(lfree == NULL)
+ {
+ rval = db_ref.dict->ReadNextEdge(node_buff, edge_ptrs, match, value,
+ match_str, child_node_off);
+ }
+ else
+ {
+#ifdef __LOCK_FREE__
+ LockFreeData snapshot;
+ int lf_ret;
+ size_t edge_off_prev = edge_ptrs.offset;
+ lfree->ReaderLockFreeStart(snapshot);
+#endif
+ rval = db_ref.dict->ReadNextEdge(node_buff, edge_ptrs, match, value,
+ match_str, child_node_off);
+#ifdef __LOCK_FREE__
+ lf_ret = lfree->ReaderLockFreeStop(snapshot, edge_off_prev, value);
+ if(lf_ret == MBError::TRY_AGAIN)
+ return lf_ret;
+#endif
+ }
+
+ if(rval != MBError::SUCCESS)
+ break;
+
+ match_str = curr_node_key + match_str;
+ if(child_node_off > 0)
+ {
+ inode = new_iterator_node(match_str, NULL);
+ if(inode != NULL)
+ {
+ rval = child_node_list->AddToTail(inode);
+ if(rval != MBError::SUCCESS)
+ {
+ free_iterator_node(inode);
+ return rval;
+ }
+ }
+ }
+
+ if(match != MATCH_NONE)
+ {
+ inode = new_iterator_node(match_str, &value);
+ if(inode != NULL)
+ {
+ rval = kv_per_node->AddToTail(inode);
+ if(rval != MBError::SUCCESS)
+ {
+ free_iterator_node(inode);
+ return rval;
+ }
+ }
+ }
+ }
+
+ if(rval == MBError::OUT_OF_BOUND)
+ rval = MBError::SUCCESS;
+ return rval;
+}
+
+int DB::iterator::load_node(const std::string &curr_node_key,
+ size_t &parent_edge_off)
+{
+ int rval;
+
+ if(curr_node_key.size() == 0)
+ {
+ rval = db_ref.dict->ReadRootNode(node_buff, edge_ptrs, match, value);
+ }
+ else
+ {
+ size_t node_offset;
+ rval = get_node_offset(curr_node_key, parent_edge_off, node_offset);
+ if(rval != MBError::SUCCESS)
+ return rval;
+ rval = db_ref.dict->ReadNode(node_offset, node_buff, edge_ptrs, match,
+ value, false);
+ }
+
+ return rval;
+}
+
+int DB::iterator::load_kv_for_node(const std::string &curr_node_key)
+{
+ int rval;
+ MBlsq child_node_list(free_iterator_node);
+ size_t parent_edge_off;
+
+ if(lfree == NULL)
+ {
+ rval = load_node(curr_node_key, parent_edge_off);
+ if(rval == MBError::SUCCESS)
+ rval = load_kvs(curr_node_key, &child_node_list);
+ }
+ else
+ {
+#ifdef __LOCK_FREE__
+ LockFreeData snapshot;
+ int lf_ret;
+#endif
+ while(true)
+ {
+#ifdef __LOCK_FREE__
+ lfree->ReaderLockFreeStart(snapshot);
+#endif
+ rval = load_node(curr_node_key, parent_edge_off);
+ if(rval == MBError::SUCCESS)
+ {
+ rval = load_kvs(curr_node_key, &child_node_list);
+#ifdef __LOCK_FREE__
+ if(rval == MBError::TRY_AGAIN)
+ {
+ kv_per_node->Clear();
+ child_node_list.Clear();
+ continue;
+ }
+#endif
+ }
+#ifdef __LOCK_FREE__
+ lf_ret = lfree->ReaderLockFreeStop(snapshot, parent_edge_off, value);
+ if(lf_ret == MBError::TRY_AGAIN)
+ {
+ kv_per_node->Clear();
+ child_node_list.Clear();
+ continue;
+ }
+#endif
+ break;
+ }
+ }
+
+ if(rval == MBError::SUCCESS)
+ {
+ iterator_node *inode;
+ while((inode = (iterator_node *) child_node_list.RemoveFromHead()))
+ {
+ node_stack->AddToHead(inode);
+ }
+ }
+ else
+ {
+ std::cerr << "failed to run ietrator: " << MBError::get_error_str(rval) << "\n";
+ kv_per_node->Clear();
+ child_node_list.Clear();
+ }
+ return rval;
+}
+
+// Find next iterator match
+DB::iterator* DB::iterator::next()
+{
+ iterator_node *inode;
+
+ while(kv_per_node->Count() == 0)
+ {
+ inode = (iterator_node *) node_stack->RemoveFromHead();
+ if(inode == NULL)
+ return NULL;
+
+ int rval = load_kv_for_node(*inode->key);
+ free_iterator_node(inode);
+ if(rval != MBError::SUCCESS)
+ return NULL;
+ }
+
+ if(kv_per_node->Count() > 0)
+ {
+ inode = (iterator_node *) kv_per_node->RemoveFromHead();
+ match = MATCH_NODE_OR_EDGE;
+ key = *inode->key;
+ value.TransferValueFrom(inode->data, inode->data_len);
+ value.bucket_index = inode->bucket_index;
+ free_iterator_node(inode);
+ return this;
+ }
+
+ return NULL;
+}
+
+// There is no need to perform lock-free check in next_dbt_buffer
+// since it can only be called by writer.
+bool DB::iterator::next_dbt_buffer(struct _DBTraverseNode *dbt_n)
+{
+ int rval;
+ size_t node_off;
+ size_t curr_edge_off;
+ std::string match_str;
+
+ memset(dbt_n, 0, sizeof(*dbt_n));
+ do {
+ curr_edge_off = edge_ptrs.offset;
+ while((rval = db_ref.dict->ReadNextEdge(node_buff, edge_ptrs, match,
+ value, match_str, node_off, false)) == MBError::SUCCESS)
+ {
+ if(edge_ptrs.len_ptr[0] > LOCAL_EDGE_LEN)
+ {
+ dbt_n->edgestr_offset = Get5BInteger(edge_ptrs.ptr);
+ dbt_n->edgestr_size = edge_ptrs.len_ptr[0] - 1;
+ dbt_n->edgestr_link_offset = curr_edge_off;
+ dbt_n->buffer_type |= BUFFER_TYPE_EDGE_STR;
+ }
+
+ if(node_off > 0)
+ {
+ dbt_n->node_offset = node_off;
+ dbt_n->node_link_offset = curr_edge_off + EDGE_NODE_LEADING_POS;
+ dbt_n->buffer_type |= BUFFER_TYPE_NODE;
+ db_ref.dict->ReadNodeHeader(node_off, dbt_n->node_size, match, dbt_n->data_offset,
+ dbt_n->data_link_offset);
+ if(match == MATCH_NODE)
+ dbt_n->buffer_type |= BUFFER_TYPE_DATA;
+ }
+ else if(match == MATCH_EDGE)
+ {
+ dbt_n->data_offset = Get6BInteger(edge_ptrs.offset_ptr);
+ dbt_n->data_link_offset = curr_edge_off + EDGE_NODE_LEADING_POS;
+ dbt_n->buffer_type |= BUFFER_TYPE_DATA;
+ }
+
+ if(dbt_n->buffer_type != BUFFER_TYPE_NONE)
+ {
+ dbt_n->edge_offset = curr_edge_off;
+ return true;
+ }
+
+ curr_edge_off = edge_ptrs.offset;
+ }
+
+ if(rval == MBError::OUT_OF_BOUND)
+ {
+ node_off = (size_t ) node_stack->RemoveIntFromHead();
+ if(node_off == 0)
+ break;
+ rval = db_ref.dict->ReadNode(node_off, node_buff, edge_ptrs, match,
+ value, false);
+ if(rval != MBError::SUCCESS)
+ throw rval;
+ }
+ else
+ {
+ throw rval;
+ }
+ } while(true);
+
+ return false;
+}
+
+// Add an node offset to the iterator queue
+void DB::iterator::add_node_offset(size_t node_offset)
+{
+ int rval = node_stack->AddIntToHead(node_offset);
+ if(rval != MBError::SUCCESS)
+ throw rval;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include "lock.h"
+
+namespace mabain {
+
+// rwlock within a process
+pthread_rwlock_t MBLock::mb_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
+
+void MBLock::Init(pthread_rwlock_t *rw_lock)
+{
+#ifdef __SHM_LOCK__
+ // using process lock for initializing shared memory lock
+ pthread_rwlock_wrlock(&mb_rw_lock);
+ mb_rw_lock_ptr = rw_lock;
+ pthread_rwlock_unlock(&mb_rw_lock);
+#endif
+}
+
+MBLock::MBLock()
+{
+#ifndef __SHM_LOCK__
+ mb_rw_lock_ptr = &MBLock::mb_rw_lock;
+#endif
+}
+
+MBLock::~MBLock()
+{
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __LOCK_H__
+#define __LOCK_H__
+
+#include <pthread.h>
+
+namespace mabain {
+
+// multiple-thread/process reader/writer lock
+
+class MBLock
+{
+public:
+ MBLock();
+ ~MBLock();
+
+ // Writer lock
+ inline int WrLock();
+ // Reader lock
+ inline int RdLock();
+ // Writer/reader unlock
+ inline int UnLock();
+
+ inline int TryWrLock();
+
+ void Init(pthread_rwlock_t *rw_lock);
+
+private:
+ // a global lock variable
+ static pthread_rwlock_t mb_rw_lock;
+
+ pthread_rwlock_t *mb_rw_lock_ptr;
+};
+
+inline int MBLock::WrLock()
+{
+ if(mb_rw_lock_ptr == NULL)
+ return -1;
+ return pthread_rwlock_wrlock(mb_rw_lock_ptr);
+}
+
+inline int MBLock::RdLock()
+{
+ if(mb_rw_lock_ptr == NULL)
+ return -1;
+ return pthread_rwlock_rdlock(mb_rw_lock_ptr);
+}
+
+inline int MBLock::UnLock()
+{
+ if(mb_rw_lock_ptr == NULL)
+ return -1;
+ return pthread_rwlock_unlock(mb_rw_lock_ptr);
+}
+
+inline int MBLock::TryWrLock()
+{
+ if(mb_rw_lock_ptr == NULL)
+ return -1;
+ return pthread_rwlock_trywrlock(mb_rw_lock_ptr);
+}
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <iostream>
+
+#include "lock_free.h"
+#include "mabain_consts.h"
+#include "error.h"
+#include "integer_4b_5b.h"
+#include "drm_base.h"
+#include "dict_mem.h"
+
+namespace mabain {
+
+LockFree::LockFree()
+{
+ shm_data_ptr = NULL;
+}
+
+LockFree::~LockFree()
+{
+}
+
+void LockFree::LockFreeInit(LockFreeShmData *lock_free_ptr, IndexHeader *hdr, int mode)
+{
+ shm_data_ptr = lock_free_ptr;
+ header = hdr;
+ if(mode & CONSTS::ACCESS_MODE_WRITER)
+ {
+ // Clear the lock free data
+ shm_data_ptr->counter.store(0, MEMORY_ORDER_WRITER);
+ shm_data_ptr->offset.store(MAX_6B_OFFSET, MEMORY_ORDER_WRITER);
+ }
+}
+
+//////////////////////////////////////////////////
+// DO NOT CHANGE THE STORE ORDER IN THIS FUNCTION.
+//////////////////////////////////////////////////
+void LockFree::WriterLockFreeStop()
+{
+ int index = shm_data_ptr->counter % MAX_OFFSET_CACHE;
+ shm_data_ptr->offset_cache[index].store(shm_data_ptr->offset, MEMORY_ORDER_WRITER);
+
+ shm_data_ptr->counter.fetch_add(1, MEMORY_ORDER_WRITER);
+ shm_data_ptr->offset.store(MAX_6B_OFFSET, MEMORY_ORDER_WRITER);
+}
+
+//////////////////////////////////////////////////
+// DO NOT CHANGE THE LOAD ORDER IN THIS FUNCTION.
+//////////////////////////////////////////////////
+int LockFree::ReaderLockFreeStop(const LockFreeData &snapshot, size_t reader_offset,
+ MBData &mbdata)
+{
+ LockFreeData curr;
+ curr.offset = shm_data_ptr->offset.load(MEMORY_ORDER_READER);
+ curr.counter = shm_data_ptr->counter.load(MEMORY_ORDER_READER);
+
+ if(curr.offset == reader_offset)
+ {
+ if(mbdata.options & CONSTS::OPTION_READ_SAVED_EDGE)
+ {
+ if(reader_offset == mbdata.edge_ptrs.offset)
+ {
+ mbdata.options &= ~CONSTS::OPTION_READ_SAVED_EDGE;
+ return MBError::SUCCESS;
+ }
+ }
+ else
+ {
+ mbdata.options |= CONSTS::OPTION_READ_SAVED_EDGE;
+ }
+
+ // Save the edge
+ switch(header->excep_updating_status)
+ {
+ case EXCEP_STATUS_ADD_EDGE:
+ case EXCEP_STATUS_ADD_DATA_OFF:
+ case EXCEP_STATUS_ADD_NODE:
+ case EXCEP_STATUS_REMOVE_EDGE:
+ case EXCEP_STATUS_RC_NODE:
+ case EXCEP_STATUS_RC_EDGE_STR:
+ case EXCEP_STATUS_RC_DATA:
+ memcpy(mbdata.edge_ptrs.edge_buff, header->excep_buff, EDGE_SIZE);
+ mbdata.edge_ptrs.offset = shm_data_ptr->offset.load(MEMORY_ORDER_READER);
+ break;
+ case EXCEP_STATUS_CLEAR_EDGE:
+ default:
+ memset(mbdata.edge_ptrs.edge_buff, 0, EDGE_SIZE);
+ mbdata.edge_ptrs.offset = shm_data_ptr->offset.load(MEMORY_ORDER_READER);
+ break;
+ }
+ if(mbdata.edge_ptrs.offset == reader_offset)
+ {
+ InitTempEdgePtrs(mbdata.edge_ptrs);
+ }
+ else
+ {
+ mbdata.options &= ~CONSTS::OPTION_READ_SAVED_EDGE;
+ mbdata.edge_ptrs.offset = MAX_6B_OFFSET;
+ }
+ return MBError::TRY_AGAIN;
+ }
+
+ if(mbdata.options & CONSTS::OPTION_READ_SAVED_EDGE)
+ mbdata.options &= ~CONSTS::OPTION_READ_SAVED_EDGE;
+
+ // Note it is expected that count_diff can overflow.
+ uint32_t count_diff = curr.counter - snapshot.counter;
+ if(count_diff == 0)
+ return MBError::SUCCESS; // Writer was doing nothing. Reader can proceed.
+ if(count_diff >= MAX_OFFSET_CACHE)
+ return MBError::TRY_AGAIN; // Cache is overwritten. Have to retry.
+
+ for(unsigned i = 0; i < count_diff; i++)
+ {
+ int index = (snapshot.counter + i) % MAX_OFFSET_CACHE;
+ if(reader_offset == shm_data_ptr->offset_cache[index].load(MEMORY_ORDER_READER))
+ return MBError::TRY_AGAIN;
+ }
+
+ // Need to recheck the counter difference
+ count_diff = shm_data_ptr->counter.load(MEMORY_ORDER_READER) - snapshot.counter;
+ if(count_diff >= MAX_OFFSET_CACHE)
+ return MBError::TRY_AGAIN;
+
+ // Writer was modifying different edges. It is safe to for the reader to proceed.
+ return MBError::SUCCESS;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __LOCK_FREE_H__
+#define __LOCK_FREE_H__
+
+#include <stdint.h>
+#include <string.h>
+#include <atomic>
+
+#include "mb_data.h"
+
+namespace mabain {
+
+// C++11 std::atomic shared memory variable for lock-free
+// reader/writer concurrency.
+
+#define MAX_OFFSET_CACHE 4
+#define MEMORY_ORDER_WRITER std::memory_order_release
+#define MEMORY_ORDER_READER std::memory_order_consume
+
+struct _IndexHeader;
+typedef struct _IndexHeader IndexHeader;
+
+typedef struct _LockFreeData
+{
+ uint32_t counter;
+ size_t offset;
+} LockFreeData;
+
+typedef struct _LockFreeShmData
+{
+ std::atomic<uint32_t> counter;
+ std::atomic<size_t> offset;
+ std::atomic<size_t> offset_cache[MAX_OFFSET_CACHE];
+} LockFreeShmData;
+
+class LockFree
+{
+public:
+ LockFree();
+ ~LockFree();
+
+ void LockFreeInit(LockFreeShmData *lock_free_ptr, IndexHeader *hdr, int mode = 0);
+ inline void WriterLockFreeStart(size_t offset);
+ void WriterLockFreeStop();
+ inline void ReaderLockFreeStart(LockFreeData &snapshot);
+ // If there was race condition, this function returns MBError::TRY_AGAIN.
+ int ReaderLockFreeStop(const LockFreeData &snapshot, size_t reader_offset,
+ MBData &mbdata);
+
+private:
+ LockFreeShmData *shm_data_ptr;
+ const IndexHeader *header;
+};
+
+inline void LockFree::WriterLockFreeStart(size_t offset)
+{
+ shm_data_ptr->offset.store(offset, MEMORY_ORDER_WRITER);
+}
+
+inline void LockFree::ReaderLockFreeStart(LockFreeData &snapshot)
+{
+ snapshot.counter = shm_data_ptr->counter.load(MEMORY_ORDER_READER);
+}
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <stdarg.h>
+#include <iostream>
+#include <unistd.h>
+
+#include "logger.h"
+#include "error.h"
+
+#define MAX_NUM_LOG 10
+
+namespace mabain {
+
+std::string Logger::log_file = "";
+std::ofstream* Logger::log_stream = NULL;
+int Logger::log_level_ = LOG_LEVEL_WARN;
+int Logger::roll_size = 50*1024*1024;
+const char* Logger::LOG_LEVEL[4] =
+{
+ " ERROR: ",
+ " WARN: ",
+ " INFO: ",
+ " DEBUG: "
+};
+
+Logger::Logger()
+{
+}
+
+Logger::~Logger()
+{
+}
+
+void Logger::Close()
+{
+ if(log_stream != NULL)
+ {
+ if(log_stream->is_open())
+ log_stream->close();
+
+ delete log_stream;
+ log_stream = NULL;
+ }
+}
+
+// Should only be called by writer for now.
+// Readers will send logs to stdout or stderr.
+void Logger::InitLogFile(const std::string &logfile)
+{
+ if(!logfile.empty())
+ {
+ log_file = logfile;
+ log_level_ = LOG_LEVEL_INFO;
+ log_stream = new std::ofstream();
+ log_stream->open(log_file.c_str(), std::ios::out | std::ios::app);
+ }
+}
+
+void Logger::FillDateTime(char *buffer, int bufsize)
+{
+ time_t rawtime;
+ struct tm timeinfo;
+ time (&rawtime);
+ if(localtime_r(&rawtime, &timeinfo))
+ strftime(buffer, bufsize, "%Y-%m-%d.%X", &timeinfo);
+ else
+ snprintf(buffer, bufsize, "Time unknown");
+}
+
+void Logger::Rotate()
+{
+ if(log_stream == NULL)
+ return;
+ if(log_stream->is_open())
+ log_stream->close();
+
+ std::string filepath_old;
+ std::string filepath_new;
+ for(int i = MAX_NUM_LOG-2; i > 0; i--)
+ {
+ filepath_old = log_file + "." + std::to_string(i);
+ if(access(filepath_old.c_str(), R_OK) == 0)
+ {
+ filepath_new = log_file + "." + std::to_string(i+1);
+ if(rename(filepath_old.c_str(), filepath_new.c_str()))
+ std::cerr << "failed to move log file\n";
+ }
+ }
+ filepath_new = log_file + ".1";
+ if(rename(log_file.c_str(), filepath_new.c_str()))
+ std::cerr << "failed to move log file\n";
+
+ log_stream->open(log_file.c_str(), std::ios::out | std::ios::app);
+}
+
+void Logger::Log(int level, const std::string &message)
+{
+ if(level > log_level_)
+ return;
+
+ if(log_stream != NULL)
+ {
+ char buffer[80];
+ FillDateTime(buffer, sizeof(buffer));
+ *log_stream << buffer << LOG_LEVEL[level] << message << std::endl;
+ if(log_stream->tellp() > roll_size)
+ Logger::Rotate();
+ }
+ else if(level < LOG_LEVEL_INFO)
+ std::cerr << message << std::endl;
+ else
+ std::cout << message << std::endl;
+}
+
+void Logger::Log(int level, const char *format, ... )
+{
+ if(level > log_level_)
+ return;
+
+ char message[256];
+ va_list args;
+ va_start(args, format);
+ vsprintf(message, format, args);
+ if(log_stream != NULL)
+ {
+ char buffer[64];
+ FillDateTime(buffer, sizeof(buffer));
+ *log_stream << buffer << LOG_LEVEL[level] << message << std::endl;
+ if(log_stream->tellp() > roll_size)
+ Logger::Rotate();
+ }
+ else if(level < LOG_LEVEL_INFO)
+ std::cerr << message << std::endl;
+ else
+ std::cout << message << std::endl;
+ va_end(args);
+}
+
+int Logger::SetLogLevel(int level)
+{
+ if(level < 0 || level > LOG_LEVEL_DEBUG)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "invaid logging level %d", level);
+ return MBError::INVALID_ARG;
+ }
+
+ log_level_ = level;
+
+ return MBError::SUCCESS;
+}
+
+std::ofstream* Logger::GetLogStream()
+{
+ return log_stream;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __MB_LOGGER_H__
+#define __MB_LOGGER_H__
+
+#include <fstream>
+#include <string>
+
+namespace mabain {
+
+#define LOG_LEVEL_ERROR 0
+#define LOG_LEVEL_WARN 1
+#define LOG_LEVEL_INFO 2
+#define LOG_LEVEL_DEBUG 3
+
+class Logger
+{
+public:
+ ~Logger();
+
+ // Logs a message
+ static void Log(int level, const std::string &message);
+ // Variable length log
+ static void Log(int level, const char *formt, ... );
+ static void FillDateTime(char *buffer, int bufsize);
+ static void InitLogFile(const std::string &logfile);
+ // Set log level. Default is logging everything except for debug.
+ static int SetLogLevel(int level);
+ static void Close();
+ static std::ofstream* GetLogStream();
+
+private:
+ // This is a singleton class. Make sure nobody can create an instance.
+ Logger();
+ static void Rotate();
+
+ static std::string log_file;
+ static std::ofstream *log_stream;
+ static const char* LOG_LEVEL[4];
+ static int log_level_;
+ static int roll_size;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include "mabain_consts.h"
+
+namespace mabain {
+
+const int CONSTS::ACCESS_MODE_READER = 0x0;
+const int CONSTS::ACCESS_MODE_WRITER = 0x1;
+const int CONSTS::ASYNC_WRITER_MODE = 0x2;
+const int CONSTS::SYNC_ON_WRITE = 0x4;
+const int CONSTS::USE_SLIDING_WINDOW = 0x8;
+const int CONSTS::MEMORY_ONLY_MODE = 0x10;
+
+const int CONSTS::OPTION_ALL_PREFIX = 0x1;
+const int CONSTS::OPTION_FIND_AND_STORE_PARENT = 0x2;
+const int CONSTS::OPTION_RC_MODE = 0x4;
+const int CONSTS::OPTION_READ_SAVED_EDGE = 0x8;
+
+const int CONSTS::MAX_KEY_LENGHTH = 256;
+const int CONSTS::MAX_DATA_SIZE = 0x7FFF;
+
+int CONSTS::WriterOptions()
+{
+ int options = ACCESS_MODE_WRITER;
+ return options;
+}
+
+int CONSTS::ReaderOptions()
+{
+ int options = ACCESS_MODE_READER;
+ return options;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __MABAIN_H__
+#define __MABAIN_H__
+
+namespace mabain {
+
+class CONSTS
+{
+public:
+ static const int ACCESS_MODE_READER;
+ static const int ACCESS_MODE_WRITER;
+ static const int ASYNC_WRITER_MODE;
+ static const int SYNC_ON_WRITE;
+ static const int USE_SLIDING_WINDOW;
+ static const int MEMORY_ONLY_MODE;
+ static const int OPTION_ALL_PREFIX;
+
+ static const int OPTION_FIND_AND_STORE_PARENT;
+ static const int OPTION_RC_MODE;
+ static const int OPTION_READ_SAVED_EDGE; // Used internally only
+ // not init shared memory ptr, not update db counter
+ static const int MAX_KEY_LENGHTH;
+ static const int MAX_DATA_SIZE;
+
+ static int WriterOptions();
+ static int ReaderOptions();
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Shridhar Bhalerao <shbhaler@cisco.com>
+
+#include <iostream>
+#include <fstream>
+#include <limits.h>
+#include <sstream>
+
+#include "mb_backup.h"
+#include "mb_data.h"
+#include "integer_4b_5b.h"
+
+namespace mabain {
+
+DBBackup::DBBackup(const DB &db) : db_ref(db)
+{
+ if(!(db.GetDBOptions() & CONSTS::ACCESS_MODE_WRITER))
+ throw (int) MBError::NOT_ALLOWED;
+
+ Dict *dict = db_ref.GetDictPtr();
+ if(dict == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+ header = dict->GetHeaderPtr();
+ if(header == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+}
+
+DBBackup::~DBBackup()
+{
+}
+
+
+void DBBackup::copy_file (const std::string &src_path, const std::string &dest_path,
+ char *buffer, int buffer_size)
+{
+ FILE *fp_read, *fp_write;
+
+ if ((fp_read = fopen (src_path.c_str(),"rb")) == NULL)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "Backup failed: Could not open file %s", src_path.c_str());
+ throw MBError::OPEN_FAILURE;
+ }
+ if ((fp_write = fopen(dest_path.c_str(),"wb")) == NULL)
+ {
+ fclose(fp_read);
+ Logger::Log(LOG_LEVEL_ERROR, "Backup failed: Could not open file %s", dest_path.c_str());
+ throw MBError::OPEN_FAILURE;
+ }
+
+ int rval = MBError::SUCCESS;
+ int read_count, write_count;
+
+ while(true)
+ {
+ read_count = fread(buffer, buffer_size, 1, fp_read);
+
+ if(feof(fp_read) && read_count == 0)
+ break;
+ if (read_count != 1)
+ {
+ rval = MBError::READ_ERROR;
+ break;
+ }
+
+ write_count = fwrite(buffer,buffer_size, 1, fp_write);
+ if(write_count != 1)
+ {
+ rval = MBError::WRITE_ERROR;
+ break;
+ }
+ }
+ fclose(fp_read);
+ fclose(fp_write);
+
+ if(rval != MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "Backup failed %s", MBError::get_error_str(rval));
+ throw rval;
+ }
+}
+
+int DBBackup::Backup(const char * bk_dir)
+{
+ if(bk_dir == NULL)
+ throw (int) MBError::INVALID_ARG;
+
+ std::string bk_header_path = std::string(bk_dir) + "/_mabain_h";
+ if(access(bk_header_path.c_str(), R_OK) == 0)
+ throw (int) MBError::OPEN_FAILURE;
+
+ if(!db_ref.is_open())
+ throw (int) db_ref.Status();
+
+ if(header->m_data_offset > MAX_6B_OFFSET || header->m_index_offset > MAX_6B_OFFSET)
+ throw (int) MBError::INVALID_SIZE;
+ if(header->data_block_size == 0 || header->data_block_size % BLOCK_SIZE_ALIGN != 0)
+ throw (int) MBError::INVALID_SIZE;
+ if(header->index_block_size == 0 || header->index_block_size % BLOCK_SIZE_ALIGN != 0)
+ throw (int) MBError::INVALID_SIZE;
+
+ db_ref.Flush();
+
+ int num_data_files, num_index_files;
+ // get the number of files from the header: max_bytes/number_of_bytes_per_file
+ num_data_files = (header->m_data_offset/header->data_block_size) + 1;
+ num_index_files = (header->m_index_offset/header->index_block_size) + 1;
+
+ char *buffer = (char *) malloc(BLOCK_SIZE_ALIGN);
+ if(buffer == NULL)
+ throw (int) MBError::NO_MEMORY;
+
+ // loop through all data files
+ const std::string &orig_dir = db_ref.GetDBDir();
+ std::string read_file_path_base = orig_dir + "/_mabain_d";
+ std::string write_file_path_base = std::string(bk_dir) + "/_mabain_d";
+ std::string read_file_path;
+ std::string write_file_path;
+
+ for (int i = 0; i < num_data_files; i++)
+ {
+ read_file_path = read_file_path_base + std::to_string(i);
+ write_file_path = write_file_path_base + std::to_string(i);
+ copy_file(read_file_path, write_file_path, buffer, BLOCK_SIZE_ALIGN);
+ }
+
+ read_file_path_base = orig_dir + "/_mabain_i";
+ write_file_path_base = std::string(bk_dir) + "/_mabain_i";
+ for (int i = 0; i < num_index_files; i++)
+ {
+ read_file_path = read_file_path_base + std::to_string(i);
+ write_file_path = write_file_path_base + std::to_string(i);
+ copy_file(read_file_path, write_file_path, buffer, BLOCK_SIZE_ALIGN);
+ }
+
+ read_file_path = orig_dir + "/_mabain_h";
+ write_file_path = std::string(bk_dir) + "/_mabain_h";
+ //header is size of page_size
+ copy_file(read_file_path, write_file_path, buffer, RollableFile::page_size);
+
+ free(buffer);
+
+ //reset number readers/writers in backed up DB.
+ int rval;
+ DB db = DB(bk_dir, CONSTS::ACCESS_MODE_READER, 0, 0);
+ rval = db.UpdateNumHandlers(CONSTS::ACCESS_MODE_WRITER, -1);
+ if(rval != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_WARN,"failed to reset number of writer for DB %s", bk_dir);
+
+ rval = db.UpdateNumHandlers(CONSTS::ACCESS_MODE_READER, INT_MIN);
+ if(rval != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_WARN,"failed to reset number of writer for DB %s", bk_dir);
+ db.Close();
+ return MBError::SUCCESS;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Shridhar Bhalerao <shbhaler@cisco.com>
+
+#ifndef __DBBackup_H__
+#define __DBBackup_H__
+
+#include <string>
+
+#include "db.h"
+#include "dict.h"
+
+namespace mabain {
+
+class DBBackup
+{
+public:
+ DBBackup(const DB &db);
+ ~DBBackup();
+ int Backup(const char* bkup_dir);
+
+private:
+ static void copy_file (const std::string &src_path,
+ const std::string &dest_path, char *buffer, int buffer_size);
+
+ const DB &db_ref;
+ const IndexHeader *header;
+};
+
+}
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string.h>
+
+#include "mb_data.h"
+#include "error.h"
+
+namespace mabain {
+
+MBData::MBData()
+{
+ data_len = 0;
+ buff_len = 0;
+ buff = NULL;
+
+ match_len = 0;
+ next = false;
+ options = 0;
+ free_buffer = false;
+}
+
+MBData::MBData(int size, int match_options)
+{
+ buff_len = size;
+ buff = NULL;
+ if(buff_len > 0)
+ buff = static_cast<uint8_t*>(malloc(buff_len + 1));
+
+ if(buff != NULL)
+ {
+ free_buffer = true;
+ }
+ else
+ {
+ buff_len = 0;
+ free_buffer = false;
+ }
+
+ data_len = 0;
+ match_len = 0;
+ next = false;
+ options = match_options;
+}
+
+// Caller must free data.
+int MBData::TransferValueTo(uint8_t* &data, int &dlen)
+{
+ if(buff == NULL || data_len <= 0)
+ {
+ dlen = 0;
+ data = NULL;
+ return MBError::INVALID_ARG;
+ }
+
+ if(free_buffer)
+ {
+ data = buff;
+ buff = NULL;
+ dlen = data_len;
+ free_buffer = false;
+ buff_len = 0;
+ }
+ else
+ {
+ data = (uint8_t *) malloc(data_len + 1);
+ if(data == NULL)
+ return MBError::NO_MEMORY;
+ memcpy(data, buff, data_len);
+ dlen = data_len;
+ }
+
+ return MBError::SUCCESS;
+}
+
+// Data must be allocated using malloc or calloc.
+int MBData::TransferValueFrom(uint8_t* &data, int dlen)
+{
+ if(data == NULL)
+ return MBError::INVALID_ARG;
+
+ if(free_buffer && buff != NULL)
+ free(buff);
+ buff = data;
+ buff_len = dlen;
+ data_len = dlen;
+ free_buffer = true;
+
+ data = NULL;
+ return MBError::SUCCESS;
+}
+
+MBData::~MBData()
+{
+ if(free_buffer)
+ free(buff);
+}
+
+// This function is for prefix match only.
+void MBData::Clear()
+{
+ match_len = 0;
+ data_len = 0;
+ next = false;
+}
+
+int MBData::Resize(int size)
+{
+ if(size > buff_len)
+ {
+ buff_len = size;
+ if(free_buffer)
+ {
+ free(buff);
+ }
+ else
+ {
+ free_buffer = true;
+ }
+
+ buff = static_cast<uint8_t*>(malloc(buff_len + 1));
+ if(buff == NULL)
+ {
+ buff_len = 0;
+ free_buffer = false;
+ return MBError::NO_MEMORY;
+ }
+
+ }
+ return MBError::SUCCESS;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __MBDATA_H__
+#define __MBDATA_H__
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "mabain_consts.h"
+
+#define NUM_ALPHABET 256
+#define NODE_EDGE_KEY_FIRST 8
+#define DB_ITER_STATE_INIT 0x00
+#define DB_ITER_STATE_MORE 0x01
+#define DB_ITER_STATE_DONE 0x02
+#define DATA_BLOCK_SIZE_DEFAULT 64LLU*1024*1024 // 64M
+#define INDEX_BLOCK_SIZE_DEFAULT 64LLU*1024*1024 // 64M
+#define BLOCK_SIZE_ALIGN 4*1024*1024 // 4K
+#define BUFFER_TYPE_NONE 0
+#define BUFFER_TYPE_EDGE_STR 0x01
+#define BUFFER_TYPE_NODE 0x02
+#define BUFFER_TYPE_DATA 0x04
+#define MATCH_NONE 0
+#define MATCH_EDGE 1
+#define MATCH_NODE 2
+#define MATCH_NODE_OR_EDGE 3
+
+namespace mabain {
+
+typedef struct _EdgePtrs
+{
+ size_t offset;
+ uint8_t *ptr;
+
+ uint8_t *len_ptr;
+ uint8_t *flag_ptr;
+ uint8_t *offset_ptr;
+
+ // temp buffer for edge, must be bigger than EDGE_SIZE.
+ uint8_t edge_buff[16];
+ // temp usage for calling UpdateNode or entry removing
+ int curr_nt;
+ // temp usage for entry removing
+ size_t curr_node_offset;
+ // temp usage for entry removing
+ int curr_edge_index;
+ // temp usage for entry removing and rc
+ size_t parent_offset;
+} EdgePtrs;
+
+// Data class for find and remove
+// All memeber variable in this class should be kept public so that it can
+// be easily accessed by the caller to get the data/value buffer and buffer len.
+class MBData
+{
+public:
+ MBData();
+ MBData(int size, int options);
+ ~MBData();
+ // Clear the data for repeated usage
+ void Clear();
+ int Resize(int size);
+ int TransferValueTo(uint8_t* &data, int &dlen);
+ int TransferValueFrom(uint8_t* &data, int dlen);
+
+ // data length
+ int data_len;
+ // data buffer
+ uint8_t *buff;
+ // buffer length
+ int buff_len;
+
+ // data offset
+ size_t data_offset;
+ uint16_t bucket_index;
+
+ // Search options
+ int options;
+
+ // temp data for multiple common prefix search
+ // If true, indicate the search should be continued for common prefix search
+ bool next;
+ // match length so far; only populated when match is found.
+ int match_len;
+ struct _EdgePtrs edge_ptrs;
+ // temp buffer to hold the node
+ uint8_t node_buff[NUM_ALPHABET+NODE_EDGE_KEY_FIRST];
+
+private:
+ bool free_buffer;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <sys/time.h>
+
+#include "mb_rc.h"
+#include "dict.h"
+#include "dict_mem.h"
+#include "integer_4b_5b.h"
+
+#define MAX_PRUNE_COUNT 3 // maximum lru eviction attempts
+#define NUM_ASYNC_TASK 10 // number of other tasks to be checked during eviction
+#define PRUNE_TASK_CHECK 100 // every Xth eviction check Y number of other tasks
+#define RC_TASK_CHECK 100 // every Xth async task try to reclaim resources
+#define MIN_RC_OFFSET_GAP 1ULL*1024*1024 // 1M
+
+
+namespace mabain {
+
+ResourceCollection::ResourceCollection(const DB &db, int rct)
+ : DBTraverseBase(db), rc_type(rct)
+{
+ async_writer_ptr = NULL;
+}
+
+ResourceCollection::~ResourceCollection()
+{
+}
+
+#define CIRCULAR_INDEX_DIFF(x, y) ((x)>(y) ? ((x)-(y)) : (0xFFFF-(y)+(x)))
+#define CIRCULAR_PRUNE_DIFF(x, y) ((x)>=(y) ? ((x)-(y)) : (0xFFFF-(y)+(x)))
+int ResourceCollection::LRUEviction()
+{
+ int64_t pruned = 0;
+ int64_t count = 0;
+ int rval = MBError::SUCCESS;
+
+ Logger::Log(LOG_LEVEL_INFO, "running LRU eviction for bucket %u", header->eviction_bucket_index);
+
+ uint16_t index_diff = CIRCULAR_INDEX_DIFF(header->eviction_bucket_index,
+ (header->num_update/header->entry_per_bucket) % 0xFFFF);
+ uint16_t prune_diff = 0xFFFF - index_diff;
+ if(index_diff < 655)
+ prune_diff = uint16_t(prune_diff * 0.35); // prune 35%
+ else if(index_diff < 3276)
+ prune_diff = uint16_t(prune_diff * 0.30); // prune 30%
+ else if(index_diff < 6554)
+ prune_diff = uint16_t(prune_diff * 0.25); // prune 25%
+ else if(index_diff < 9830)
+ prune_diff = uint16_t(prune_diff * 0.20); // prune 20%
+ else
+ prune_diff = uint16_t(prune_diff * 0.15); // prune 15%
+ if(prune_diff == 0)
+ prune_diff = 1;
+
+ for(DB::iterator iter = db_ref.begin(false); iter != db_ref.end(); ++iter)
+ {
+ if(CIRCULAR_PRUNE_DIFF(iter.value.bucket_index, header->eviction_bucket_index) < prune_diff)
+ {
+ rval = dict->Remove((const uint8_t *)iter.key.data(), iter.key.size());
+ if(rval != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_DEBUG, "failed to run eviction %s", MBError::get_error_str(rval));
+ else
+ pruned++;
+ }
+
+ if(async_writer_ptr != NULL)
+ {
+ if(count++ > PRUNE_TASK_CHECK)
+ {
+ count = 0;
+ rval = async_writer_ptr->ProcessTask(NUM_ASYNC_TASK, false);
+ if(rval == MBError::RC_SKIPPED)
+ break;
+ }
+ }
+ }
+
+ if(rval != MBError::RC_SKIPPED)
+ {
+ // It is expected that eviction_bucket_index can overflow since we are only
+ // interested in circular difference.
+ header->eviction_bucket_index += prune_diff;
+ // If not enough pruned, need to retry.
+ if(pruned < int64_t(prune_diff * header->entry_per_bucket * 0.75))
+ rval = MBError::TRY_AGAIN;
+ Logger::Log(LOG_LEVEL_INFO, "LRU eviction done %d pruned, current bucket index %u",
+ pruned, header->eviction_bucket_index);
+ }
+ else
+ {
+ Logger::Log(LOG_LEVEL_INFO, "LRU eviction skipped %d pruned", pruned);
+ }
+
+ return rval;
+}
+
+void ResourceCollection::ReclaimResource(int64_t min_index_size,
+ int64_t min_data_size,
+ int64_t max_dbsz,
+ int64_t max_dbcnt,
+ AsyncWriter *awr)
+{
+ if(!db_ref.is_open())
+ throw db_ref.Status();
+
+ async_writer_ptr = awr;
+ timeval start, stop;
+ uint64_t timediff;
+
+ // Check LRU eviction first
+ if(header->m_data_offset + header->m_index_offset > (size_t) max_dbsz ||
+ header->count > max_dbcnt)
+ {
+ int cnt = 0;
+ gettimeofday(&start,NULL);
+ while(cnt < MAX_PRUNE_COUNT) {
+ if(LRUEviction() != MBError::TRY_AGAIN)
+ break;
+ cnt++;
+ }
+ gettimeofday(&stop,NULL);
+ timediff = (stop.tv_sec - start.tv_sec)*1000000 + (stop.tv_usec - start.tv_usec);
+ if(timediff > 1000000)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "LRU eviction finished in %lf seconds",
+ timediff/1000000.);
+ }
+ else
+ {
+ Logger::Log(LOG_LEVEL_INFO, "LRU eviction finished in %lf milliseconds",
+ timediff/1000.);
+ }
+ }
+
+ if(min_index_size > 0 || min_data_size > 0)
+ {
+ Prepare(min_index_size, min_data_size);
+ Logger::Log(LOG_LEVEL_INFO, "defragmentation started for [index - %s] [data - %s]",
+ rc_type & RESOURCE_COLLECTION_TYPE_INDEX ? "yes":"no",
+ rc_type & RESOURCE_COLLECTION_TYPE_DATA ? " yes":"no");
+ gettimeofday(&start, NULL);
+
+ ReorderBuffers();
+ CollectBuffers();
+ Finish();
+
+ gettimeofday(&stop, NULL);
+ async_writer_ptr = NULL;
+ timediff = (stop.tv_sec - start.tv_sec)*1000000 + (stop.tv_usec - start.tv_usec);
+ if(timediff > 1000000)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "defragmentation finished in %lf seconds",
+ timediff/1000000.);
+ }
+ else
+ {
+ Logger::Log(LOG_LEVEL_INFO, "defragmentation finished in %lf milliseconds",
+ timediff/1000.);
+ }
+ }
+}
+
+/////////////////////////////////////////////////////////
+////////////////// Private Methods //////////////////////
+/////////////////////////////////////////////////////////
+
+void ResourceCollection::Prepare(int64_t min_index_size, int64_t min_data_size)
+{
+ // make sure there is enough grabaged index buffers before initiating collection
+ if(min_index_size == 0 || header->pending_index_buff_size < min_index_size)
+ {
+ rc_type &= ~RESOURCE_COLLECTION_TYPE_INDEX;
+ }
+
+ // make sure there is enough grabaged data buffers before initiating collection
+ if(min_data_size == 0 || header->pending_data_buff_size < min_data_size)
+ {
+ rc_type &= ~RESOURCE_COLLECTION_TYPE_DATA;
+ }
+
+ // minimum defragmentation throshold is not reached, skip grabage collection
+ if(rc_type == 0)
+ {
+ Logger::Log(LOG_LEVEL_DEBUG, "pending_index_buff_size (%llu) min_index_size (%llu)",
+ header->pending_index_buff_size, min_index_size);
+
+ Logger::Log(LOG_LEVEL_DEBUG, "pending_data_buff_size (%llu) min_data_size (%llu)",
+ header->pending_data_buff_size, min_data_size);
+
+ Logger::Log(LOG_LEVEL_DEBUG, "garbage collection skipped since pending"
+ "sizes smaller than required");
+ throw (int) MBError::RC_SKIPPED;
+ }
+
+ index_free_lists->Empty();
+ data_free_lists->Empty();
+
+ rc_loop_counter = 0;
+ index_reorder_cnt = 0;
+ data_reorder_cnt = 0;
+ index_rc_status = MBError::NOT_INITIALIZED;
+ data_rc_status = MBError::NOT_INITIALIZED;
+ index_reorder_status = MBError::NOT_INITIALIZED;
+ data_reorder_status = MBError::NOT_INITIALIZED;
+ header->rc_m_index_off_pre = header->m_index_offset;
+ header->rc_m_data_off_pre = header->m_data_offset;
+
+ if(async_writer_ptr)
+ {
+ // set rc root to at the end of max size
+ rc_index_offset = dmm->GetResourceCollectionOffset();
+ rc_data_offset = dict->GetResourceCollectionOffset();
+
+ // make sure there is some space left at the end of current index
+ // [start|....|current_index_offset|...MIN_RC_OFFSET_GAP/more...|rc_index_offset|.....|end]
+ if(rc_index_offset < header->m_index_offset + MIN_RC_OFFSET_GAP ||
+ rc_data_offset < header->m_data_offset + MIN_RC_OFFSET_GAP)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "not enough space for rc, index: "
+ "%llu %d, %llu, data: %llu %d, %llu",
+ header->m_index_offset, MIN_RC_OFFSET_GAP, rc_index_offset,
+ header->m_data_offset, MIN_RC_OFFSET_GAP, rc_data_offset);
+ throw (int) MBError::OUT_OF_BOUND;
+ }
+
+ // update current indexs to rc offsets
+ header->m_index_offset = rc_index_offset;
+ header->m_data_offset = rc_data_offset;
+
+ // create rc root node
+ size_t rc_off = dmm->InitRootNode_RC();
+ header->rc_root_offset.store(rc_off, MEMORY_ORDER_WRITER);
+ }
+
+ Logger::Log(LOG_LEVEL_DEBUG, "setting rc index off start to: %llu", header->m_index_offset);
+ Logger::Log(LOG_LEVEL_DEBUG, "setting rc data off start to: %llu", header->m_data_offset);
+}
+
+void ResourceCollection::CollectBuffers()
+{
+ if((rc_type & RESOURCE_COLLECTION_TYPE_INDEX) &&
+ (index_reorder_status != MBError::SUCCESS))
+ return;
+ if((rc_type & RESOURCE_COLLECTION_TYPE_DATA) &&
+ (data_reorder_status != MBError::SUCCESS))
+ return;
+
+ TraverseDB(RESOURCE_COLLECTION_PHASE_COLLECT);
+
+ if(rc_type & RESOURCE_COLLECTION_TYPE_INDEX)
+ index_rc_status = MBError::SUCCESS;
+ if(rc_type & RESOURCE_COLLECTION_TYPE_DATA)
+ data_rc_status = MBError::SUCCESS;
+}
+
+void ResourceCollection::Finish()
+{
+ if(index_rc_status == MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "index buffer size reclaimed: %lld",
+ (header->rc_m_index_off_pre > index_size) ?
+ (header->rc_m_index_off_pre - index_size) : 0);
+ header->m_index_offset = index_size;
+ header->pending_index_buff_size = 0;
+ }
+ else
+ {
+ if(header->rc_m_index_off_pre == 0)
+ throw (int) MBError::INVALID_ARG;
+ header->m_index_offset = header->rc_m_index_off_pre;
+ }
+ if(data_rc_status == MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "data buffer size reclaimed: %lld",
+ (header->rc_m_data_off_pre > data_size) ?
+ (header->rc_m_data_off_pre - data_size) : 0);
+ header->m_data_offset = data_size;
+ header->pending_data_buff_size = 0;
+ }
+ else
+ {
+ if(header->rc_m_data_off_pre == 0)
+ throw (int) MBError::INVALID_ARG;
+ header->m_data_offset = header->rc_m_data_off_pre;
+ }
+
+ if(async_writer_ptr != NULL)
+ {
+ index_free_lists->Empty();
+ data_free_lists->Empty();
+ ProcessRCTree();
+ }
+
+ header->rc_m_index_off_pre = 0;
+ header->rc_m_data_off_pre = 0;
+
+ dict->RemoveUnused(header->m_data_offset, true);
+ dmm->RemoveUnused(header->m_index_offset, true);
+}
+
+bool ResourceCollection::MoveIndexBuffer(int phase, size_t &offset_src, int size)
+{
+ index_size = dmm->CheckAlignment(index_size, size);
+
+ if(index_size == offset_src)
+ return false;
+
+ uint8_t *ptr_src;
+ uint8_t *ptr_dst;
+ size_t offset_dst;
+ int rval;
+
+ if(phase == RESOURCE_COLLECTION_PHASE_REORDER)
+ {
+ if(index_size + size <= offset_src)
+ return false;
+
+ offset_dst = header->m_index_offset;
+ rval = dmm->Reserve(offset_dst, size, ptr_dst);
+ if(rval != MBError::SUCCESS)
+ throw rval;
+ header->m_index_offset = offset_dst + size;
+ index_reorder_cnt++;
+ }
+ else
+ {
+#ifdef __DEBUG__
+ assert(index_size + size <= offset_src);
+#endif
+ offset_dst = index_size;
+ ptr_dst = dmm->GetShmPtr(offset_dst, size);
+ }
+
+ ptr_src = dmm->GetShmPtr(offset_src, size);
+ BufferCopy(offset_dst, ptr_dst, offset_src, ptr_src, size, dmm);
+
+ offset_src = offset_dst;
+ return true;
+}
+
+bool ResourceCollection::MoveDataBuffer(int phase, size_t &offset_src, int size)
+{
+ data_size = dict->CheckAlignment(data_size, size);
+ if(data_size == offset_src)
+ return false;
+
+ uint8_t *ptr_src;
+ uint8_t *ptr_dst;
+ size_t offset_dst;
+ int rval;
+
+ if(phase == RESOURCE_COLLECTION_PHASE_REORDER)
+ {
+ if(data_size + size < offset_src)
+ return false;
+
+ offset_dst = header->m_data_offset;
+ rval = dict->Reserve(offset_dst, size, ptr_dst);
+ if(rval != MBError::SUCCESS)
+ throw rval;
+ header->m_data_offset = offset_dst + size;
+ data_reorder_cnt++;
+ }
+ else
+ {
+#ifdef __DEBUG__
+ assert(data_size + size <= offset_src);
+#endif
+ offset_dst = data_size;
+ ptr_dst = dict->GetShmPtr(offset_dst, size);
+ }
+
+ ptr_src = dict->GetShmPtr(offset_src, size);
+ BufferCopy(offset_dst, ptr_dst, offset_src, ptr_src, size, dict);
+
+ offset_src = offset_dst;
+ return true;
+}
+
+void ResourceCollection::DoTask(int phase, DBTraverseNode &dbt_node)
+{
+ if(phase == RESOURCE_COLLECTION_PHASE_REORDER)
+ {
+ // collect stats for adjusting values in header
+ if(dbt_node.buffer_type & BUFFER_TYPE_DATA)
+ db_cnt++;
+ if(dbt_node.buffer_type & BUFFER_TYPE_EDGE_STR)
+ edge_str_size += dbt_node.edgestr_size;
+ if(dbt_node.buffer_type & BUFFER_TYPE_NODE)
+ node_cnt++;
+ }
+
+ header->excep_lf_offset = dbt_node.edge_offset;
+ if(rc_type & RESOURCE_COLLECTION_TYPE_INDEX)
+ {
+ if(dbt_node.buffer_type & BUFFER_TYPE_NODE)
+ {
+ if(MoveIndexBuffer(phase, dbt_node.node_offset, dbt_node.node_size))
+ {
+ Write6BInteger(header->excep_buff, dbt_node.node_offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStart(dbt_node.edge_offset);
+#endif
+ header->excep_offset = dbt_node.node_link_offset;
+ header->excep_updating_status = EXCEP_STATUS_RC_NODE;
+ dmm->WriteData(header->excep_buff, OFFSET_SIZE, dbt_node.node_link_offset);
+ header->excep_updating_status = 0;
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+#endif
+ // Update data_link_offset since node may have been moved.
+ if(dbt_node.buffer_type & BUFFER_TYPE_DATA)
+ dbt_node.data_link_offset = dbt_node.node_offset + 2;
+ }
+ index_size += dbt_node.node_size;
+ }
+
+ if(dbt_node.buffer_type & BUFFER_TYPE_EDGE_STR)
+ {
+ if(MoveIndexBuffer(phase, dbt_node.edgestr_offset, dbt_node.edgestr_size))
+ {
+ Write5BInteger(header->excep_buff, dbt_node.edgestr_offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStart(dbt_node.edge_offset);
+#endif
+ header->excep_offset = dbt_node.edgestr_link_offset;
+ header->excep_updating_status = EXCEP_STATUS_RC_EDGE_STR;
+ dmm->WriteData(header->excep_buff, OFFSET_SIZE-1, dbt_node.edgestr_link_offset);
+ header->excep_updating_status = 0;
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+#endif
+ }
+ index_size += dbt_node.edgestr_size;
+ }
+ }
+
+ if(rc_type & RESOURCE_COLLECTION_TYPE_DATA)
+ {
+ if(dbt_node.buffer_type & BUFFER_TYPE_DATA)
+ {
+ if(MoveDataBuffer(phase, dbt_node.data_offset, dbt_node.data_size))
+ {
+ Write6BInteger(header->excep_buff, dbt_node.data_offset);
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStart(dbt_node.edge_offset);
+#endif
+ header->excep_offset = dbt_node.data_link_offset;;
+ header->excep_updating_status = EXCEP_STATUS_RC_DATA;
+ dmm->WriteData(header->excep_buff, OFFSET_SIZE, dbt_node.data_link_offset);
+ header->excep_updating_status = 0;
+#ifdef __LOCK_FREE__
+ lfree->WriterLockFreeStop();
+#endif
+ }
+ data_size += dbt_node.data_size;
+ }
+ }
+
+ header->excep_updating_status = 0;
+
+ if(async_writer_ptr != NULL)
+ {
+ if(rc_loop_counter++ > RC_TASK_CHECK)
+ {
+ rc_loop_counter = 0;
+ async_writer_ptr->ProcessTask(NUM_ASYNC_TASK, true);
+ }
+ }
+}
+
+
+void ResourceCollection::ReorderBuffers()
+{
+ if(rc_type & RESOURCE_COLLECTION_TYPE_INDEX)
+ Logger::Log(LOG_LEVEL_INFO, "index size before reorder: %llu", header->m_index_offset);
+ if(rc_type & RESOURCE_COLLECTION_TYPE_DATA)
+ Logger::Log(LOG_LEVEL_INFO, "data size before reorder: %llu", header->m_data_offset);
+
+ db_cnt = 0;
+ edge_str_size = 0;
+ node_cnt = 0;
+ TraverseDB(RESOURCE_COLLECTION_PHASE_REORDER);
+
+ if(rc_type & RESOURCE_COLLECTION_TYPE_INDEX)
+ Logger::Log(LOG_LEVEL_INFO, "index size after reorder: %llu", header->m_index_offset);
+ if(rc_type & RESOURCE_COLLECTION_TYPE_DATA)
+ Logger::Log(LOG_LEVEL_INFO, "data size after reorder: %llu", header->m_data_offset);
+
+ if(rc_type & RESOURCE_COLLECTION_TYPE_INDEX)
+ {
+ index_reorder_status = MBError::SUCCESS;
+ Logger::Log(LOG_LEVEL_INFO, "number of index buffer reordered: %lld", index_reorder_cnt);
+ }
+ if(rc_type & RESOURCE_COLLECTION_TYPE_DATA)
+ {
+ data_reorder_status = MBError::SUCCESS;
+ Logger::Log(LOG_LEVEL_INFO, "number of data buffer reordered: %lld", data_reorder_cnt);
+ }
+
+ if(db_cnt != header->count)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "adjusting db count to %lld from %lld", db_cnt, header->count);
+ header->count = db_cnt;
+ }
+ header->edge_str_size = edge_str_size;
+ header->n_states = node_cnt;
+}
+
+void ResourceCollection::ProcessRCTree()
+{
+ Logger::Log(LOG_LEVEL_INFO, "resource collection done, traversing the rc tree %llu entries", header->rc_count);
+
+ int count = 0;
+ int rval;
+ for(DB::iterator iter = db_ref.begin(false, true); iter != db_ref.end(); ++iter)
+ {
+ iter.value.options = 0;
+ rval = dict->Add((const uint8_t *)iter.key.data(), iter.key.size(), iter.value, true);
+ if(rval != MBError::SUCCESS)
+ Logger::Log(LOG_LEVEL_WARN, "failed to add: %s", MBError::get_error_str(rval));
+ if(count++ > RC_TASK_CHECK)
+ {
+ count = 0;
+ async_writer_ptr->ProcessTask(NUM_ASYNC_TASK, false);
+ }
+
+ if(header->m_index_offset > rc_index_offset ||
+ header->m_data_offset > rc_data_offset)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "not enough space for insertion: %llu, %llu",
+ header->m_index_offset, header->m_data_offset);
+ break;
+ }
+ }
+
+ header->rc_count = 0;
+ header->rc_root_offset.store(0, MEMORY_ORDER_WRITER);
+
+ // Clear the rc tree
+ dmm->ClearRootEdges_RC();
+}
+
+int ResourceCollection::ExceptionRecovery()
+{
+ if(!db_ref.is_open())
+ return db_ref.Status();
+
+ int rval = MBError::SUCCESS;
+ if(header->rc_m_index_off_pre != 0 && header->rc_m_data_off_pre != 0)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "previous rc was not completed successfully, retrying...");
+ try {
+ // This is a blocking call and should be called when writer starts up.
+ ReclaimResource(1, 1, MAX_6B_OFFSET, MAX_6B_OFFSET, NULL);
+ } catch (int err) {
+ if(err != MBError::RC_SKIPPED)
+ rval = err;
+ }
+
+ if(rval != MBError::SUCCESS)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to run rc recovery: %s, clear db!!!", MBError::get_error_str(rval));
+ dict->RemoveAll();
+ }
+ }
+
+ header->rc_root_offset = 0;
+ header->rc_count = 0;
+
+ return rval;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __MB_RC_H__
+#define __MB_RC_H__
+
+#include "db.h"
+#include "dict.h"
+#include "mbt_base.h"
+#include "async_writer.h"
+
+#define RESOURCE_COLLECTION_TYPE_INDEX 0x01
+#define RESOURCE_COLLECTION_TYPE_DATA 0x02
+
+#define RESOURCE_COLLECTION_PHASE_REORDER 0x01
+#define RESOURCE_COLLECTION_PHASE_COLLECT 0x02
+
+namespace mabain {
+
+// A garbage collector class
+class ResourceCollection : public DBTraverseBase
+{
+public:
+ ResourceCollection(const DB &db,
+ int rct = RESOURCE_COLLECTION_TYPE_INDEX |
+ RESOURCE_COLLECTION_TYPE_DATA);
+ virtual ~ResourceCollection();
+
+ void ReclaimResource(int64_t min_index_size, int64_t min_data_size,
+ int64_t max_dbsz, int64_t max_dbcnt,
+ AsyncWriter *awr = NULL);
+
+ // This function should be called when writer starts up.
+ int ExceptionRecovery();
+
+private:
+ void DoTask(int phase, DBTraverseNode &dbt_node);
+ void Prepare(int64_t min_index_size, int64_t min_data_size);
+ void CollectBuffers();
+ void ReorderBuffers();
+ void Finish();
+ bool MoveIndexBuffer(int phase, size_t &offset_src, int size);
+ bool MoveDataBuffer(int phase, size_t &offset_src, int size);
+ int LRUEviction();
+ void ProcessRCTree();
+
+ int rc_type;
+ int index_rc_status;
+ int data_rc_status;
+ int index_reorder_status;
+ int data_reorder_status;
+
+ // data for print gc stats only
+ int64_t index_reorder_cnt;
+ int64_t data_reorder_cnt;
+
+ // Async writer pointer
+ AsyncWriter *async_writer_ptr;
+
+ // resource collection offsets
+ size_t rc_index_offset;
+ size_t rc_data_offset;
+ int64_t rc_loop_counter;
+
+ int64_t db_cnt;
+ size_t edge_str_size;
+ int64_t node_cnt;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include "mbt_base.h"
+
+namespace mabain {
+
+DBTraverseBase::DBTraverseBase(const DB &db) : db_ref(db), rw_buffer(NULL)
+{
+ if(!(db.GetDBOptions() & CONSTS::ACCESS_MODE_WRITER))
+ throw (int) MBError::NOT_ALLOWED;
+
+ dict = db_ref.GetDictPtr();
+ if(dict == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+ dmm = dict->GetMM();
+ if(dmm == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+ header = dict->GetHeaderPtr();
+ if(header == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+ index_free_lists = dmm->GetFreeList();
+ if(index_free_lists == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+ data_free_lists = dict->GetFreeList();
+ if(data_free_lists == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+ lfree = dict->GetLockFreePtr();
+ if(lfree == NULL)
+ throw (int) MBError::NOT_INITIALIZED;
+
+ rw_buffer_size = 1024;
+ rw_buffer = new uint8_t[rw_buffer_size];
+}
+
+DBTraverseBase::~DBTraverseBase()
+{
+ if(rw_buffer != NULL)
+ delete [] rw_buffer;
+}
+
+void DBTraverseBase::TraverseDB(int arg)
+{
+ DB::iterator iter = DB::iterator(db_ref, DB_ITER_STATE_INIT);
+ int rval = iter.init_no_next();
+ if(rval != MBError::SUCCESS)
+ throw rval;
+
+ DBTraverseNode dbt_node;
+ index_size = dmm->GetRootOffset() + dmm->GetNodeSizePtr()[NUM_ALPHABET-1];
+ data_size = dict->GetStartDataOffset();
+ while(iter.next_dbt_buffer(&dbt_node))
+ {
+ GetAlignmentSize(dbt_node);
+
+ // Run-time determination
+ DoTask(arg, dbt_node);
+
+ if(dbt_node.buffer_type & BUFFER_TYPE_NODE)
+ {
+ iter.add_node_offset(dbt_node.node_offset);
+ }
+ }
+}
+
+void DBTraverseBase::GetAlignmentSize(DBTraverseNode &dbt_node) const
+{
+ if(dbt_node.buffer_type & BUFFER_TYPE_EDGE_STR)
+ dbt_node.edgestr_size = index_free_lists->GetAlignmentSize(dbt_node.edgestr_size);
+
+ if(dbt_node.buffer_type & BUFFER_TYPE_NODE)
+ dbt_node.node_size = index_free_lists->GetAlignmentSize(dbt_node.node_size);
+
+ if(dbt_node.buffer_type & BUFFER_TYPE_DATA)
+ {
+ uint16_t data_size[2];
+ if(dict->ReadData((uint8_t *)&data_size[0], DATA_HDR_BYTE, dbt_node.data_offset)
+ != DATA_HDR_BYTE)
+ throw (int) MBError::READ_ERROR;
+ dbt_node.data_size = data_free_lists->GetAlignmentSize(data_size[0] + DATA_HDR_BYTE);
+ }
+}
+
+void DBTraverseBase::BufferCopy(size_t offset_dst, uint8_t *ptr_dst,
+ size_t offset_src, const uint8_t *ptr_src,
+ int size, DRMBase *drm)
+{
+ if(ptr_src != NULL)
+ {
+ if(ptr_dst != NULL)
+ {
+ memcpy(ptr_dst, ptr_src, size);
+ }
+ else
+ {
+ drm->WriteData(ptr_src, size, offset_dst);
+ }
+ }
+ else
+ {
+ if(size > rw_buffer_size)
+ ResizeRWBuffer(size);
+ if(drm->ReadData(rw_buffer, size, offset_src) != size)
+ throw (int) MBError::READ_ERROR;
+
+ if(ptr_dst != NULL)
+ {
+ memcpy(ptr_dst, rw_buffer, size);
+ }
+ else
+ {
+ drm->WriteData(rw_buffer, size, offset_dst);
+ }
+ }
+}
+
+void DBTraverseBase::ResizeRWBuffer(int size)
+{
+ if(rw_buffer != NULL)
+ delete [] rw_buffer;
+ rw_buffer = new uint8_t[size];
+ rw_buffer_size = size;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __DBTraverseBase_H__
+#define __DBTraverseBase_H__
+
+#include "db.h"
+#include "dict.h"
+
+namespace mabain {
+
+typedef struct _DBTraverseNode
+{
+ size_t edge_offset;
+ size_t node_offset;
+ size_t node_link_offset;
+ int node_size;
+ size_t edgestr_offset;
+ size_t edgestr_link_offset;
+ int edgestr_size;
+ size_t data_offset;
+ size_t data_link_offset;
+ int data_size;
+ int buffer_type;
+} DBTraverseNode;
+
+// An abstract base class for writer to traverse mabain DB
+class DBTraverseBase
+{
+public:
+ DBTraverseBase(const DB &db);
+ ~DBTraverseBase();
+
+ // Traverse DB via DFS
+ void TraverseDB(int arg = 0);
+
+protected:
+ virtual void DoTask(int arg, DBTraverseNode &dbt_node) = 0;
+ void BufferCopy(size_t offset_dst, uint8_t *ptr_dst,
+ size_t offset_src, const uint8_t *ptr_src,
+ int size, DRMBase *drm);
+
+ // DBTraverseBase does not own these objects or pointers.
+ const DB &db_ref;
+ Dict *dict;
+ DictMem *dmm;
+ IndexHeader *header;
+ FreeList *index_free_lists;
+ FreeList *data_free_lists;
+ LockFree *lfree;
+
+ // Used for tracking index and data sizes that have been traversed.
+ size_t index_size;
+ size_t data_size;
+
+private:
+ void GetAlignmentSize(DBTraverseNode &dbt_node) const;
+ void ResizeRWBuffer(int size);
+
+ uint8_t *rw_buffer;
+ int rw_buffer_size;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <cstdlib>
+
+#include <sys/mman.h>
+#include <errno.h>
+#include <string.h>
+#include <assert.h>
+
+#include "file_io.h"
+#include "mmap_file.h"
+#include "error.h"
+#include "logger.h"
+#include "rollable_file.h"
+
+namespace mabain {
+
+MmapFileIO::MmapFileIO(const std::string &fpath, int mode, off_t filesize, bool sync)
+ : FileIO(fpath, mode, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH, sync)
+{
+ mmap_file = false;
+ mmap_size = 0;
+ mmap_start = 0xFFFFFFFFFFFFFFFF;
+ mmap_end = 0;
+ addr = NULL;
+
+ max_offset = 0;
+ curr_offset = 0;
+
+ if(options & MMAP_ANONYMOUS_MODE)
+ {
+ // Do not open file in anonymous mode.
+ return;
+ }
+
+ Logger::Log(LOG_LEVEL_DEBUG, "opening file " + fpath);
+
+ int fd = Open();
+ if(fd < 0)
+ {
+ int level = LOG_LEVEL_DEBUG;
+ if(mode & O_CREAT)
+ level = LOG_LEVEL_ERROR;
+ Logger::Log(level, "failed to open file %s with mode %d, errno %d",
+ fpath.c_str(), mode, errno);
+ return;
+ }
+
+ if(filesize > 0 && (mode & O_CREAT))
+ {
+ if(TruncateFile(filesize) != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "failed to truncate file %s with size %d",
+ fpath.c_str(), static_cast<int>(filesize));
+ Close();
+ }
+ }
+}
+
+MmapFileIO::~MmapFileIO()
+{
+ UnMapFile();
+}
+
+uint8_t* MmapFileIO::MapFile(size_t size, off_t offset, bool sliding)
+{
+ int mode = PROT_READ;
+ if(options & O_RDWR)
+ mode |= PROT_WRITE;
+
+ if(options & MMAP_ANONYMOUS_MODE)
+ {
+ assert(offset == 0 && !sliding);
+ addr = static_cast<unsigned char *>(mmap(NULL, size, mode,
+ MAP_SHARED | MAP_ANONYMOUS, -1, 0));
+ }
+ else
+ {
+ addr = static_cast<unsigned char *>(FileIO::MapFile(size, mode,
+ MAP_SHARED, offset));
+ }
+
+ if(addr == MAP_FAILED)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "%s mmap (%s) failed errno=%d offset=%llu size=%llu",
+ (options & MMAP_ANONYMOUS_MODE) ? "anon":"",
+ path.c_str(), errno, offset, size);
+ return NULL;
+ }
+
+ if(!sliding)
+ {
+ mmap_file = true;
+ mmap_size = size;
+ mmap_start = offset;
+ mmap_end = offset + size;
+ }
+
+ if(mode & PROT_WRITE)
+ {
+ Logger::Log(LOG_LEVEL_DEBUG, "mmap file %s, sliding=%d, size=%d, offset=%d",
+ path.c_str(), sliding, size, offset);
+ }
+
+ return addr;
+}
+
+void MmapFileIO::UnMapFile()
+{
+ if(mmap_file && addr != NULL)
+ {
+ munmap(addr, mmap_size);
+ addr = NULL;
+ }
+}
+
+// SeqWrite:
+// find the current offset first
+// call RandomWrite using the offset
+// reset the offset based on the number of bytes written
+size_t MmapFileIO::SeqWrite(const void *data, size_t size)
+{
+ size_t bytes_written = MmapFileIO::RandomWrite(data, size, curr_offset);
+ curr_offset += bytes_written;
+ return bytes_written;
+}
+
+size_t MmapFileIO::RandomWrite(const void *data, size_t size, off_t offset)
+{
+ if(data == NULL)
+ return 0;
+
+ size_t bytes_written;
+
+ if(!mmap_file)
+ {
+ // If the file is not mmaped, we just need to write to the file.
+ bytes_written = FileIO::RandomWrite(data, size, offset);
+ if(bytes_written + offset > max_offset)
+ {
+ max_offset = bytes_written + offset;
+ }
+ return bytes_written;
+ }
+
+ // If the file is mmaped, we need to write to the memory or file.
+ off_t offset_end = static_cast<off_t>(size) + offset;
+ const unsigned char *ptr = static_cast<const unsigned char *>(data);
+ if(offset < mmap_start)
+ {
+ if(offset_end <= mmap_start)
+ {
+ // no overlap with mmap region
+ bytes_written = FileIO::RandomWrite(ptr, size, offset);
+ }
+ else if(offset_end <= mmap_end)
+ {
+ // partial overlap with left region
+ size_t written_left = mmap_start - offset;
+ bytes_written = FileIO::RandomWrite(ptr, written_left, offset);
+ ptr += written_left;
+ // partial overlap with mmap region
+ memcpy(addr+mmap_start, ptr, size-written_left);
+ bytes_written += size - written_left;
+ if(sync_on_write)
+ RollableFile::ShmSync(addr+mmap_start, size-written_left);
+ }
+ else
+ {
+ // partial overlap with left region
+ size_t written_left = mmap_start - offset;
+ bytes_written = FileIO::RandomWrite(ptr, written_left, offset);
+ ptr += written_left;
+ // full overlap with mmap region
+ memcpy(addr+mmap_start, ptr, mmap_size);
+ bytes_written += mmap_size;
+ written_left += mmap_size;
+ ptr += mmap_size;
+ if(sync_on_write)
+ RollableFile::ShmSync(addr+mmap_start, mmap_size);
+ // partial overlap with right region
+ bytes_written += FileIO::RandomWrite(ptr, size-written_left, mmap_end);
+ }
+ }
+ else if(offset < mmap_end)
+ {
+ if(offset_end <= mmap_end)
+ {
+ // full data is within the mmap region
+ memcpy(addr+offset, ptr, size);
+ bytes_written = size;
+ if(sync_on_write)
+ RollableFile::ShmSync(addr+offset, size);
+ }
+ else
+ {
+ // partial overlap with mmap region
+ size_t written_left = mmap_end - offset;
+ memcpy(addr+offset, ptr, written_left);
+ ptr += written_left;
+ bytes_written = written_left;
+ if(sync_on_write)
+ RollableFile::ShmSync(addr+offset, written_left);
+ // partial overlap with the right region
+ bytes_written += FileIO::RandomWrite(ptr, size-written_left, mmap_end);
+ }
+ }
+ else
+ {
+ // no overlap with mmap region
+ bytes_written = FileIO::RandomWrite(ptr, size, offset);
+ }
+
+ if(bytes_written + offset > max_offset)
+ max_offset = bytes_written + offset;
+
+ return bytes_written;
+}
+
+// SeqRead:
+// find the current offset first
+// call RandomRead using the offset
+// reset the offset based on the number of bytes read
+size_t MmapFileIO::SeqRead(void *buff, size_t size)
+{
+ size_t bytes_read = MmapFileIO::RandomRead(buff, size, curr_offset);
+ curr_offset += bytes_read;
+ return bytes_read;
+}
+
+size_t MmapFileIO::RandomRead(void *buff, size_t size, off_t offset)
+{
+ if(buff == NULL)
+ return 0;
+
+ if(!mmap_file)
+ {
+ // If file is not mmaped, go directly to file IO.
+ return FileIO::RandomRead(buff, size, offset);
+ }
+
+ size_t bytes_read;
+ off_t offset_end = static_cast<off_t>(size) + offset;
+ unsigned char *ptr = static_cast<unsigned char *>(buff);
+ if(offset < mmap_start)
+ {
+ if(offset_end <= mmap_start)
+ {
+ // no overlap with mmap region
+ bytes_read = FileIO::RandomRead(ptr, size, offset);
+ }
+ else if(offset_end <= mmap_end)
+ {
+ // partial overlap with left region
+ size_t read_left = mmap_start - offset;
+ bytes_read = FileIO::RandomRead(ptr, read_left, offset);
+ ptr += read_left;
+ // partial overlap with mmap region
+ memcpy(ptr, addr+mmap_start, size-read_left);
+ bytes_read += size - read_left;
+ }
+ else
+ {
+ // partial overlap with left region
+ size_t read_left = mmap_start - offset;
+ bytes_read = FileIO::RandomRead(ptr, read_left, offset);
+ ptr += read_left;
+ // full overlap with mmap region
+ memcpy(ptr, addr+mmap_start, mmap_size);
+ bytes_read += mmap_size;
+ read_left += mmap_size;
+ ptr += mmap_size;
+ // partial overlap with right region
+ bytes_read += FileIO::RandomRead(ptr, size-read_left, mmap_end);
+ }
+ }
+ else if(offset < mmap_end)
+ {
+ if(offset_end <= mmap_end)
+ {
+ // full data is within the mmap region
+ memcpy(ptr, addr+offset, size);
+ bytes_read = size;
+ }
+ else
+ {
+ // full overlap with mmap region
+ size_t read_left = mmap_end - offset;
+ memcpy(ptr, addr+offset, read_left);
+ ptr += read_left;
+ bytes_read = read_left;
+ // partial overlap with the right region
+ bytes_read += FileIO::RandomRead(ptr, size-read_left, mmap_end);
+ }
+ }
+ else
+ {
+ // no overlap with mmap region
+ bytes_read = FileIO::RandomRead(ptr, size, offset);
+ }
+
+ return bytes_read;
+}
+
+bool MmapFileIO::IsMapped() const
+{
+ return mmap_file;
+}
+
+uint8_t* MmapFileIO::GetMapAddr() const
+{
+ return addr;
+}
+
+void MmapFileIO::Flush()
+{
+ if(options & MMAP_ANONYMOUS_MODE)
+ return;
+
+ if(addr != NULL)
+ msync(addr, mmap_size, MS_SYNC);
+ FileIO::Flush();
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __MMAP_FILE__
+#define __MMAP_FILE__
+
+#include <iostream>
+#include <string>
+#include <stdint.h>
+
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/types.h>
+
+#include "file_io.h"
+#include "logger.h"
+
+namespace mabain {
+
+// Memory mapped file class
+class MmapFileIO : public FileIO
+{
+public:
+ MmapFileIO(const std::string &fpath, int mode, off_t filesize, bool sync=false);
+ ~MmapFileIO();
+
+ uint8_t* MapFile(size_t size, off_t offset, bool sliding=false);
+ bool IsMapped() const;
+ size_t SeqWrite(const void *data, size_t size);
+ size_t RandomWrite(const void *data, size_t size, off_t offset);
+ size_t SeqRead(void *buff, size_t size);
+ size_t RandomRead(void *buff, size_t size, off_t offset);
+ void UnMapFile();
+ uint8_t* GetMapAddr() const;
+ void Flush();
+
+private:
+ bool mmap_file;
+ size_t mmap_size;
+ off_t mmap_start;
+ off_t mmap_end;
+ unsigned char *addr;
+ // The maximal offset where data have been written
+ size_t max_offset;
+ // Current offset for sequential reading of writing only
+ off_t curr_offset;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string.h>
+
+#include "resource_pool.h"
+#include "mabain_consts.h"
+#include "mmap_file.h"
+#include "error.h"
+
+namespace mabain {
+
+ResourcePool::ResourcePool()
+{
+ pthread_mutex_init(&pool_mutex, NULL);
+}
+
+ResourcePool::~ResourcePool()
+{
+ pthread_mutex_destroy(&pool_mutex);
+}
+
+void ResourcePool::RemoveAll()
+{
+ pthread_mutex_lock(&pool_mutex);
+ file_pool.clear();
+ pthread_mutex_unlock(&pool_mutex);
+}
+
+// check if a in-memory db already exists
+bool ResourcePool::CheckExistence(const std::string &header_path)
+{
+ pthread_mutex_lock(&pool_mutex);
+ auto search = file_pool.find(header_path);
+ pthread_mutex_unlock(&pool_mutex);
+
+ return (search != file_pool.end());
+}
+
+void ResourcePool::RemoveResourceByPath(const std::string &path)
+{
+ pthread_mutex_lock(&pool_mutex);
+ file_pool.erase(path);
+ pthread_mutex_unlock(&pool_mutex);
+}
+
+void ResourcePool::RemoveResourceByDB(const std::string &db_path)
+{
+ pthread_mutex_lock(&pool_mutex);
+
+ for(auto it = file_pool.begin(); it != file_pool.end();)
+ {
+ if(it->first.compare(0, db_path.size(), db_path) == 0)
+ it = file_pool.erase(it);
+ else
+ it++;
+ }
+
+ pthread_mutex_unlock(&pool_mutex);
+}
+
+std::shared_ptr<MmapFileIO> ResourcePool::OpenFile(const std::string &fpath,
+ int mode,
+ size_t file_size,
+ bool &map_file,
+ bool create_file)
+{
+ std::shared_ptr<MmapFileIO> mmap_file;
+
+ pthread_mutex_lock(&pool_mutex);
+
+ auto search = file_pool.find(fpath);
+ if(search == file_pool.end())
+ {
+ int flags = O_RDWR;
+ if(create_file)
+ flags |= O_CREAT;
+ if(mode & CONSTS::MEMORY_ONLY_MODE)
+ flags |= MMAP_ANONYMOUS_MODE;
+
+ mmap_file = std::shared_ptr<MmapFileIO>
+ (
+ new MmapFileIO(fpath,
+ flags,
+ file_size,
+ mode & CONSTS::SYNC_ON_WRITE)
+ );
+ if(map_file)
+ {
+ if(mmap_file->MapFile(file_size, 0) != NULL)
+ {
+ if(!(mode & CONSTS::MEMORY_ONLY_MODE))
+ mmap_file->Close();
+ }
+ else
+ {
+ map_file = false;
+ }
+ }
+
+ file_pool[fpath] = mmap_file;
+ }
+ else
+ {
+ mmap_file = search->second;
+ }
+
+ pthread_mutex_unlock(&pool_mutex);
+ return mmap_file;
+}
+
+int ResourcePool::AddResourceByPath(const std::string &path, std::shared_ptr<MmapFileIO> resource)
+{
+ int rval = MBError::IN_DICT;
+
+ pthread_mutex_lock(&pool_mutex);
+ auto search = file_pool.find(path);
+ if(search == file_pool.end())
+ {
+ file_pool[path] = resource;
+ rval = MBError::SUCCESS;
+ }
+ pthread_mutex_unlock(&pool_mutex);
+
+ return rval;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __RESOURCE_POOL__
+#define __RESOURCE_POOL__
+
+#include <map>
+#include <memory>
+#include <string>
+#include <pthread.h>
+
+#include "mmap_file.h"
+
+namespace mabain {
+
+// A singleton class for managing resource/file descriptors using
+// shared_ptr. All db handles for the same db will share the same
+// file descriptors so that we won't be running out of file descriptors
+// when there are a large number of DB handles opened.
+class ResourcePool
+{
+public:
+ ~ResourcePool();
+
+ std::shared_ptr<MmapFileIO> OpenFile(const std::string &fpath, int mode,
+ size_t file_size, bool &map_file,
+ bool create_file);
+ void RemoveResourceByDB(const std::string &db_path);
+ void RemoveResourceByPath(const std::string &path);
+ void RemoveAll();
+ bool CheckExistence(const std::string &header_path);
+ int AddResourceByPath(const std::string &path, std::shared_ptr<MmapFileIO> resource);
+
+ static ResourcePool& getInstance() {
+ static ResourcePool instance; // only one instance per process
+ return instance;
+ }
+
+private:
+ ResourcePool();
+
+ std::map<std::string, std::shared_ptr<MmapFileIO>> file_pool;
+ pthread_mutex_t pool_mutex;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <sstream>
+#include <cstdlib>
+#include <stdint.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <errno.h>
+#include <climits>
+
+#include "db.h"
+#include "rollable_file.h"
+#include "logger.h"
+#include "error.h"
+#include "resource_pool.h"
+
+namespace mabain {
+
+#define SLIDING_MEM_SIZE 16LLU*1024*1024 // 16M
+#define MAX_NUM_BLOCK 2*1024 // 2K
+#define RC_OFFSET_PERCENTAGE 75 // default rc offset is placed at 75% of maximum size
+
+const long RollableFile::page_size = sysconf(_SC_PAGESIZE);
+int RollableFile::ShmSync(uint8_t *addr, int size)
+{
+ off_t page_offset = ((off_t) addr) % RollableFile::page_size;
+ return msync(addr-page_offset, size+page_offset, MS_SYNC);
+}
+
+RollableFile::RollableFile(const std::string &fpath, size_t blocksize, size_t memcap, int access_mode,
+ long max_block, int in_rc_offset_percentage)
+ : path(fpath),
+ block_size(blocksize),
+ mmap_mem(memcap),
+ sliding_mmap(access_mode & CONSTS::USE_SLIDING_WINDOW),
+ mode(access_mode),
+ max_num_block(max_block),
+ rc_offset_percentage(in_rc_offset_percentage),
+ mem_used(0)
+{
+ sliding_addr = NULL;
+ sliding_mem_size = SLIDING_MEM_SIZE;
+ sliding_size = 0;
+ sliding_start = 0;
+ sliding_map_off = 0;
+ shm_sliding_start_ptr = NULL;
+
+ if(mode & CONSTS::ACCESS_MODE_WRITER)
+ {
+ if(max_num_block == 0 || max_num_block > MAX_NUM_BLOCK)
+ max_num_block = MAX_NUM_BLOCK;
+
+ Logger::Log(LOG_LEVEL_INFO, "maximal block number for %s is %d",
+ fpath.c_str(), max_num_block);
+
+ if(rc_offset_percentage == 0 || rc_offset_percentage > 100 || rc_offset_percentage < 50)
+ rc_offset_percentage = RC_OFFSET_PERCENTAGE;
+
+ Logger::Log(LOG_LEVEL_INFO, "rc_offset_percentage is set to %d", rc_offset_percentage);
+ }
+
+ Logger::Log(LOG_LEVEL_INFO, "opening rollable file %s for %s, mmap size: %d",
+ path.c_str(), (mode & CONSTS::ACCESS_MODE_WRITER)?"writing":"reading", mmap_mem);
+ if(!sliding_mmap)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "sliding mmap is turned off for " + fpath);
+ }
+ else
+ {
+ // page_size is used to check page alignment when mapping file to memory.
+ if(RollableFile::page_size < 0)
+ {
+ Logger::Log(LOG_LEVEL_WARN, "failed to get page size, turning off sliding memory");
+ sliding_mmap = false;
+ }
+ else
+ {
+ Logger::Log(LOG_LEVEL_INFO, "sliding mmap is turned on for " + fpath);
+ }
+ }
+
+ files.assign(3, NULL);
+ if(mode & CONSTS::SYNC_ON_WRITE)
+ Logger::Log(LOG_LEVEL_INFO, "Sync is turned on for " + fpath);
+}
+
+void RollableFile::InitShmSlidingAddr(std::atomic<size_t> *shm_sliding_addr)
+{
+ shm_sliding_start_ptr = shm_sliding_addr;
+#ifdef __DEBUG__
+ assert(shm_sliding_start_ptr != NULL);
+#endif
+}
+
+void RollableFile::Close()
+{
+ if(sliding_addr != NULL)
+ {
+ munmap(sliding_addr, sliding_size);
+ sliding_addr = NULL;
+ }
+}
+
+RollableFile::~RollableFile()
+{
+ Close();
+}
+
+int RollableFile::OpenAndMapBlockFile(size_t block_order, bool create_file)
+{
+ if(block_order >= max_num_block)
+ {
+ int level = LOG_LEVEL_DEBUG;
+ if(mode & CONSTS::ACCESS_MODE_WRITER)
+ level = LOG_LEVEL_WARN;
+ Logger::Log(level, "block number %d ovferflow", block_order);
+ return MBError::NO_RESOURCE;
+ }
+
+ int rval = MBError::SUCCESS;
+ std::stringstream ss;
+ ss << block_order;
+#ifdef __DEBUG__
+ assert(files[block_order] == NULL);
+#endif
+
+ bool map_file;
+ if(mmap_mem > mem_used)
+ map_file = true;
+ else
+ map_file = false;
+ if(!map_file && (mode & CONSTS::MEMORY_ONLY_MODE))
+ return MBError::NO_MEMORY;
+
+ files[block_order] = ResourcePool::getInstance().OpenFile(path+ss.str(),
+ mode,
+ block_size,
+ map_file,
+ create_file);
+ if(map_file)
+ mem_used += block_size;
+ else if(mode & CONSTS::MEMORY_ONLY_MODE)
+ rval = MBError::MMAP_FAILED;
+ return rval;
+}
+
+// Need to make sure the required size at offset is aligned with
+// block_size and mmap_size. We should not write the size in two
+// different blocks or one in mmaped region and the other one on disk.
+size_t RollableFile::CheckAlignment(size_t offset, int size)
+{
+ size_t block_offset = offset % block_size;
+
+ if(block_offset + size > block_size)
+ {
+ // Start at the begining of the next block
+ offset = offset + block_size - block_offset;
+ }
+
+ return offset;
+}
+
+int RollableFile::CheckAndOpenFile(size_t order, bool create_file)
+{
+ int rval = MBError::SUCCESS;
+
+ if(order >= static_cast<size_t>(files.size()))
+ files.resize(order+3, NULL);
+
+ if(files[order] == NULL)
+ rval = OpenAndMapBlockFile(order, create_file);
+
+ return rval;
+}
+
+// Get shared memory address for existing buffer
+// No need to check alignment
+uint8_t* RollableFile::GetShmPtr(size_t offset, int size)
+{
+ size_t order = offset / block_size;
+ int rval = CheckAndOpenFile(order, false);
+
+ if(rval != MBError::SUCCESS)
+ return NULL;
+
+ if(files[order]->IsMapped())
+ {
+ size_t index = offset % block_size;
+ return files[order]->GetMapAddr() + index;
+ }
+
+ if(sliding_mmap)
+ {
+ if(static_cast<off_t>(offset) >= sliding_start &&
+ offset + size <= sliding_start + sliding_size)
+ {
+ if(sliding_addr != NULL)
+ return sliding_addr + (offset % block_size) - sliding_map_off;
+ }
+ }
+
+ return NULL;
+}
+
+int RollableFile::Reserve(size_t &offset, int size, uint8_t* &ptr, bool map_new_sliding)
+{
+ int rval;
+ ptr = NULL;
+ offset = CheckAlignment(offset, size);
+
+ size_t order = offset / block_size;
+ rval = CheckAndOpenFile(order, true);
+ if(rval != MBError::SUCCESS)
+ return rval;
+
+ if(files[order]->IsMapped())
+ {
+ size_t index = offset % block_size;
+ ptr = files[order]->GetMapAddr() + index;
+ return rval;
+ }
+
+ if(sliding_mmap)
+ {
+ if(static_cast<off_t>(offset) >= sliding_start &&
+ offset + size <= sliding_start + sliding_size)
+ {
+ if(sliding_addr != NULL)
+ ptr = sliding_addr + (offset % block_size) - sliding_map_off;
+ }
+ else if(map_new_sliding && offset >= sliding_start + sliding_size)
+ {
+ ptr = NewSlidingMapAddr(order, offset, size);
+ if(ptr != NULL)
+ {
+ // Load the mmap starting offset to shared memory so that readers
+ // can map the same region when reading it.
+ shm_sliding_start_ptr->store(sliding_start, std::memory_order_relaxed);
+ }
+ }
+ }
+
+ return rval;
+}
+
+uint8_t* RollableFile::NewSlidingMapAddr(size_t order, size_t offset, int size)
+{
+ if(sliding_addr != NULL)
+ {
+ // No need to call msync since munmap will write all memory
+ // update to disk
+ //msync(sliding_addr, sliding_size, MS_SYNC);
+ munmap(sliding_addr, sliding_size);
+ }
+
+ if(sliding_start == 0)
+ {
+ sliding_start = offset;
+ }
+ else
+ {
+ sliding_start += sliding_size;
+ }
+
+ // Check page alignment
+ int page_alignment = sliding_start % RollableFile::page_size;
+ if(page_alignment != 0)
+ {
+ sliding_start -= page_alignment;
+ if(sliding_start < 0)
+ sliding_start = 0;
+ }
+ sliding_map_off = sliding_start % block_size;
+ if(sliding_map_off + sliding_mem_size > block_size)
+ {
+ sliding_size = block_size - sliding_map_off;
+ }
+ else
+ {
+ sliding_size = sliding_mem_size;
+ }
+
+ order = sliding_start / block_size;
+ sliding_addr = files[order]->MapFile(sliding_size, sliding_map_off, true);
+ if(sliding_addr != NULL)
+ {
+ if(static_cast<off_t>(offset) >= sliding_start &&
+ offset+size <= sliding_start+sliding_size)
+ return sliding_addr + (offset % block_size) - sliding_map_off;
+ }
+ else
+ {
+ Logger::Log(LOG_LEVEL_WARN, "last mmap failed, disable sliding mmap");
+ sliding_mmap = false;
+ }
+
+ return NULL;
+}
+
+size_t RollableFile::RandomWrite(const void *data, size_t size, off_t offset)
+{
+ size_t order = offset / block_size;
+ int rval = CheckAndOpenFile(order, false);
+ if(rval != MBError::SUCCESS)
+ return 0;
+
+ // Check sliding map
+ if(sliding_mmap && sliding_addr != NULL)
+ {
+ if(offset >= sliding_start && offset+size <= sliding_start+sliding_size)
+ {
+ uint8_t *start_addr = sliding_addr + (offset % block_size) - sliding_map_off;
+ memcpy(start_addr, data, size);
+ if(mode & CONSTS::SYNC_ON_WRITE)
+ {
+ off_t page_off = ((off_t) start_addr) % RollableFile::page_size;
+ if(msync(start_addr-page_off, size+page_off, MS_SYNC) == -1)
+ std::cout<<"msync error\n";
+ }
+ return size;
+ }
+ }
+
+ int index = offset % block_size;
+ return files[order]->RandomWrite(data, size, index);
+}
+
+void* RollableFile::NewReaderSlidingMap(size_t order)
+{
+ off_t start_off = shm_sliding_start_ptr->load(std::memory_order_relaxed);
+ if(start_off == 0 || start_off == sliding_start || start_off/block_size != (unsigned)order)
+ return NULL;
+
+ if(sliding_addr != NULL)
+ munmap(sliding_addr, sliding_size);
+ sliding_start = start_off;
+ sliding_map_off = sliding_start % block_size;
+ if(sliding_map_off + SLIDING_MEM_SIZE > block_size)
+ {
+ sliding_size = block_size - sliding_map_off;
+ }
+ else
+ {
+ sliding_size = SLIDING_MEM_SIZE;
+ }
+
+ sliding_addr = files[order]->MapFile(sliding_size, sliding_map_off, true);
+ return sliding_addr;
+}
+
+size_t RollableFile::RandomRead(void *buff, size_t size, off_t offset)
+{
+ size_t order = offset / block_size;
+
+ int rval = CheckAndOpenFile(order, false);
+ if(rval != MBError::SUCCESS && rval != MBError::MMAP_FAILED)
+ return 0;
+
+ if(sliding_mmap)
+ {
+ if(!(mode & CONSTS::ACCESS_MODE_WRITER))
+ NewReaderSlidingMap(order);
+
+ // Check sliding map
+ if(sliding_addr != NULL && offset >= sliding_start &&
+ offset+size <= sliding_start+sliding_size)
+ {
+ memcpy(buff, sliding_addr+(offset%block_size)-sliding_map_off, size);
+ return size;
+ }
+ }
+
+ int index = offset % block_size;
+ return files[order]->RandomRead(buff, size, index);
+}
+
+void RollableFile::PrintStats(std::ostream &out_stream) const
+{
+ out_stream << "Rollable file: " << path << " stats:" << std::endl;
+ out_stream << "\tshared memory size: " << mmap_mem << std::endl;
+ if(sliding_mmap)
+ {
+ out_stream << "\tsliding mmap start: " << sliding_start << std::endl;
+ out_stream << "\tsliding mmap size: " << sliding_mem_size << std::endl;
+ }
+}
+
+void RollableFile::ResetSlidingWindow()
+{
+ if(sliding_addr != NULL)
+ {
+ munmap(sliding_addr, sliding_size);
+ sliding_addr = NULL;
+ }
+
+ sliding_size = 0;
+ sliding_start = 0;
+ sliding_map_off = 0;
+}
+
+void RollableFile::Flush()
+{
+ for (std::vector<std::shared_ptr<MmapFileIO>>::iterator it = files.begin();
+ it != files.end(); ++it)
+ {
+ if(*it != NULL)
+ {
+ (*it)->Flush();
+ }
+ }
+}
+
+size_t RollableFile::GetResourceCollectionOffset() const
+{
+ return int((rc_offset_percentage / 100.0f) * max_num_block) * block_size;
+}
+
+void RollableFile::RemoveUnused(size_t max_size, bool writer_mode)
+{
+ unsigned ibeg = max_size/(block_size + 1) + 1;
+ for(auto i = ibeg; i < files.size(); i++)
+ {
+ if(files[i] != NULL)
+ {
+ if(files[i]->IsMapped() && mem_used > block_size)
+ mem_used -= block_size;
+ if(writer_mode)
+ {
+ ResourcePool::getInstance().RemoveResourceByPath(files[i]->GetFilePath());
+ unlink(files[i]->GetFilePath().c_str());
+ }
+ files[i] = NULL;
+ }
+ }
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __ROLLABLE_FILE_H__
+#define __ROLLABLE_FILE_H__
+
+#include <string>
+#include <vector>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <assert.h>
+#include <atomic>
+#include <memory>
+
+#include "mmap_file.h"
+#include "logger.h"
+
+namespace mabain {
+
+// Memory mapped file that can be rolled based on block size
+class RollableFile {
+public:
+ RollableFile(const std::string &fpath, size_t blocksize,
+ size_t memcap, int access_mode, long max_block=0, int rc_offset_percentage=75);
+ ~RollableFile();
+
+ size_t RandomWrite(const void *data, size_t size, off_t offset);
+ size_t RandomRead(void *buff, size_t size, off_t offset);
+ void InitShmSlidingAddr(std::atomic<size_t> *shm_sliding_addr);
+ int Reserve(size_t &offset, int size, uint8_t* &ptr, bool map_new_sliding=true);
+ uint8_t* GetShmPtr(size_t offset, int size);
+ size_t CheckAlignment(size_t offset, int size);
+ void PrintStats(std::ostream &out_stream = std::cout) const;
+ void Close();
+ void ResetSlidingWindow();
+
+ void Flush();
+ size_t GetResourceCollectionOffset() const;
+ void RemoveUnused(size_t max_size, bool writer_mode);
+
+ static const long page_size;
+ static int ShmSync(uint8_t *addr, int size);
+
+private:
+ int OpenAndMapBlockFile(size_t block_order, bool create_file);
+ int CheckAndOpenFile(size_t block_order, bool create_file);
+ uint8_t* NewSlidingMapAddr(size_t order, size_t offset, int size);
+ void* NewReaderSlidingMap(size_t order);
+
+ std::string path;
+ size_t block_size;
+ size_t mmap_mem;
+ bool sliding_mmap;
+ int mode;
+ size_t sliding_mem_size;
+ // shared memory sliding start offset for reader
+ // Note writer does not flush sliding mmap during writing.
+ // Readers have to mmap the same region so that they won't
+ // read unflushed data from disk.
+ std::atomic<size_t> *shm_sliding_start_ptr;
+
+ size_t max_num_block;
+
+ std::vector<std::shared_ptr<MmapFileIO>> files;
+ uint8_t* sliding_addr;
+ size_t sliding_size;
+ off_t sliding_start;
+ off_t sliding_map_off;
+
+ int rc_offset_percentage;
+ size_t mem_used;
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <pthread.h>
+
+#include "dict.h"
+#include "error.h"
+#include "async_writer.h"
+#include "./util/shm_mutex.h"
+
+#ifdef __SHM_QUEUE__
+
+namespace mabain {
+
+int Dict::SHMQ_Add(const char *key, int key_len, const char *data, int data_len,
+ bool overwrite)
+{
+ if(key_len > MB_ASYNC_SHM_KEY_SIZE || data_len > MB_ASYNC_SHM_DATA_SIZE)
+ {
+ return MBError::OUT_OF_BOUND;
+ }
+
+ AsyncNode *node_ptr = SHMQ_AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ memcpy(node_ptr->key, key, key_len);
+ memcpy(node_ptr->data, data, data_len);
+ node_ptr->key_len = key_len;
+ node_ptr->data_len = data_len;
+ node_ptr->overwrite = overwrite;
+
+ node_ptr->type = MABAIN_ASYNC_TYPE_ADD;
+ return SHMQ_PrepareSlot(node_ptr);
+}
+
+int Dict::SHMQ_Remove(const char *key, int len)
+{
+ if(len > MB_ASYNC_SHM_KEY_SIZE)
+ return MBError::OUT_OF_BOUND;
+
+ AsyncNode *node_ptr = SHMQ_AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ memcpy(node_ptr->key, key, len);
+ node_ptr->key_len = len;
+ node_ptr->type = MABAIN_ASYNC_TYPE_REMOVE;
+ return SHMQ_PrepareSlot(node_ptr);
+}
+
+int Dict::SHMQ_RemoveAll()
+{
+ AsyncNode *node_ptr = SHMQ_AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ node_ptr->type = MABAIN_ASYNC_TYPE_REMOVE_ALL;
+ return SHMQ_PrepareSlot(node_ptr);
+}
+
+int Dict::SHMQ_Backup(const char *backup_dir)
+{
+ if(backup_dir == NULL)
+ return MBError::INVALID_ARG;
+ if(strlen(backup_dir) >= MB_ASYNC_SHM_DATA_SIZE)
+ return MBError::OUT_OF_BOUND;
+
+ AsyncNode *node_ptr = SHMQ_AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+ snprintf(node_ptr->data, MB_ASYNC_SHM_DATA_SIZE, "%s", backup_dir);
+ node_ptr->type = MABAIN_ASYNC_TYPE_BACKUP;
+ return SHMQ_PrepareSlot(node_ptr);
+}
+
+int Dict::SHMQ_CollectResource(int64_t m_index_rc_size,
+ int64_t m_data_rc_size,
+ int64_t max_dbsz,
+ int64_t max_dbcnt)
+{
+ AsyncNode *node_ptr = SHMQ_AcquireSlot();
+ if(node_ptr == NULL)
+ return MBError::MUTEX_ERROR;
+
+ int64_t *data_ptr = (int64_t *) node_ptr->data;
+ node_ptr->data_len = sizeof(int64_t)*4;
+ data_ptr[0] = m_index_rc_size;
+ data_ptr[1] = m_data_rc_size;
+ data_ptr[2] = max_dbsz;
+ data_ptr[3] = max_dbcnt;
+ node_ptr->type = MABAIN_ASYNC_TYPE_RC;
+
+ return SHMQ_PrepareSlot(node_ptr);
+}
+
+AsyncNode* Dict::SHMQ_AcquireSlot() const
+{
+ uint32_t index = header->queue_index.fetch_add(1, std::memory_order_release);
+ AsyncNode *node_ptr = queue + (index % header->async_queue_size);
+
+ struct timespec tm_exp;
+ tm_exp.tv_sec = time(NULL) + MB_ASYNC_SHM_LOCK_TMOUT;
+ tm_exp.tv_nsec = 0;
+ int rval = pthread_mutex_timedlock(&node_ptr->mutex, &tm_exp);
+ if(rval != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "shared memory mutex lock failed: %d", rval);
+ return NULL;
+ }
+
+ while(node_ptr->in_use.load(std::memory_order_consume))
+ {
+ tm_exp.tv_sec += MB_ASYNC_SHM_LOCK_TMOUT;
+ rval = pthread_cond_timedwait(&node_ptr->cond, &node_ptr->mutex, &tm_exp);
+ if(rval != 0)
+ {
+ Logger::Log(LOG_LEVEL_ERROR, "shared memory conditional wait failed: %d", rval);
+ if(rval == ETIMEDOUT)
+ {
+ Logger::Log(LOG_LEVEL_INFO, "pthread_cond_timedwait timedout, "
+ "check if async writer is running");
+ }
+ if((rval = pthread_mutex_unlock(&node_ptr->mutex)) != 0)
+ Logger::Log(LOG_LEVEL_ERROR, "shm mutex unlock failed: %d", rval);
+ return NULL;
+ }
+ }
+
+ return node_ptr;
+}
+
+int Dict::SHMQ_PrepareSlot(AsyncNode *node_ptr) const
+{
+ node_ptr->in_use.store(true, std::memory_order_release);
+ pthread_cond_signal(&node_ptr->cond);
+ if(pthread_mutex_unlock(&node_ptr->mutex) != 0)
+ return MBError::MUTEX_ERROR;
+
+ return MBError::SUCCESS;
+}
+
+bool Dict::SHMQ_Busy() const
+{
+ if((header->queue_index.load(std::memory_order_consume) != header->writer_index) || header->rc_flag == 1)
+ return true;
+
+ size_t rc_off = header->rc_root_offset.load(std::memory_order_consume);
+ return rc_off != 0;
+}
+
+}
+
+#endif
--- /dev/null
+CPP=g++
+
+CPPFLAGS = -I. -I.. -I../util -Wall -Werror -g -O2 -c -std=c++11
+LDFLAGS = -lpthread -lcrypto
+
+TESTSOURCES=$(wildcard *.cpp)
+
+all: mb_test mb_test1 mb_test2 mb_test_mp
+
+mb_test: mabain_test.cpp ../libmabain.so
+ $(CPP) $(CPPFLAGS) mabain_test.cpp
+ $(CPP) mabain_test.o -o mb_test -L../ -lmabain $(LDFLAGS)
+
+mb_test1: mbtest1.cpp ../libmabain.so
+ $(CPP) $(CPPFLAGS) mbtest1.cpp
+ $(CPP) mbtest1.o -o mb_test1 -L../ -lmabain $(LDFLAGS)
+
+mb_test2: mbtest2.cpp ../libmabain.so
+ $(CPP) $(CPPFLAGS) mbtest2.cpp
+ $(CPP) mbtest2.o -o mb_test2 -L../ -lmabain $(LDFLAGS)
+
+mb_test_mp: mbtest_mp.cpp ../libmabain.so
+ $(CPP) $(CPPFLAGS) mbtest_mp.cpp
+ $(CPP) mbtest_mp.o -o mb_test_mp -L../ -lmabain $(LDFLAGS)
+
+clean:
+ -rm -rf *.o mb_test*
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <assert.h>
+#include <unistd.h>
+#include <sys/time.h>
+
+#include "../db.h"
+#include "../dict_mem.h"
+#include "../dict.h"
+#include "../error.h"
+#include "../mabain_consts.h"
+#include "../resource_pool.h"
+
+using namespace mabain;
+
+static const char* MB_DIR = "/var/tmp/mabain_test/";
+static bool debug = false;
+
+static void clean_db_dir()
+{
+ std::string cmd = std::string("rm -rf ") + MB_DIR + "/_mabain_* >" + MB_DIR + "/out 2>" + MB_DIR + "/err";
+ if(system(cmd.c_str()) != 0) {
+ }
+ cmd = std::string("rm -rf ") + MB_DIR + "/backup/_mabain_* >" + MB_DIR + "/out 2>" + MB_DIR + "/err";
+ if(system(cmd.c_str()) != 0) {
+ }
+}
+
+#define VALUE_LENGTH 16
+static void load_test(std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ std::cout << "Loading " << list_file << "\n";
+ DB db(mbconf);
+ if(db.Status() != MBError::SUCCESS) {
+ std::cout << "DB error: " << MBError::get_error_str(db.Status()) << "\n";
+ exit(1);
+ }
+ std::ifstream in(list_file.c_str());
+ if(!in.is_open()) {
+ std::cerr << "cannot open file " << list_file << "\n";
+ in.close();
+ return;
+ }
+
+ int count = 0;
+ int count_existing = 0;
+ std::string line;
+ char buff[256];
+ int rval;
+ struct timeval start, stop;
+
+ memset(buff, 0, sizeof(buff));
+ assert(sizeof(buff) >= VALUE_LENGTH);
+ for(int i = 0; i < VALUE_LENGTH; i++) {
+ buff[i] = 'a' + i;
+ }
+
+ gettimeofday(&start, NULL);
+ while(std::getline(in, line)) {
+ if(line.length() > (unsigned)CONSTS::MAX_KEY_LENGHTH) {
+ if(debug)
+ std::cout<<line<<"\n";
+ continue;
+ }
+
+ rval = db.Add(line.c_str(), line.length(), buff, VALUE_LENGTH);
+ if(rval == MBError::SUCCESS) {
+ count++;
+ }
+ else if(rval == MBError::IN_DICT) {
+ count_existing++;
+ }
+ else {
+ std::cout << "Adding: " << line << ": " << MBError::get_error_str(rval) << "\n";
+ }
+
+ if(count % 1000000 == 0) std::cout << "Added " << count << "\n";
+ }
+ gettimeofday(&stop, NULL);
+ in.close();
+ db.PrintStats();
+ int64_t db_cnt = db.Count();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " " << count_existing <<
+ " time: " << 1.0*tmdiff/count << "\n";
+ assert(expected_count == count);
+ assert(expected_count == db_cnt);
+}
+
+static void update_test(std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ std::cout << "Updating " << list_file << "\n";
+ DB db(mbconf);
+ if(db.Status() != MBError::SUCCESS) {
+ std::cout << "DB error: " << MBError::get_error_str(db.Status()) << "\n";
+ exit(1);
+ }
+ std::ifstream in(list_file.c_str());
+ if(!in.is_open()) {
+ std::cerr << "cannot open file " << list_file << "\n";
+ in.close();
+ return;
+ }
+
+ struct timeval start, stop;
+ int64_t count = 0;
+ std::string line;
+ int rval;
+ gettimeofday(&start, NULL);
+ while(std::getline(in, line)) {
+ if(line.length() > (unsigned)CONSTS::MAX_KEY_LENGHTH) {
+ if(debug)
+ std::cout<<line<<"\n";
+ continue;
+ }
+
+ rval = db.Add(line, line, true);
+ assert(rval == MBError::SUCCESS);
+ count++;
+ if(count % 1000000 == 0) std::cout << "Added " << count << "\n";
+ }
+ in.close();
+ gettimeofday(&stop, NULL);
+ db.PrintStats();
+ int64_t db_cnt = db.Count();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " time: " << 1.0*tmdiff/db_cnt << "\n";
+ assert(expected_count == db_cnt);
+}
+
+static void lookup_test(std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ std::cout << "Lookup test: " << list_file << "\n";
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+ std::ifstream in(list_file.c_str());
+ assert(in.is_open());
+ int count = 0;
+ std::string line;
+ int rval;
+ MBData data;
+ struct timeval start, stop;
+ int found = 0;
+ gettimeofday(&start, NULL);
+ while(std::getline(in, line)) {
+ if(line.length() > (unsigned) CONSTS::MAX_KEY_LENGHTH) {
+ continue;
+ }
+ rval = db.Find(line.c_str(), line.length(), data);
+ if(rval == MBError::SUCCESS) {
+ found++;
+ } else {
+ if(debug)
+ std::cout << "failed: " << line << "\n";
+ }
+ count++;
+ if(count % 1000000 == 0) std::cout << "Looked up " << count << "\n";
+ }
+ gettimeofday(&stop, NULL);
+ in.close();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " time: " << 1.0*tmdiff/count << "\n";
+ std::cout << "found: " << found << "\n";
+ assert(found == expected_count);
+}
+
+static void prefix_lookup_test(std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ std::cout << "Lookup test: " << list_file << "\n";
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+ std::ifstream in(list_file.c_str());
+ assert(in.is_open());
+ int count = 0;
+ std::string line;
+ int rval;
+ MBData data(256, CONSTS::OPTION_ALL_PREFIX);
+ struct timeval start, stop;
+ int nfound = 0;
+ gettimeofday(&start, NULL);
+ while(std::getline(in, line)) {
+ if(line.length() >(unsigned) CONSTS::MAX_KEY_LENGHTH) {
+ continue;
+ }
+
+ while(data.next) {
+ rval = db.FindPrefix(line.c_str(), line.length(), data);
+ if(rval == MBError::SUCCESS) nfound++;
+ }
+ data.Clear();
+ count++;
+ if(count % 1000000 == 0) std::cout << "Looked up " << count << "\n";
+ }
+ gettimeofday(&stop, NULL);
+ in.close();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " found " << nfound <<
+ " time: " << 1.0*tmdiff/count << "\n";
+ assert(expected_count == nfound);
+}
+
+static void longest_prefix_lookup_test(std::string &list_file, MBConfig &mbconf,
+ int64_t expected_count)
+{
+ std::cout << "Lookup test: " << list_file << "\n";
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+ std::ifstream in(list_file.c_str());
+ assert(in.is_open());
+ int count = 0;
+ std::string line;
+ int rval;
+ MBData data;
+ struct timeval start, stop;
+ int nfound = 0;
+ gettimeofday(&start, NULL);
+ while(std::getline(in, line)) {
+ if(line.length() > (unsigned) CONSTS::MAX_KEY_LENGHTH) {
+ continue;
+ }
+ data.Clear();
+
+ rval = db.FindLongestPrefix(line.c_str(), line.length(), data);
+ if(rval == MBError::SUCCESS) {
+ nfound++;
+ }
+ count++;
+ if(count % 1000000 == 0) std::cout << "Looked up " << count << "\n";
+ }
+ gettimeofday(&stop, NULL);
+ in.close();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " found " << nfound <<
+ " time: " << 1.0*tmdiff/count << "\n";
+ assert(expected_count == nfound);
+}
+
+static void delete_odd_test(std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ std::cout << "deletion odd-entry test: " << list_file << "\n";
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+ std::ifstream in(list_file.c_str());
+ assert(in.is_open());
+ int count = 0;
+ int nfound = 0;
+ std::string line;
+ int rval;
+ struct timeval start, stop;
+
+ gettimeofday(&start, NULL);
+ while(std::getline(in, line)) {
+ if(line.length() > (unsigned) CONSTS::MAX_KEY_LENGHTH) {
+ continue;
+ }
+ count++;
+ if(count % 2 == 0)
+ {
+ rval = db.Remove(line);
+ if(rval == MBError::SUCCESS) {
+ nfound++;
+ } else {
+ std::cerr << "failed to remove " << line << ": " <<
+ MBError::get_error_str(rval) << "\n";
+ }
+ if(count % 1000000 == 0) std::cout << "deleted " << count << "\n";
+ }
+ }
+ gettimeofday(&stop, NULL);
+ in.close();
+ db.PrintStats();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " deleted " << nfound <<
+ " time: " << 1.0*tmdiff/count << "\n";
+ assert(expected_count == nfound);
+}
+
+static void delete_test(std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ std::cout << "deletion test: " << list_file << "\n";
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+ std::ifstream in(list_file.c_str());
+ assert(in.is_open());
+ int count = 0;
+ int nfound = 0;
+ std::string line;
+ int rval;
+ struct timeval start, stop;
+ int64_t db_count = db.Count();
+
+ gettimeofday(&start, NULL);
+ while(std::getline(in, line)) {
+ if(line.length() > (unsigned) CONSTS::MAX_KEY_LENGHTH) {
+ continue;
+ }
+ rval = db.Remove(line);
+ if(rval == MBError::SUCCESS) {
+ nfound++;
+ db_count--;
+ assert(db_count == db.Count());
+ } else if(debug) {
+ std::cerr << "failed to remove " << line << ": " <<
+ MBError::get_error_str(rval) << "\n";
+ }
+ count++;
+ if(count % 1000000 == 0) std::cout << "deleted " << count << "\n";
+ }
+ gettimeofday(&stop, NULL);
+ in.close();
+ db.PrintStats();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " deleted " << nfound <<
+ " time: " << 1.0*tmdiff/count << "\n";
+ assert(expected_count == nfound);
+}
+
+static void shrink_test(std::string &list_file, MBConfig &mbconf,
+ int64_t expected_count)
+{
+ std::cout << "shrink test: " << list_file << "\n";
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+ struct timeval start, stop;
+
+ gettimeofday(&start, NULL);
+ int rval = db.CollectResource(1024, 1024, 0xFFFFFFFFFFFFF, 0xFFFFFFFFFFFF);
+ gettimeofday(&stop, NULL);
+ assert(rval == MBError::SUCCESS);
+
+ db.PrintStats();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "shrink time: " << tmdiff/1000000. << "\n";
+}
+
+static void iterator_test(MBConfig &mbconf, int64_t expected_count)
+{
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+
+ int count = 0;
+ struct timeval start, stop;
+ gettimeofday(&start, NULL);
+ for(DB::iterator iter = db.begin(); iter != db.end(); ++iter) {
+ count++;
+ if(count % 1000000 == 0) {
+ std::cout << "iterated " << count << "\n";
+ }
+ }
+ gettimeofday(&stop, NULL);
+ db.PrintStats();
+ int64_t db_cnt = db.Count();
+ db.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " time: " << 1.0*tmdiff/count << "\n";
+ assert(count == db_cnt);
+ assert(expected_count == count);
+}
+
+static void prune_test(MBConfig &mbconf, int64_t expected_count)
+{
+ DB db(mbconf);
+ assert(db.is_open());
+
+ int count = 0;
+ struct timeval start, stop;
+ gettimeofday(&start, NULL);
+ // Delete one-third of the DB
+ for(DB::iterator iter = db.begin(); iter != db.end(); ++iter) {
+ count++;
+ if(count % 3 == 0) {
+ db.Remove(iter.key);
+ }
+ if(count % 1000000 == 0) {
+ std::cout << "iterated " << count << "\n";
+ }
+ }
+
+ int64_t db_cnt = db.Count();
+ db.PrintStats();
+ assert(db.Close() == MBError::SUCCESS);
+
+ gettimeofday(&stop, NULL);
+ assert(expected_count == db_cnt);
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " time: " << 1.0*tmdiff/count << "\n";
+}
+
+static void async_eviction_test(std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ // Set up writer
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW |
+ CONSTS::ASYNC_WRITER_MODE;
+ // Use a small size so that we can test eviction
+ DB db_async(mbconf);
+ assert(db_async.is_open());
+
+ mbconf.options = CONSTS::ACCESS_MODE_READER | CONSTS::USE_SLIDING_WINDOW;
+ DB db(mbconf);
+ assert(db_async.is_open());
+ assert(db.SetAsyncWriterPtr(&db_async) == MBError::SUCCESS);
+ assert(db.AsyncWriterEnabled());
+
+ std::string line;
+ std::ifstream in(list_file.c_str());
+ assert(in.is_open());
+ struct timeval start, stop;
+ int64_t count = 0;
+ gettimeofday(&start, NULL);
+ int check_eviction_count = 3333;
+ if(expected_count > 1000000) {
+ check_eviction_count = 3333333;
+ } else if(expected_count > 100000) {
+ check_eviction_count = 333333;
+ } else if(expected_count > 10000) {
+ check_eviction_count = 33333;
+ }
+ while(std::getline(in, line)) {
+ if(line.length() > (unsigned) CONSTS::MAX_KEY_LENGHTH) {
+ continue;
+ }
+
+ assert(db.Add(line, line) == MBError::SUCCESS);
+ count++;
+
+ if(count % 1000000 == 0) {
+ std::cout << "added " << count << "\n";
+ }
+ if(count % check_eviction_count == 0) {
+ // Prune by total count
+ assert(db.CollectResource(0xFFFFFFFFFFFF, 0xFFFFFFFFFFFF, 0xFFFFFFFFFFFF, 100) == MBError::SUCCESS);
+ }
+ }
+ gettimeofday(&stop, NULL);
+ in.close();
+
+ assert(db.UnsetAsyncWriterPtr(&db_async) == MBError::SUCCESS);
+ db.Close();
+
+ while(db_async.AsyncWriterBusy()) {
+ usleep(50);
+ }
+ db_async.PrintStats();
+ //TODOOOassert(db_async.Count() == expected_count);
+ db_async.Close();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " time: " << 1.0*tmdiff/expected_count << "\n";
+}
+
+static void backup_test(std::string &backup_dir, std::string &list_file, MBConfig &mbconf, int64_t expected_count)
+{
+ DB db(mbconf);
+ assert(db.Status() == MBError::SUCCESS);
+
+ MBData data;
+ int count = 0;
+ struct timeval start, stop;
+ gettimeofday(&start, NULL);
+
+ DB::iterator iter = db.begin();
+ std::string first_key = iter.key;
+
+ // Create Backup DB directory
+ std::string backup_mkdir_cmd = "mkdir " + backup_dir;
+ if(system(backup_mkdir_cmd.c_str()) != 0) {
+ }
+
+ // backup DB
+ assert(db.Backup(backup_dir.c_str()) == MBError::SUCCESS);
+
+ // Remove key to verify existence later in backed up DB
+ assert(db.Remove(first_key) == MBError::SUCCESS);
+ assert(db.Find(first_key, data) != MBError::SUCCESS);
+
+ // Open Backed up DB
+ MBConfig mbconf_bak;
+ memset(&mbconf_bak, 0, sizeof(mbconf_bak));
+ mbconf_bak.mbdir = backup_dir.c_str();
+ mbconf_bak.block_size_index = BLOCK_SIZE_ALIGN;
+ mbconf_bak.block_size_data = 2*BLOCK_SIZE_ALIGN;
+ mbconf_bak.num_entry_per_bucket = 512;
+ DB db_bak(mbconf_bak);
+
+ // Verify backed up DB has the first_key
+ assert(db_bak.Find(first_key, data) == MBError::SUCCESS);
+
+ // Now add removed key back
+ assert(db.Add(first_key.c_str(), first_key.length(), data, true) == MBError::SUCCESS);
+
+ gettimeofday(&stop, NULL);
+ count = db.Count();
+ int64_t tmdiff = (stop.tv_sec-start.tv_sec)*1000000 + (stop.tv_usec-start.tv_usec);
+ std::cout << "count: " << count << " time: " << 1.0*tmdiff/count << "\n";
+ assert(db.Count() == expected_count);
+ assert(db_bak.Count() == expected_count);
+ db_bak.Close();
+ db.Close();
+}
+
+
+static void SetTestStatus(bool success)
+{
+ std::string cmd;
+ if(success) {
+ cmd = std::string("touch ") + MB_DIR + "/_success";
+ } else {
+ cmd = std::string("rm ") + MB_DIR + "/_success >" + MB_DIR + "/out 2>" + MB_DIR + "/err";
+ }
+ if(system(cmd.c_str()) != 0) {
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ if(argc > 1) {
+ MB_DIR = argv[1];
+ }
+ clean_db_dir();
+ SetTestStatus(false);
+
+ std::string test_list_file = "./test_list";
+ if(argc > 2) {
+ test_list_file = argv[2];
+ }
+
+ std::string backup_dir = std::string(MB_DIR) + "/backup/";
+ if(argc > 3) {
+ backup_dir = argv[3];
+ }
+
+ DB::SetLogFile(std::string(MB_DIR) + "/mabain.log");
+ std::ifstream test_in(test_list_file);
+ assert(test_in.is_open());
+ std::string file;
+ std::string mode;
+ std::string remove;
+ int64_t memcap;
+ int64_t expected_count;
+ bool remove_db;
+ uint32_t db_id = 1;
+ MBConfig mbconf;
+ memset(&mbconf, 0, sizeof(mbconf));
+
+ mbconf.mbdir = MB_DIR;
+ mbconf.block_size_index = BLOCK_SIZE_ALIGN;
+ mbconf.block_size_data = 2*BLOCK_SIZE_ALIGN;
+ mbconf.num_entry_per_bucket = 512;
+ while(test_in >> file >> mode >> memcap >> remove >> expected_count) {
+ if(file[0] == '#') {
+ std::cout << file << "\n";
+ continue;
+ }
+ std::cout << "============================================\n";
+ std::cout << file << " " << mode << " " << memcap << " " << remove << " " << expected_count << "\n";
+ mbconf.connect_id = db_id;
+ mbconf.memcap_index = memcap;
+ mbconf.memcap_data = memcap;
+
+ if(remove.compare("remove") == 0) {
+ remove_db = true;
+ } else {
+ remove_db = false;
+ }
+ if(mode.compare("load") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW;
+ load_test(file, mbconf, expected_count);
+ } else if(mode.compare("update") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW;
+ update_test(file, mbconf, expected_count);
+ } else if(mode.compare("lookup") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_READER | CONSTS::USE_SLIDING_WINDOW;
+ lookup_test(file, mbconf, expected_count);
+ } else if(mode.compare("prefix") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_READER | CONSTS::USE_SLIDING_WINDOW;
+ prefix_lookup_test(file, mbconf, expected_count);
+ } else if(mode.compare("longest_prefix") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_READER;
+ longest_prefix_lookup_test(file, mbconf, expected_count);
+ } else if(mode.compare("iterator") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_READER;
+ iterator_test(mbconf, expected_count);
+ } else if(mode.compare("delete") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER;
+ delete_test(file, mbconf, expected_count);
+ } else if(mode.compare("delete_odd") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW;
+ delete_odd_test(file, mbconf, expected_count);
+ } else if(mode.compare("shrink") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW;
+ shrink_test(file, mbconf, expected_count);
+ } else if(mode.compare("prune") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW;
+ prune_test(mbconf, expected_count);
+ } else if(mode.compare("eviction") == 0) {
+ async_eviction_test(file, mbconf, expected_count);
+ } else if(mode.compare("backup") == 0) {
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER;
+ backup_test(backup_dir, file, mbconf, expected_count);
+ } else {
+ std::cerr << "Unknown test\n";
+ abort();
+ }
+
+ if(remove_db) {
+ clean_db_dir();
+ ResourcePool::getInstance().RemoveAll(); // need to remove resouce for next run
+ mbconf.block_size_index += BLOCK_SIZE_ALIGN;
+ mbconf.block_size_data += BLOCK_SIZE_ALIGN;
+ }
+ db_id++;
+ }
+
+ test_in.close();
+ DB::CloseLogFile();
+ SetTestStatus(true);
+ return 0;
+}
--- /dev/null
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <assert.h>
+#include <atomic>
+#include <fstream>
+
+#include "../db.h"
+
+#include "./test_key.h"
+
+using namespace mabain;
+
+static std::atomic<int64_t> key_low;
+static std::atomic<int64_t> key_high;
+static int64_t memcap_i = 256*1024*1024;
+static int64_t memcap_d = 256*1024*1024;
+static pthread_t wid = 0;
+
+static std::string mbdir = "/var/tmp/mabain_test";
+static bool stop_processing = false;
+static uint32_t run_time = 3600;
+
+static void* run_mb_test(void *arg);
+
+static void* Reader(void *arg)
+{
+ DB db(mbdir.c_str(), CONSTS::ReaderOptions(), memcap_i, memcap_d);
+ if(!db.is_open()) {
+ std::cerr << "failed tp open db\n";
+ abort();
+ }
+
+ int64_t ikey;
+ int rval;
+ TestKey tkey_int(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey_sha1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ TestKey tkey_sha2(MABAIN_TEST_KEY_TYPE_SHA_256);
+ std::string keystr;
+ MBData mbd;
+ int ktype;
+
+ while(!stop_processing) {
+ //usleep(5);
+
+ if(key_high.load(std::memory_order_consume) == key_low.load(std::memory_order_consume))
+ continue;
+ ikey = key_low.load(std::memory_order_consume);
+ ikey += rand() % (key_high.load(std::memory_order_consume) - key_low.load(std::memory_order_consume));
+
+ ktype = rand() % 3;
+ switch(ktype) {
+ case 0:
+ keystr = tkey_int.get_key(ikey);
+ break;
+ case 1:
+ keystr = tkey_sha1.get_key(ikey);
+ break;
+ case 2:
+ keystr = tkey_sha2.get_key(ikey);
+ break;
+ }
+ rval = db.Find(keystr, mbd);
+
+ if(ikey < key_low.load(std::memory_order_consume))
+ continue;
+
+ if(rval == MBError::SUCCESS) {
+ if(keystr != std::string((char *)mbd.buff, mbd.data_len)) {
+ std::cout << "value not match for key " << ikey << ": " << keystr << "\n";
+ abort();
+ }
+ } else if(rval != MBError::NOT_EXIST) {
+ std::cout << "unexpected return from Find: " << rval << "\n";
+ abort();
+ }
+ }
+
+ db.Close();
+ return NULL;
+}
+
+static void Verify(DB &db)
+{
+ int64_t key0 = key_low.load(std::memory_order_consume);
+ int64_t key1 = key_high.load(std::memory_order_consume);
+ TestKey tkey_int(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey_sha1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ TestKey tkey_sha2(MABAIN_TEST_KEY_TYPE_SHA_256);
+ MBData mbd;
+ std::string keystr;
+ for(int64_t i = key0; i < key1; i++) {
+ keystr = tkey_int.get_key(i);
+ if(db.Find(keystr, mbd) != MBError::SUCCESS) {
+ std::cout << "failed to find key " << i << ": " << keystr << std::endl;
+ abort();
+ }
+ assert(keystr == std::string((char *)mbd.buff, mbd.data_len));
+
+ keystr = tkey_sha1.get_key(i);
+ if(db.Find(keystr, mbd) != MBError::SUCCESS) {
+ std::cout << "failed to find key " << i << ": " << keystr << std::endl;
+ abort();
+ }
+ assert(keystr == std::string((char *)mbd.buff, mbd.data_len));
+
+ keystr = tkey_sha2.get_key(i);
+ if(db.Find(keystr, mbd) != MBError::SUCCESS) {
+ std::cout << "failed to find key " << i << ": " << keystr << std::endl;
+ abort();
+ }
+ assert(keystr == std::string((char *)mbd.buff, mbd.data_len));
+ }
+}
+
+static void* AddThread(void *arg)
+{
+ DB db(mbdir.c_str(), CONSTS::ReaderOptions());
+ int num = rand() % 1500;
+
+ int64_t key;
+ TestKey tkey_int(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey_sha1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ TestKey tkey_sha2(MABAIN_TEST_KEY_TYPE_SHA_256);
+ std::string keystr;
+
+ for(int i = 0; i < num; i++) {
+ key = key_high.fetch_add(1, std::memory_order_release);
+ keystr = tkey_int.get_key(key);
+ assert(db.Add(keystr, keystr) == MBError::SUCCESS);
+
+ keystr = tkey_sha1.get_key(key);
+ assert(db.Add(keystr, keystr) == MBError::SUCCESS);
+
+ keystr = tkey_sha2.get_key(key);
+ assert(db.Add(keystr, keystr) == MBError::SUCCESS);
+ }
+ db.Close();
+ return NULL;
+}
+
+static void Populate(int nt)
+{
+ pthread_t tid[256];
+ assert(nt < 256);
+ for(int i = 0; i < nt; i++) {
+ if(pthread_create(&tid[i], NULL, AddThread, NULL) != 0) {
+ std::cout << "failed to create MultiThreadAdd thread\n";
+ abort();
+ }
+ }
+
+ for(int i = 0; i < nt; i++) {
+ if(pthread_join(tid[i], NULL) != 0) {
+ std::cout << "failed to join MultiThreadAdd thread\n";
+ abort();
+ }
+ }
+}
+
+static void load_key_ids()
+{
+ int64_t klow, khigh;
+ std::ifstream ifs;
+ std::string path = mbdir + "/key_id";
+ ifs.open (path.c_str(), std::ofstream::in);
+ if(ifs.is_open()) {
+ ifs >> klow;
+ ifs >> khigh;
+ ifs.close();
+ } else {
+ klow = 0;
+ khigh = 0;
+ }
+ key_low = klow;
+ key_high = khigh;
+ std::cout << "Loaded " << key_low << " " << key_high << "\n";
+}
+
+static void store_key_ids()
+{
+ std::string path = mbdir + "/key_id";
+ FILE *fp;
+ fp = fopen(path.c_str(), "w");
+ if(!fp) return;
+ fprintf(fp, "%d\n%d", (int)key_low.load(std::memory_order_consume),
+ (int)key_high.load(std::memory_order_consume));
+ fclose(fp);
+}
+
+static void* DeleteThread(void *arg)
+{
+ DB db(mbdir.c_str(), CONSTS::ReaderOptions());
+ int num = rand() % 5;
+
+ int64_t key;
+ TestKey tkey_int(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey_sha1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ TestKey tkey_sha2(MABAIN_TEST_KEY_TYPE_SHA_256);
+ std::string keystr;
+
+ for(int i = 0; i < num; i++) {
+ key = key_low.fetch_add(1, std::memory_order_release);
+
+ keystr = tkey_int.get_key(key);
+ assert(db.Remove(keystr) == MBError::SUCCESS);
+
+ keystr = tkey_sha1.get_key(key);
+ assert(db.Remove(keystr) == MBError::SUCCESS);
+
+ keystr = tkey_sha2.get_key(key);
+ assert(db.Remove(keystr) == MBError::SUCCESS);
+ }
+
+ db.Close();
+ return NULL;
+}
+
+static void Prune(int nt)
+{
+ pthread_t tid[256];
+ assert(nt < 256);
+ for(int i = 0; i < nt; i++) {
+ if(pthread_create(&tid[i], NULL, DeleteThread, NULL) != 0) {
+ std::cout << "failed to create MultiThreadAdd thread\n";
+ abort();
+ }
+ }
+
+ for(int i = 0; i < nt; i++) {
+ if(pthread_join(tid[i], NULL) != 0) {
+ std::cout << "failed to join MultiThreadAdd thread\n";
+ abort();
+ }
+ }
+}
+
+void stop_mb_test()
+{
+ if(wid != 0) {
+ if(pthread_join(wid, NULL) != 0) {
+ std::cout << "cannot join mbtest thread\n";
+ }
+ }
+}
+
+static void CheckCount()
+{
+ DB *db = new DB(mbdir.c_str(), CONSTS::ReaderOptions(), memcap_i, memcap_d);
+ if(db == NULL) return;
+
+ int64_t count = 0;
+ for(DB::iterator iter = db->begin(false, false); iter != db->end(); ++iter) {
+ count++;
+ }
+ std::cout << "Count using iterator: " << count << "\tcount from API: "
+ << db->Count() << "\n";
+ delete db;
+}
+
+void start_mb_test()
+{
+ if(pthread_create(&wid, NULL, run_mb_test, NULL) != 0) {
+ std::cout << "failed to create test thread" << std::endl;
+ }
+}
+
+static void* run_mb_test(void *arg)
+{
+ int64_t run_stop_time = time(NULL) + run_time;
+ int nreaders = 4;
+ int nupdates = 4;
+ srand(time(NULL));
+
+ load_key_ids();
+
+ MBConfig mbconf;
+ int options;
+ memset(&mbconf, 0, sizeof(mbconf));
+ mbconf.mbdir = mbdir.c_str();
+ options = CONSTS::WriterOptions() | CONSTS::ASYNC_WRITER_MODE;
+ mbconf.options = options;
+ mbconf.memcap_index = 128ULL*1024*1024;
+ mbconf.memcap_data = 128ULL*1024*1024;
+ mbconf.block_size_index = 64U*1024*1024;
+ mbconf.block_size_data = 64U*1024*1024;
+ mbconf.max_num_data_block = 3;
+ mbconf.max_num_index_block = 3;
+ mbconf.num_entry_per_bucket = 500;
+ DB *db = new DB(mbconf);
+ if(!db->is_open()) {
+ std::cerr << "failed to open writer db" << std::endl;
+ delete db;
+ abort();
+ }
+
+ pthread_t tid;
+ for(int i = 0; i < nreaders; i++) {
+ if(pthread_create(&tid, NULL, Reader, NULL) != 0) {
+ std::cout << "failed to create reader thread" << std::endl;
+ abort();
+ }
+ }
+
+ int rcn = 0;
+ int64_t loop_cnt = 0;
+ while(!stop_processing) {
+ Populate(nupdates);
+ Prune(nupdates);
+
+ std::cout << "LOOP " << loop_cnt << ": " << db->Count() << "\n";
+ if(loop_cnt % 5 == 0) {
+ std::cout << "RUN RC\n";
+ int rval = db->CollectResource(24LL*1024*1024, 24LL*1024*1024, 128LL*1024*1024, 0xFFFFFFFFFFFF);
+ if(rval == MBError::SUCCESS) {
+ rcn++;
+ if(rcn % 57 == 0 && !(options & CONSTS::ASYNC_WRITER_MODE)) {
+ // Note if in async mode, the DB handle cannot be used for lookup.
+ std::cout << "Verifying after rc" << std::endl;
+ Verify(*db);
+ } else if(rcn % 20 == 0) {
+ db->Close();
+ delete db;
+ mbconf.options = options;
+ db = new DB(mbconf);
+ if(!db->is_open()) {
+ delete db;
+ abort();
+ }
+ CheckCount();
+ } else if(rcn % 20000523 == 0) {
+ if(system("reboot") != 0) {
+ }
+ }
+ }
+ }
+
+ sleep(1);
+ loop_cnt++;
+ if(time(NULL) >= run_stop_time) {
+ stop_processing = true;
+ }
+ }
+
+ db->Close();
+ delete db;
+ store_key_ids();
+
+
+ return NULL;
+}
+
+static void SetTestStatus(bool success)
+{
+ std::string cmd;
+ if(success) {
+ cmd = std::string("touch ") + mbdir + "/_success";
+ } else {
+ cmd = std::string("rm ") + mbdir + "/_success >" + mbdir + "/out 2>" + mbdir + "/err";
+ }
+ if(system(cmd.c_str()) != 0) {
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ if(argc > 1) {
+ mbdir = std::string(argv[1]);
+ std::cout << "Test db directory is " << mbdir << "\n";
+ }
+ if(argc > 2) {
+ run_time = atoi(argv[2]);
+ std::cout << "running " << argv[0] << " for " << run_time << " seconds...\n";
+ }
+
+ DB::SetLogFile(mbdir + "/mabain.log");
+ SetTestStatus(false);
+ run_mb_test(NULL);
+ SetTestStatus(true);
+ DB::CloseLogFile();
+ return 0;
+}
--- /dev/null
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <assert.h>
+#include <atomic>
+#include <fstream>
+#include <cstring>
+
+#include "../db.h"
+
+#include "./test_key.h"
+
+using namespace mabain;
+
+static const char *mbdir = "/var/tmp/mabain_test/";
+static pthread_t wid = 0;
+static MBConfig mbconf;
+static bool stop_processing = false;
+static uint32_t run_time = 3600;
+
+static void* run_mb_test(void *arg);
+
+static void* TestThread(void *arg)
+{
+ DB *dbw = (DB *) arg;
+ mbconf.options = CONSTS::ReaderOptions();
+ DB db(mbconf);
+ if(!db.is_open()) {
+ std::cerr << "failed tp open db\n";
+ abort();
+ }
+ assert(db.SetAsyncWriterPtr(dbw) == MBError::SUCCESS);
+ assert(db.AsyncWriterEnabled());
+
+
+ int64_t ikey = 0;
+ int rval;
+ TestKey tkey_int(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey_sha1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ TestKey tkey_sha2(MABAIN_TEST_KEY_TYPE_SHA_256);
+ std::string keystr;
+ MBData mbd;
+
+ while(!stop_processing) {
+ keystr = tkey_int.get_key(ikey);
+ rval = db.Find(keystr, mbd);
+ if(rval != MBError::SUCCESS) {
+ db.Add(keystr, keystr);
+ }
+ keystr = tkey_sha1.get_key(ikey);
+ rval = db.Find(keystr, mbd);
+ if(rval != MBError::SUCCESS) {
+ db.Add(keystr, keystr);
+ }
+ keystr = tkey_sha2.get_key(ikey);
+ rval = db.Find(keystr, mbd);
+ if(rval != MBError::SUCCESS) {
+ db.Add(keystr, keystr);
+ }
+
+ ikey++;
+ usleep(1);
+ }
+
+ assert(db.UnsetAsyncWriterPtr(dbw) == MBError::SUCCESS);
+ db.Close();
+ return NULL;
+}
+
+void stop_mb_test()
+{
+ if(wid != 0) {
+ if(pthread_join(wid, NULL) != 0) {
+ std::cout << "cannot join mbtest thread\n";
+ }
+ }
+}
+
+void start_mb_test()
+{
+ if(pthread_create(&wid, NULL, run_mb_test, NULL) != 0) {
+ std::cout << "failed to create test thread" << std::endl;
+ }
+}
+
+static void* run_mb_test(void *arg)
+{
+ int64_t run_stop_time = time(NULL) + run_time;
+ int nreaders = 8;
+ srand(time(NULL));
+
+ DB::SetLogFile("/var/tmp/mabain_test/mabain.log");
+ memset(&mbconf, 0, sizeof(mbconf));
+ mbconf.mbdir = mbdir;
+ mbconf.block_size_index = 32*1024*1024;
+ mbconf.block_size_data = 32*1024*1024;
+ mbconf.memcap_index = 128LL*1024*1024;
+ mbconf.memcap_data = 128LL*1024*1024;
+
+ int options = CONSTS::WriterOptions() | CONSTS::ASYNC_WRITER_MODE;
+ mbconf.options = options;
+ DB *db = new DB(mbconf);
+ if(!db->is_open()) {
+ std::cerr << "failed to open writer db" << std::endl;
+ delete db;
+ abort();
+ }
+
+ pthread_t tid[256];
+ assert(nreaders < 256);
+ for(int i = 0; i < nreaders; i++) {
+ if(pthread_create(&tid[i], NULL, TestThread, db) != 0) {
+ std::cout << "failed to create reader thread" << std::endl;
+ abort();
+ }
+ }
+
+ int sleep_time;
+ while(!stop_processing) {
+ std::cout << "Running rc... " << db->Count() <<"\n";
+ db->CollectResource(8*1024*1024LL, 8*1024*1024LL, 1000000000000, 3000000);
+
+ sleep_time = (rand() % 10) + 1;
+ sleep(sleep_time);
+
+ if(time(NULL) >= run_stop_time) {
+ stop_processing = true;
+ }
+ }
+
+ for(int i = 0; i < nreaders; i++) {
+ pthread_join(tid[i], NULL);
+ }
+ db->Close();
+ delete db;
+ DB::CloseLogFile();
+ return NULL;
+}
+
+static void SetTestStatus(bool success)
+{
+ std::string cmd;
+ if(success) {
+ cmd = std::string("touch ") + mbdir + "/_success";
+ } else {
+ cmd = std::string("rm ") + mbdir + "/_success >" + mbdir + "/out 2>" + mbdir + "/err";
+ }
+ if(system(cmd.c_str()) != 0) {
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ if(argc > 1) {
+ mbdir = argv[1];
+ std::cout << "Test db directory is " << mbdir << "\n";
+ }
+ if(argc > 2) {
+ run_time = atoi(argv[2]);
+ std::cout << "running " << argv[0] << " for " << run_time << " seconds...\n";
+ }
+
+ DB::SetLogFile(std::string(mbdir) + "/mabain.log");
+ SetTestStatus(false);
+ run_mb_test(NULL);
+ DB::CloseLogFile();
+ SetTestStatus(true);
+ return 0;
+}
--- /dev/null
+#include <string.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include "../db.h"
+#include "./test_key.h"
+
+const char *db_dir = "/var/tmp/mabain_test";
+
+using namespace mabain;
+
+int main(int argc, char *argv[])
+{
+ int num = 0;
+ int options = CONSTS::ReaderOptions();
+ int duration = 0; // in seconds
+ int n0 = 0;
+ int key_type = MABAIN_TEST_KEY_TYPE_INT;
+ for(int i = 1; i < argc; i++) {
+ if(strcmp(argv[i], "-w") == 0) {
+ options |= CONSTS::WriterOptions() | CONSTS::ASYNC_WRITER_MODE;
+ } else if(strcmp(argv[i], "-d") == 0) {
+ if(++i >= argc) abort();
+ db_dir = argv[i];
+ } else if(strcmp(argv[i], "-n") == 0) {
+ if(++i >= argc) abort();
+ num = atoi(argv[i]);
+ } else if(strcmp(argv[i], "-n0") == 0) {
+ if(++i >= argc) abort();
+ n0 = atoi(argv[i]);
+ } else if(strcmp(argv[i], "-t") == 0) {
+ if(++i >= argc) abort();
+ duration = atoi(argv[i]);
+ } else if(strcmp(argv[i], "-k") == 0) {
+ if(++i >= argc) abort();
+ if(strcmp(argv[i], "int") == 0) {
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ } else if(strcmp(argv[i], "sha1") == 0) {
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+ } else {
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_256;
+ }
+ } else {
+ std::cout << "unknown argument " << argv[i] << "\n";
+ }
+ }
+
+ DB::SetLogFile(std::string(db_dir) + "/mabain.log");
+ DB *db = new DB(db_dir, options);
+ TestKey tkey(key_type);
+ assert(db->is_open());
+
+ for(int i = 0; i < num; i++) {
+ std::string kv = tkey.get_key(n0 + i);
+ int rval = db->Add(kv, kv);
+ if(rval != MBError::SUCCESS) {
+ std::cout << "failed to add " << kv << " :" << MBError::get_error_str(rval) << std::endl;
+ }
+ }
+
+ if(duration > 0) {
+ uint32_t tm_stop = time(NULL) + duration;
+ int tm_diff = tm_stop - time(NULL);
+ while(tm_diff > 0) {
+ sleep(1);
+ tm_diff = tm_stop - time(NULL);
+ }
+ std::cout << "async writer exited\n";
+ }
+
+ db->Close();
+ DB::CloseLogFile();
+ return 0;
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <openssl/sha.h>
+
+#define MABAIN_TEST_KEY_TYPE_INT 0
+#define MABAIN_TEST_KEY_TYPE_SHA_128 1
+#define MABAIN_TEST_KEY_TYPE_SHA_256 2
+#define MABAIN_TEST_KEY_BUFF_SIZE 1024
+
+namespace {
+
+class TestKey
+{
+public:
+ TestKey(int ktype) {
+ key_type = ktype;
+ key_buff[0] = '\0';
+ }
+ ~TestKey() {
+ }
+ const char* get_key(int key) {
+ switch(key_type) {
+ case MABAIN_TEST_KEY_TYPE_INT:
+ snprintf(key_buff, MABAIN_TEST_KEY_BUFF_SIZE, "%d", key);
+ break;
+ case MABAIN_TEST_KEY_TYPE_SHA_128:
+ {
+ unsigned char hash[SHA_DIGEST_LENGTH];
+ SHA_CTX sha1;
+ SHA1_Init(&sha1);
+ SHA1_Update(&sha1, (unsigned char*)&key, 4);
+ SHA1_Final(hash, &sha1);
+ int i = 0;
+ for(i = 0; i < SHA_DIGEST_LENGTH; i++)
+ {
+ sprintf(key_buff + (i * 2), "%02x", hash[i]);
+ }
+ key_buff[32] = 0;
+ }
+ break;
+ case MABAIN_TEST_KEY_TYPE_SHA_256:
+ {
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ SHA256_CTX sha256;
+ SHA256_Init(&sha256);
+ SHA256_Update(&sha256, (unsigned char*)&key, 4);
+ SHA256_Final(hash, &sha256);
+ int i = 0;
+ for(i = 0; i < SHA256_DIGEST_LENGTH; i++)
+ {
+ sprintf(key_buff + (i * 2), "%02x", hash[i]);
+ }
+ key_buff[64] = 0;
+ }
+ break;
+ default:
+ abort();
+ }
+ return (const char *) key_buff;
+ }
+
+private:
+ int key_type;
+ char key_buff[MABAIN_TEST_KEY_BUFF_SIZE];
+};
+
+}
--- /dev/null
+./key_list_1 load 1000000 keep 49998
+./key_list_1 update 1000000 keep 49998
+./key_list_1 iterator 1000000 keep 49998
+./key_list_1 lookup 1000 keep 49998
+./key_list_1 backup 1000000 keep 49998
+./key_list_1 delete 0 keep 49998
+./key_list_1 lookup 0 keep 0
+./key_list_1 eviction 0 remove 48462
+./key_list_2 load 102400000 keep 999998
+./key_list_2 delete 5000000 keep 999998
+./key_list_2 update 5000000 keep 999998
+./key_list_2 lookup 102400000 remove 999998
+./key_list_2 eviction 102400000 remove 951358
+./key_list_3 load 1000000 keep 1000
+./key_list_3 iterator 1000000 keep 1000
+./key_list_3 longest_prefix 100000 keep 1000
+./key_list_3 delete 0 keep 1000
+./key_list_3 lookup 0 remove 0
+./key_list_3 update 0 remove 1000
+./key_list_4 load 400000000 keep 9999939
+./key_list_4 update 400000000 keep 9999939
+./key_list_4 iterator 0 keep 9999939
+./key_list_4 longest_prefix 100000000 keep 9999939
+./key_list_4 delete 400000000 keep 9999939
+./key_list_4 iterator 400000000 keep 0
+./key_list_4 longest_prefix 10000000 remove 0
+./key_list_5 load 0 keep 10000000
+./key_list_5 update 0 keep 10000000
+./key_list_5 lookup 0 keep 10000000
+./key_list_5 delete 0 keep 10000000
+./key_list_5 lookup 0 remove 0
+./key_list_5 eviction 250000000 remove 9033856
+./key_list_6 load 0 keep 35313837
+./key_list_6 longest_prefix 0 keep 35313837
+./key_list_6 delete 100000000 keep 35313837
+./key_list_6 longest_prefix 0 remove 0
+./key_list_4 load 0 keep 9999939
+./key_list_4 delete_odd 0 keep 4999969
+./key_list_4 shrink 0 keep 0
+./key_list_4 lookup 0 remove 4999970
+./key_list_4 eviction 750000000 remove 9508931
+./key_list_6 load 0 keep 35313837
+./key_list_6 delete_odd 0 keep 17656918
+./key_list_6 update 0 keep 35313837
+./key_list_6 delete_odd 0 keep 17656918
+./key_list_6 shrink 0 keep 0
+./key_list_6 lookup 0 remove 17656919
+./key_list_6 load 268435456 keep 35313837
+./key_list_6 prune 268435456 keep 23542558
+./key_list_6 longest_prefix 0 keep 24548132
+./key_list_6 delete 100000000 keep 23542558
+./key_list_6 eviction 750000000 keep 27396269
+./key_list_6 lookup 0 remove 17152173
--- /dev/null
+coverage/*
+test_mabain
--- /dev/null
+TEST_BIN_NAME=test_mabain
+TEST_COVRAGE_DIR=coverage
+
+CPP=g++
+
+CPPFLAGS = -I. -I.. -I../util -I/usr/include -Wall -Werror -g -O2 -std=c++11
+CPPFLAGS += -fprofile-arcs -ftest-coverage -fPIC -D__SHM_QUEUE__
+LDFLAGS = -L/usr/lib64 -lgtest -lcrypto -lpthread
+
+TESTSOURCES=$(wildcard *.cpp)
+
+all: unit-test
+
+build: $(TESTSOURCES) ../libmabain.so
+ $(CPP) $(CPPFLAGS) -o $(TEST_BIN_NAME) $(TESTSOURCES) -L.. -lmabain $(LDFLAGS)
+
+unit-test: clean build
+ mkdir -p $(TEST_COVRAGE_DIR)
+ - LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:../ ./$(TEST_BIN_NAME)
+ - gcovr -r . --xml > $(TEST_COVRAGE_DIR)/coverage.xml
+ - gcovr -r . --html --html-details -o $(TEST_COVRAGE_DIR)/coverage.html
+
+clean:
+ -rm -rf $(TEST_BIN_NAME)
+ -rm -rf *.gcda *.gcno *.tap
+ -rm -rf $(TEST_COVRAGE_DIR)
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <iostream>
+
+#include <gtest/gtest.h>
+
+#include "../dict.h"
+#include "../dict_mem.h"
+#include "./test_key.h"
+#include "../resource_pool.h"
+#include "../mb_rc.h"
+
+using namespace mabain;
+
+namespace {
+
+class AbnormalExitTest : public ::testing::Test
+{
+public:
+ AbnormalExitTest() {
+ remove_index = -1;
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ }
+ virtual ~AbnormalExitTest() {
+ }
+ virtual void SetUp() {
+ std::string db_dir = "/var/tmp/mabain_test";
+ std::string cmd = std::string("mkdir -p ") + db_dir;
+ if(system(cmd.c_str()) != 0) {
+ }
+ unlink((db_dir + "/_mabain_h").c_str());
+ unlink((db_dir + "/_dbfl").c_str());
+ unlink((db_dir + "/_ibfl").c_str());
+ db = new DB(db_dir.c_str(), CONSTS::ACCESS_MODE_WRITER);
+ if(!db->is_open()) {
+ std::cerr << "failed to open db: " << db_dir << " " << db->StatusStr() << "\n";
+ exit(0);
+ }
+ }
+ virtual void TearDown() {
+ if(db != NULL) {
+ db->Close();
+ delete db;
+ db = NULL;
+ }
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+ void Populate(int count) {
+ db->RemoveAll();
+ std::string key_str;
+ TestKey tkey(key_type);
+ for(int key = 1; key <= count; key++) {
+ key_str = tkey.get_key(key);
+ db->Add(key_str, key_str);
+ }
+ }
+
+ void SimulateAbnormalExit(int exception_type) {
+ Dict *dict = db->GetDictPtr();
+ IndexHeader *header = dict->GetHeaderPtr();
+ DictMem *dmm = dict->GetMM();
+ std::string key_str;
+ TestKey tkey(key_type);
+ int rval;
+
+ switch(exception_type) {
+ case EXCEP_STATUS_ADD_DATA_OFF:
+ key_str = tkey.get_key(1278);
+ db->Add(key_str, key_str + "_UPDATED", true);
+ break;
+ case EXCEP_STATUS_ADD_NODE:
+ key_str = "***abc1";
+ db->Add(key_str, key_str);
+ key_str = "***abd1";
+ db->Add(key_str, key_str);
+ key_str = "***abe1";
+ db->Add(key_str, key_str);
+ key_str = "***ab";
+ db->Add(key_str, key_str);
+ break;
+ case EXCEP_STATUS_CLEAR_EDGE:
+ key_str = tkey.get_key(remove_index);
+ rval = db->Remove(key_str);
+ assert(rval == MBError::SUCCESS);
+ break;
+ case EXCEP_STATUS_REMOVE_EDGE:
+ break;
+ }
+
+ header->excep_updating_status = exception_type;
+
+ // Writer random data to simulate the db inconsistency
+ srand(time(NULL));
+ uint8_t buffer[4*4];
+ int *ptr = (int *) buffer;
+ for(int i = 0; i < 4; i++) {
+ ptr[i] = (int) rand();
+ }
+
+ switch(exception_type) {
+ case EXCEP_STATUS_ADD_EDGE:
+ dmm->WriteData(buffer, EDGE_SIZE, header->excep_lf_offset);
+ break;
+ case EXCEP_STATUS_ADD_DATA_OFF:
+ dmm->WriteData(buffer, OFFSET_SIZE, header->excep_lf_offset+EDGE_NODE_LEADING_POS);
+ break;
+ case EXCEP_STATUS_ADD_NODE:
+ dmm->WriteData(buffer, NODE_EDGE_KEY_FIRST, header->excep_offset);
+ break;
+ case EXCEP_STATUS_REMOVE_EDGE:
+ // Currently we cannot simulate this exception.
+ //dmm->WriteData(buffer, OFFSET_SIZE, header->excep_lf_offset+EDGE_NODE_LEADING_POS);
+ break;
+ case EXCEP_STATUS_CLEAR_EDGE:
+ dmm->WriteData(buffer, EDGE_SIZE, header->excep_lf_offset);
+ break;
+ default:
+ break;
+ }
+ }
+
+ int RecoverDB() {
+ Dict *dict = db->GetDictPtr();
+ int rval = dict->ExceptionRecovery();
+ return rval;
+ }
+
+ int CheckDBConcistency(int count) {
+ DB db_r("/var/tmp/mabain_test", CONSTS::ACCESS_MODE_READER);
+ assert(db_r.is_open());
+ std::string key_str;
+ MBData mbd;
+ TestKey tkey(key_type);
+ int rval;
+ int failed_cnt = 0;
+ for(int key = 1; key <= count; key++) {
+ key_str = tkey.get_key(key);
+ rval = db_r.Find(key_str, mbd);
+
+ if(key == remove_index) continue;
+ if(rval != MBError::SUCCESS) {
+ failed_cnt++;
+ continue;
+ }
+ if(key_str != std::string((char*)mbd.buff, mbd.data_len)) {
+ // Value may be updated
+ if(key_str+"_UPDATED" != std::string((char*)mbd.buff, mbd.data_len)) {
+ failed_cnt++;
+ }
+ }
+ }
+
+ db_r.Close();
+ return failed_cnt;
+ }
+
+ int CheckHalfDBConsistency(int count, bool check_even) {
+ DB db_r("/var/tmp/mabain_test", CONSTS::ACCESS_MODE_READER);
+ assert(db_r.is_open());
+ std::string key_str;
+ MBData mbd;
+ TestKey tkey(key_type);
+ int rval;
+ int failed_cnt = 0;
+ for(int key = 1; key <= count; key++) {
+ key_str = tkey.get_key(key);
+ rval = db_r.Find(key_str, mbd);
+
+ if(check_even) {
+ if(key % 2 == 1) continue;
+ } else {
+ if(key % 2 == 0) continue;
+ }
+ if(rval != MBError::SUCCESS) {
+ failed_cnt++;
+ continue;
+ }
+ if(key_str != std::string((char*)mbd.buff, mbd.data_len)) {
+ // Value may be updated
+ if(key_str+"_UPDATED" != std::string((char*)mbd.buff, mbd.data_len)) {
+ failed_cnt++;
+ }
+ }
+ }
+
+ db_r.Close();
+ return failed_cnt;
+ }
+
+protected:
+ DB *db;
+ int remove_index;
+ int key_type;
+};
+
+TEST_F(AbnormalExitTest, KEY_TYPE_INT_test)
+{
+ int count = 32331;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_ADD_EDGE);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_SHA1_test)
+{
+ int count = 18293;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_ADD_EDGE);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_SHA256_test)
+{
+ int count = 5293;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_256;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_ADD_EDGE);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, 0);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_SHA1_ADD_DATA_OFF_test)
+{
+ int count = 18293;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_ADD_DATA_OFF);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_INT_ADD_NODE_test)
+{
+ int count = 1829;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_ADD_NODE);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_INT_REMOVE_test)
+{
+ int count = 23234;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ remove_index = 2345;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_CLEAR_EDGE);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, 0);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_INT_REMOVE_test_1)
+{
+ int count = 23234;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ remove_index = 1;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_CLEAR_EDGE);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_SHA_128_REMOVE_test)
+{
+ int count = 3934;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ remove_index = 1021;
+
+ Populate(count);
+ SimulateAbnormalExit(EXCEP_STATUS_CLEAR_EDGE);
+ failed_cnt = CheckDBConcistency(count);
+ std::cout << "failed count before recovery: " << failed_cnt << "\n";
+
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ failed_cnt = CheckDBConcistency(count);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_INT_REMOVE_ODD_test)
+{
+ int count = 13234;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+
+ Populate(count);
+ for(int k = 1; k <= count; k++) {
+ if(k % 2 == 0) continue;
+
+ remove_index = k;
+ SimulateAbnormalExit(EXCEP_STATUS_CLEAR_EDGE);
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+
+ failed_cnt = CheckHalfDBConsistency(count, true);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, KEY_TYPE_SHA_256_REMOVE_EVEN_test)
+{
+ int count = 13234;
+ int failed_cnt;
+ int rval;
+
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_256;
+
+ Populate(count);
+ for(int k = 1; k <= count; k++) {
+ if(k % 2 == 1) continue;
+
+ remove_index = k;
+ SimulateAbnormalExit(EXCEP_STATUS_CLEAR_EDGE);
+ rval = RecoverDB();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+
+ failed_cnt = CheckHalfDBConsistency(count, false);
+ EXPECT_EQ(failed_cnt, 0);
+}
+
+TEST_F(AbnormalExitTest, RC_RECOVERY_test)
+{
+ int count = 1123;
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+ Populate(count);
+
+ Dict *dict = db->GetDictPtr();
+ IndexHeader *header = dict->GetHeaderPtr();
+
+ //Reset m_index_offset and m_data_offset to simulate RC process
+ header->rc_m_index_off_pre = header->m_index_offset;
+ header->rc_m_data_off_pre = header->m_data_offset;
+ if(header->pending_index_buff_size == 0) {
+ header->pending_index_buff_size = 100U;
+ }
+ if(header->pending_data_buff_size == 0) {
+ header->pending_data_buff_size = 100U;
+ }
+ header->m_index_offset += 150LL*1024*1024;
+ header->m_data_offset += 92LL*1024*1024;
+ // Now add some more
+ int count1 = 15846;
+ TestKey tkey(key_type);
+ std::string key_str;
+ for(int key = 1; key <= count1; key++) {
+ key_str = tkey.get_key(count + key);
+ int rval = db->Add(key_str, key_str);
+ if(rval != MBError::SUCCESS) {
+ std::cout << "failed to add " << key_str << "\n";
+ }
+ }
+
+ ResourceCollection rc(*db);
+ rc.ExceptionRecovery();
+
+ // Verify
+ EXPECT_EQ(count+count1, dict->Count());
+ EXPECT_EQ(841614U, header->m_index_offset);
+ EXPECT_EQ(610916U, header->m_data_offset);
+ EXPECT_EQ(0U, header->pending_index_buff_size);
+ EXPECT_EQ(0U, header->pending_data_buff_size);
+ EXPECT_EQ(0U, header->rc_m_index_off_pre);
+ EXPECT_EQ(0U, header->rc_m_data_off_pre);
+ EXPECT_EQ(0U, header->rc_root_offset);
+ EXPECT_EQ(0, header->rc_count);
+}
+
+TEST_F(AbnormalExitTest, RC_RECOVERY_RC_ROOT_TREE_test)
+{
+ int count = 23451;
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+ Populate(count);
+
+ Dict *dict = db->GetDictPtr();
+ IndexHeader *header = dict->GetHeaderPtr();
+ DictMem *dmm = dict->GetMM();
+
+ // Reset m_index_offset and m_data_offset to simulate RC process
+ header->rc_m_index_off_pre = header->m_index_offset;
+ header->rc_m_data_off_pre = header->m_data_offset;
+ if(header->pending_index_buff_size == 0) {
+ header->pending_index_buff_size = 100U;
+ }
+ if(header->pending_data_buff_size == 0) {
+ header->pending_data_buff_size = 100U;
+ }
+ header->m_index_offset += 250LL*1024*1024;
+ header->m_data_offset += 292LL*1024*1024;
+ size_t rc_off = dmm->InitRootNode_RC();
+ header->rc_root_offset.store(rc_off, MEMORY_ORDER_WRITER);
+
+ // Now add some more
+ int count1 = 2233;
+ TestKey tkey(key_type);
+ std::string key_str;
+ MBData mbd;
+ mbd.options = CONSTS::OPTION_RC_MODE;
+ for(int key = 1; key <= count1; key++) {
+ key_str = tkey.get_key(count + key);
+ mbd.buff = (uint8_t *) key_str.c_str();
+ mbd.data_len = key_str.size();
+
+ int rval = dict->Add((const uint8_t *)key_str.c_str(), key_str.size(), mbd, false);
+ if(rval != MBError::SUCCESS) {
+ std::cout << "failed to add " << key_str << "\n";
+ }
+ mbd.buff = NULL;
+ }
+
+ ResourceCollection rc(*db);
+ rc.ExceptionRecovery();
+
+ // Note the new entries added during rc will be ignored is expection occurs.
+ EXPECT_EQ(count, dict->Count());
+ EXPECT_EQ(1149932U, header->m_index_offset);
+ EXPECT_EQ(844268U, header->m_data_offset);
+ EXPECT_EQ(0U, header->pending_index_buff_size);
+ EXPECT_EQ(0U, header->pending_data_buff_size);
+ EXPECT_EQ(0U, header->rc_m_index_off_pre);
+ EXPECT_EQ(0U, header->rc_m_data_off_pre);
+ EXPECT_EQ(0U, header->rc_root_offset);
+ EXPECT_EQ(0, header->rc_count);
+}
+
+}
--- /dev/null
+
+#include <stdio.h>
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <list>
+#include <cstdlib>
+#include <iostream>
+#include <gtest/gtest.h>
+#include <time.h>
+
+#include "../db.h"
+#include "../mb_data.h"
+#include "../mb_backup.h"
+#include "../resource_pool.h"
+#include "./test_key.h"
+
+#define MB_DIR "/var/tmp/mabain_test/"
+#define MB_BACKUP_DIR "/var/tmp/mabain_backup/"
+#define MB_BACKUP_DIR_2 "/var/tmp/mabain_backup_2/"
+
+using namespace mabain;
+
+namespace {
+
+class BackupTest : public ::testing::Test
+{
+public:
+ BackupTest() {
+ }
+ virtual ~BackupTest() {
+ }
+ virtual void SetUp()
+ {
+ std::string cmd_2 = std::string("mkdir -p ") + MB_DIR;
+ std::string cmd = std::string("mkdir -p ") + MB_BACKUP_DIR;
+ if(system(cmd_2.c_str())!= 0){
+ }
+ if(system(cmd.c_str()) != 0) {
+ }
+ cmd = std::string("rm ") + MB_DIR + "_*";
+ std::string cmd_1 = std::string("rm ") + MB_BACKUP_DIR + "_*";
+ if(system(cmd.c_str()) != 0){
+ }
+ if(system(cmd_1.c_str()) != 0){
+ }
+ cmd = std::string("mkdir -p ") + MB_BACKUP_DIR_2;
+ if(system(cmd.c_str()) != 0){
+ }
+ cmd = std::string("rm ") + MB_BACKUP_DIR_2 + "_*";
+ if(system(cmd.c_str()) != 0){
+ }
+
+ }
+ virtual void TearDown() {
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+ void check_overwritten_keys(DB *db_bkp, int num)
+ {
+ TestKey tkey(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ std::string key;
+ int rval;
+ for(int i = 0; i < num; i++)
+ {
+ MBData mbd;
+ key = tkey.get_key(i);
+ rval = db_bkp->Find(key, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(std::string((const char*)mbd.buff, mbd.data_len)==key+"_new", true);
+ key = tkey1.get_key(i);
+ rval = db_bkp->Find(key, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(std::string((const char*)mbd.buff, mbd.data_len)==key+"_new", true);
+ }
+ }
+
+
+protected:
+};
+
+
+TEST_F(BackupTest, Create_backup_db)
+{
+ TestKey tkey(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ int num = 10;
+ std::string key;
+ int rval;
+ DB *db = new DB(MB_DIR, CONSTS::WriterOptions());
+ assert(db->is_open());
+ for(int i = 0; i < num; i++)
+ {
+ key = tkey.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+
+ clock_t t;
+ t = clock();
+
+ try
+ {
+ DBBackup bk(*db);
+ rval = bk.Backup(MB_BACKUP_DIR);
+ }
+ catch (int error)
+ {
+ std::cout << "Backup failed "<< MBError::get_error_str(error) <<"\n";
+ rval = error;
+
+ }
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ t = clock() -t;
+ printf ("It took me %ld clicks (%f seconds).\n",t,((float)t)/CLOCKS_PER_SEC);
+ DB *db_bkp = new DB(MB_BACKUP_DIR, CONSTS::WriterOptions());
+
+ //Test to check already inserted key.
+ for(int i = 0; i < num; i++)
+ {
+ key = tkey.get_key(i);
+ rval = db_bkp->Add(key, key);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+ key = tkey1.get_key(i);
+ rval = db_bkp->Add(key, key);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+ }
+
+ //Test to overwrite existing key and retrive it.
+ for(int i = 0; i < num; i++)
+ {
+ key = tkey.get_key(i);
+ rval = db_bkp->Add(key, key+"_new", true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key = tkey1.get_key(i);
+ rval = db_bkp->Add(key, key+"_new", true);
+ }
+
+ // Retrive the overwritten key and check it.
+ check_overwritten_keys(db_bkp, num);
+
+ db_bkp->Close();
+ delete db_bkp;
+ //Asycwriter DB backup test
+ db_bkp = new DB(MB_BACKUP_DIR, CONSTS::WriterOptions() | CONSTS::ASYNC_WRITER_MODE, 128LL*1024*1024, 128LL*1024*1024);
+ assert(db_bkp->is_open());
+#ifndef __SHM_QUEUE__
+ assert(db_bkp->AsyncWriterEnabled());
+#endif
+ DB *db_r = new DB(MB_BACKUP_DIR, CONSTS::ReaderOptions(), 128LL*1024*1024, 128LL*1024*1024);
+ assert(db_r->is_open());
+#ifndef __SHM_QUEUE__
+ assert(db_r->SetAsyncWriterPtr(db_bkp) == MBError::SUCCESS);
+ assert(db_r->AsyncWriterEnabled());
+#endif
+ rval = db_r->Backup(MB_BACKUP_DIR_2);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ while (db_r->AsyncWriterBusy()) {
+ usleep(10000);
+ }
+#ifndef __SHM_QUEUE__
+ assert(db_r->UnsetAsyncWriterPtr(db_bkp) == MBError::SUCCESS);
+#endif
+ db_r->Close();
+ db_bkp->Close();
+ delete db_r;
+ delete db_bkp;
+ db_bkp = new DB(MB_BACKUP_DIR_2, CONSTS::WriterOptions(), 128LL*1024*1024, 128LL*1024*1024);
+ check_overwritten_keys(db_bkp, num);
+
+ //Test to remove all the keys from the DB.
+ rval = db_bkp->RemoveAll();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(0, db_bkp->Count());
+ db_bkp->Close();
+ delete db_bkp;
+
+ db->Close();
+ delete db;
+}
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "../dict_mem.h"
+#include "../drm_base.h"
+#include "../integer_4b_5b.h"
+#include "../resource_pool.h"
+
+using namespace mabain;
+
+namespace {
+
+#define DICT_MEM_TEST_DIR "/var/tmp/mabain_test/"
+
+class DictMemTest : public ::testing::Test
+{
+public:
+ DictMemTest() {
+ dmm = NULL;
+ header = NULL;
+ memset(&lfree, 0, sizeof(lfree));
+ }
+ virtual ~DictMemTest() {
+ DestroyDMM();
+ }
+
+ virtual void SetUp() {
+ std::string cmd = std::string("mkdir -p ") + DICT_MEM_TEST_DIR;
+ if(system(cmd.c_str()) != 0) {
+ }
+ }
+ virtual void TearDown() {
+ std::string cmd = std::string("rm -rf ") + DICT_MEM_TEST_DIR + "/_*";
+ if(system(cmd.c_str()) != 0) {
+ }
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+ void Init() {
+ dmm = new DictMem(std::string(DICT_MEM_TEST_DIR), true, 8*1024*1024,
+ CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW,
+ 8*1024*1024, 1, 0);
+ dmm->InitRootNode();
+ EXPECT_EQ(dmm->IsValid(), true);
+ header = dmm->GetHeaderPtr();
+ EXPECT_EQ(header != NULL, true);
+ lfree.LockFreeInit(&header->lock_free, header, CONSTS::ACCESS_MODE_WRITER);
+ dmm->InitLockFreePtr(&lfree);
+ }
+
+ void DestroyDMM() {
+ if(dmm != NULL) {
+ dmm->Destroy();
+ delete dmm;
+ dmm = NULL;
+ }
+ }
+
+protected:
+ DictMem *dmm;
+ IndexHeader *header;
+ LockFree lfree;
+};
+
+TEST_F(DictMemTest, Constructor_test)
+{
+ dmm = new DictMem(std::string(DICT_MEM_TEST_DIR), true, 8*1024*1024,
+ CONSTS::ACCESS_MODE_WRITER, 8*1024*1024, 1, 0);
+ dmm->PrintStats(std::cout);
+ EXPECT_EQ(dmm->IsValid(), false);
+ dmm->InitRootNode();
+ EXPECT_EQ(dmm->IsValid(), true);
+ dmm->PrintStats(std::cout);
+ DestroyDMM();
+
+ dmm = new DictMem(std::string(DICT_MEM_TEST_DIR), false, 8*1024*1024,
+ CONSTS::ACCESS_MODE_WRITER, 8*1024*1024, 1, 0);
+ EXPECT_EQ(dmm->IsValid(), true);
+ DestroyDMM();
+
+ dmm = new DictMem(std::string(DICT_MEM_TEST_DIR), false, 8*1024*1024,
+ CONSTS::ACCESS_MODE_READER, 8*1024*1024, 1, 0);
+ EXPECT_EQ(dmm->IsValid(), false);
+
+ DestroyDMM();
+
+ int error = 0;
+ try {
+ dmm = new DictMem(std::string(DICT_MEM_TEST_DIR), false, 8*1024*1024,
+ CONSTS::ACCESS_MODE_WRITER, 12*1024*1024, 1, 0);
+ } catch (int err) {
+ error = err;
+ }
+ EXPECT_EQ(error, MBError::INVALID_SIZE);
+ EXPECT_EQ(dmm==NULL, true);
+}
+
+TEST_F(DictMemTest, AddRootEdge_test)
+{
+ Init();
+
+ EdgePtrs edge_ptrs;
+ size_t offset;
+ uint8_t *shm_ptr;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "mabain-test", 11, 1234);
+ EXPECT_EQ(edge_ptrs.flag_ptr[0], EDGE_FLAG_DATA_OFF);
+ offset = Get5BInteger(edge_ptrs.ptr);
+ shm_ptr = dmm->GetShmPtr(offset, 10);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ if(memcmp(shm_ptr, "abain-test", 10)) {
+ EXPECT_EQ(1, 2);
+ }
+ offset = Get6BInteger(edge_ptrs.offset_ptr);
+ EXPECT_EQ(offset, 1234u);
+ shm_ptr = dmm->GetShmPtr(header->excep_lf_offset, EDGE_SIZE);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ if(memcmp(shm_ptr, edge_ptrs.ptr, EDGE_SIZE)) {
+ EXPECT_EQ(1, 2);
+ }
+ EXPECT_EQ(header->excep_updating_status, EXCEP_STATUS_NONE);
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'x', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "xyz", 3, 2234);
+ EXPECT_EQ(edge_ptrs.flag_ptr[0], EDGE_FLAG_DATA_OFF);
+ offset = Get6BInteger(edge_ptrs.offset_ptr);
+ EXPECT_EQ(offset, 2234u);
+ shm_ptr = dmm->GetShmPtr(header->excep_lf_offset, EDGE_SIZE);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ if(memcmp(shm_ptr, edge_ptrs.ptr, EDGE_SIZE)) {
+ EXPECT_EQ(1, 2);
+ }
+ EXPECT_EQ(header->excep_updating_status, EXCEP_STATUS_NONE);
+}
+
+TEST_F(DictMemTest, InsertNode_test)
+{
+ Init();
+
+ EdgePtrs edge_ptrs;
+ size_t offset;
+ uint8_t *shm_ptr;
+ MBData mbd;
+ int rval;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "mabain-test", 11, 1234);
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ EXPECT_EQ(edge_ptrs.len_ptr[0] >= 6, true);
+ rval = dmm->InsertNode(edge_ptrs, 6, 1334, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(header->excep_updating_status, EXCEP_STATUS_NONE);
+ offset = Get6BInteger(edge_ptrs.offset_ptr);
+ shm_ptr = dmm->GetShmPtr(offset, dmm->GetNodeSizePtr()[0]);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ EXPECT_EQ(shm_ptr[0], FLAG_NODE_NONE | FLAG_NODE_MATCH);
+ EXPECT_EQ(shm_ptr[1], 0);
+ EXPECT_EQ(shm_ptr[8], (uint8_t)'-');
+ EXPECT_EQ(Get6BInteger(shm_ptr+2), 1334u);
+ shm_ptr = dmm->GetShmPtr(header->excep_lf_offset, EDGE_SIZE);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ if(memcmp(shm_ptr, edge_ptrs.ptr, EDGE_SIZE)) {
+ EXPECT_EQ(1, 2);
+ }
+ EXPECT_EQ(edge_ptrs.len_ptr[0], 6);
+ EXPECT_EQ(edge_ptrs.flag_ptr[0], 0);
+ if(memcmp(edge_ptrs.ptr, "abain", 5)) {
+ EXPECT_EQ(1, 2);
+ }
+}
+
+TEST_F(DictMemTest, InsertNode_test1)
+{
+ Init();
+
+ EdgePtrs edge_ptrs;
+ size_t offset;
+ uint8_t *shm_ptr;
+ MBData mbd;
+ int rval;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "mabain-testabcdefghijk", 22, 1234);
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ EXPECT_EQ(edge_ptrs.len_ptr[0] >= 6, true);
+ rval = dmm->InsertNode(edge_ptrs, 6, 1334, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(header->excep_updating_status, EXCEP_STATUS_NONE);
+ offset = Get6BInteger(edge_ptrs.offset_ptr);
+ shm_ptr = dmm->GetShmPtr(offset, dmm->GetNodeSizePtr()[0]);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ EXPECT_EQ(shm_ptr[0], FLAG_NODE_NONE | FLAG_NODE_MATCH);
+ EXPECT_EQ(shm_ptr[1], 0);
+ EXPECT_EQ(shm_ptr[8], (uint8_t)'-');
+ EXPECT_EQ(Get6BInteger(shm_ptr+2), 1334u);
+ shm_ptr = dmm->GetShmPtr(header->excep_lf_offset, EDGE_SIZE);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ if(memcmp(shm_ptr, edge_ptrs.ptr, EDGE_SIZE)) {
+ EXPECT_EQ(1, 2);
+ }
+ EXPECT_EQ(edge_ptrs.len_ptr[0], 6);
+ EXPECT_EQ(edge_ptrs.flag_ptr[0], 0);
+ if(memcmp(edge_ptrs.ptr, "abain", 5)) {
+ EXPECT_EQ(1, 2);
+ }
+}
+
+TEST_F(DictMemTest, InsertNode_test2)
+{
+ Init();
+
+ EdgePtrs edge_ptrs;
+ size_t offset;
+ uint8_t *shm_ptr;
+ MBData mbd;
+ int rval;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'x', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "xxxxxxxmabain-test", 18, 1234);
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'x', edge_ptrs), MBError::SUCCESS);
+ EXPECT_EQ(edge_ptrs.len_ptr[0] >= 6, true);
+ rval = dmm->InsertNode(edge_ptrs, 13, 1334, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(header->excep_updating_status, EXCEP_STATUS_NONE);
+ offset = Get6BInteger(edge_ptrs.offset_ptr);
+ shm_ptr = dmm->GetShmPtr(offset, dmm->GetNodeSizePtr()[0]);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ EXPECT_EQ(shm_ptr[0], FLAG_NODE_NONE | FLAG_NODE_MATCH);
+ EXPECT_EQ(shm_ptr[1], 0);
+ EXPECT_EQ(shm_ptr[8], (uint8_t)'-');
+ EXPECT_EQ(Get6BInteger(shm_ptr+2), 1334u);
+ shm_ptr = dmm->GetShmPtr(header->excep_lf_offset, EDGE_SIZE);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ if(memcmp(shm_ptr, edge_ptrs.ptr, EDGE_SIZE)) {
+ EXPECT_EQ(1, 2);
+ }
+ EXPECT_EQ(edge_ptrs.len_ptr[0], 13);
+ EXPECT_EQ(edge_ptrs.flag_ptr[0], 0);
+ EXPECT_EQ(Get5BInteger(edge_ptrs.ptr), 3631u);
+}
+
+TEST_F(DictMemTest, AddLink_test)
+{
+ Init();
+ EdgePtrs edge_ptrs;
+ uint8_t *shm_ptr;
+ MBData mbd;
+ int rval;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "mabain-test", 18, 1234);
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ rval = dmm->AddLink(edge_ptrs, 7, (const uint8_t *)"klsakkslslsldds",
+ 15, 12345, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(edge_ptrs.offset, 1681u);
+ EXPECT_EQ(header->excep_updating_status, EXCEP_STATUS_NONE);
+ EXPECT_EQ(Get6BInteger(edge_ptrs.offset_ptr), 3609u);
+ shm_ptr = dmm->GetShmPtr(3609, 10);
+ EXPECT_EQ((int)shm_ptr[0], 0);
+ EXPECT_EQ((int)shm_ptr[1], 1);
+ EXPECT_EQ((char)shm_ptr[8], 't');
+ EXPECT_EQ((char)shm_ptr[9], 'k');
+ EXPECT_EQ(Get5BInteger(edge_ptrs.ptr), 3655u);
+ shm_ptr = dmm->GetShmPtr(3655, 10);
+ EXPECT_EQ(std::string((const char *)shm_ptr, 5).compare("abain"), 0);
+}
+
+TEST_F(DictMemTest, UpdateNode_test)
+{
+ Init();
+ EdgePtrs edge_ptrs;
+ uint8_t *shm_ptr;
+ MBData mbd;
+ int rval;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "mabain-test", 18, 1234);
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ rval = dmm->AddLink(edge_ptrs, 7, (const uint8_t *)"klsakkslslsldds",
+ 15, 12345, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ uint8_t tmp_buff[256];
+ int match_len = 7;
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ bool next = dmm->FindNext((const uint8_t*)"abcdefg", 7, match_len, edge_ptrs, tmp_buff);
+ EXPECT_EQ(next, false);
+ rval = dmm->UpdateNode(edge_ptrs, (const uint8_t*)"abcdefg", 7, 12345);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(edge_ptrs.offset, 1681u);
+ EXPECT_EQ(Get5BInteger(edge_ptrs.ptr), 3655u);
+ shm_ptr = dmm->GetShmPtr(3655, 6);
+ EXPECT_EQ(std::string((const char*)shm_ptr, 6).compare("abain-"), 0);
+}
+
+TEST_F(DictMemTest, FindNext_test)
+{
+ Init();
+ EdgePtrs edge_ptrs;
+ MBData mbd;
+ int rval;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "mabain-abc", 18, 1234);
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ rval = dmm->AddLink(edge_ptrs, 7, (const uint8_t *)"hijk", 4, 12345, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ bool next;
+ uint8_t key[256];
+ uint8_t tmp_buff[256];
+ int match_len = 7;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ next = dmm->FindNext((const uint8_t*)"xyz293ksk", 9, match_len,
+ edge_ptrs, tmp_buff);
+ EXPECT_EQ(next, false);
+ rval = dmm->UpdateNode(edge_ptrs, (const uint8_t*)"xyz293ksk", 9, 22345);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+
+ for(int i = 0; i < 256; i++) {
+ key[i] = (uint8_t) i;
+ }
+ for(int i = 0; i < 256; i++) {
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ next = dmm->FindNext((const uint8_t*)key+i, 1, match_len,
+ edge_ptrs, tmp_buff);
+ if((char)key[i] == 'a' || (char)key[i] == 'h' || (char) key[i] == 'x') {
+ EXPECT_EQ(next, true);
+ } else {
+ EXPECT_EQ(next, false);
+ }
+ }
+}
+
+TEST_F(DictMemTest, GetRootEdge_test)
+{
+ Init();
+
+ EdgePtrs edge_ptrs;
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, 'm', edge_ptrs), MBError::SUCCESS);
+ dmm->AddRootEdge(edge_ptrs, (const uint8_t *) "mabain-unittest", 15, 1234);
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ EXPECT_EQ(dmm->GetRootEdge(false, 'm', edge_ptrs), MBError::SUCCESS);
+ EXPECT_EQ(edge_ptrs.offset, 1681u);
+ EXPECT_EQ(Get5BInteger(edge_ptrs.ptr), 3592u);
+
+ uint8_t *ptr = dmm->GetShmPtr(3592, 14);
+ EXPECT_EQ(memcmp(ptr, "abain-unittest", 14)==0, true);
+}
+
+TEST_F(DictMemTest, GetRootEdge_Writer_test)
+{
+ Init();
+
+ EdgePtrs edge_ptrs;
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ for(int i = 0; i < 256; i++) {
+ EXPECT_EQ(dmm->GetRootEdge_Writer(false, i, edge_ptrs), MBError::SUCCESS);
+ }
+}
+
+TEST_F(DictMemTest, ClearRootEdge_test)
+{
+ Init();
+ uint8_t buff[1024];
+ EdgePtrs edge_ptrs;
+
+ dmm->WriteData(buff, 0, dmm->GetRootOffset());
+
+ dmm->ClearRootEdge(0);
+ dmm->GetRootEdge(0, 0, edge_ptrs);
+ EXPECT_EQ(memcmp(edge_ptrs.edge_buff, DictMem::empty_edge, EDGE_SIZE)==0, true);
+
+ dmm->ClearRootEdge(10);
+ dmm->GetRootEdge(0, 10, edge_ptrs);
+ EXPECT_EQ(memcmp(edge_ptrs.edge_buff, DictMem::empty_edge, EDGE_SIZE)==0, true);
+
+ dmm->ClearRootEdge(111);
+ dmm->GetRootEdge(0, 111, edge_ptrs);
+ EXPECT_EQ(memcmp(edge_ptrs.edge_buff, DictMem::empty_edge, EDGE_SIZE)==0, true);
+}
+
+TEST_F(DictMemTest, ReserveData_test)
+{
+ Init();
+
+ uint8_t buff[256];
+ int size;
+ size_t offset = 0;
+
+ size = 100;
+ dmm->ReserveData(buff, size, offset, false);
+ EXPECT_EQ(header->m_index_offset, 3692u);
+ EXPECT_EQ(header->m_index_offset, offset+100);
+}
+
+TEST_F(DictMemTest, NextEdge_test)
+{
+ Init();
+}
+
+TEST_F(DictMemTest, RemoveEdgeByIndex_test)
+{
+ Init();
+}
+
+TEST_F(DictMemTest, InitRootNode_test)
+{
+ Init();
+
+ dmm->InitRootNode();
+ EXPECT_EQ(header->m_index_offset, 3592u);
+}
+
+TEST_F(DictMemTest, WriteEdge_test)
+{
+ Init();
+
+ header->m_index_offset = 10000;
+ EdgePtrs edge_ptrs;
+ edge_ptrs.offset = 1234;
+ edge_ptrs.ptr = edge_ptrs.edge_buff;
+ dmm->WriteEdge(edge_ptrs);
+ EXPECT_EQ(header->excep_updating_status, EXCEP_STATUS_NONE);
+}
+
+TEST_F(DictMemTest, WriteData_test)
+{
+ Init();
+
+ size_t offset;
+ int size;
+ uint8_t buff[32];
+ uint8_t *shm_ptr;
+
+ offset = 12345;
+ size = 21;
+ header->m_index_offset = offset + 10000;
+ for(int i = 0; i < size; i++) {
+ buff[i] = (uint8_t) i;
+ }
+ dmm->WriteData(buff, size, offset);
+ dmm->Flush();
+ shm_ptr = dmm->GetShmPtr(offset, size);
+ EXPECT_EQ(shm_ptr != NULL, true);
+ for(int i = 0; i < size; i++) {
+ EXPECT_EQ((int)shm_ptr[i], i);
+ }
+}
+
+TEST_F(DictMemTest, Flush_test)
+{
+ Init();
+ uint8_t buff[32];
+ header->m_index_offset = 10000000;
+ dmm->WriteData(buff, 32, 10000);
+ dmm->Flush();
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "../dict.h"
+#include "../drm_base.h"
+#include "../integer_4b_5b.h"
+#include "../resource_pool.h"
+
+using namespace mabain;
+
+namespace {
+
+#define DICT_TEST_DIR "/var/tmp/mabain_test/"
+#define ONE_MEGA 1024*1024
+#define FAKE_KEY "test-key-129ksjkjdjdkfjdkfjkdjfkdfjkdjkkdsalslsdlkflsdfsd"
+#define FAKE_DATA "This is a test; fake data; sdkll vlksaflksafdlfsadflkdkvkvkv ldlsldklkdsk4930 90234924894388438348348348878&^&^YYYYYY"
+
+class DictTest : public ::testing::Test
+{
+public:
+ DictTest() {
+ dict = NULL;
+ header = NULL;
+ }
+ virtual ~DictTest() {
+ DestroyDict();
+ }
+
+ virtual void SetUp() {
+ std::string cmd = std::string("mkdir -p ") + DICT_TEST_DIR;
+ if(system(cmd.c_str()) != 0) {
+ }
+ }
+ virtual void TearDown() {
+ ResourcePool::getInstance().RemoveAll();
+ std::string cmd = std::string("rm -rf ") + DICT_TEST_DIR + "/_*";
+ if(system(cmd.c_str()) != 0) {
+ }
+ }
+
+ void InitDict(bool init_header, int opts, int blk_size, int bucket_sz) {
+ dict = new Dict(std::string(DICT_TEST_DIR), init_header, 0,
+ opts, 256LL*ONE_MEGA, 256LL*ONE_MEGA,
+ 32*ONE_MEGA, blk_size, 100, 150, bucket_sz, 0);
+ if(init_header) {
+ EXPECT_EQ(dict->Init(0), MBError::SUCCESS);
+ }
+ if(init_header) EXPECT_EQ(dict->Status(), MBError::SUCCESS);
+ else EXPECT_EQ(MBError::NOT_INITIALIZED, dict->Status());
+ header = dict->GetHeaderPtr();
+ EXPECT_EQ(header != NULL, true);
+ }
+
+ int AddKV(int key_len, int data_len, bool overwrite) {
+ MBData mbd;
+ mbd.data_len = data_len;
+ mbd.buff = (uint8_t *) FAKE_DATA;
+ int rval = dict->Add((const uint8_t *)FAKE_KEY, key_len, mbd, overwrite);
+ mbd.buff = NULL;
+ return rval;
+ }
+
+ int GetNodeOffset(const uint8_t *node_key, int key_len, size_t &node_offset) {
+ if(dict == NULL)
+ return MBError::INVALID_ARG;
+ MBData mbd;
+ node_offset = 0;
+ mbd.options = CONSTS::OPTION_FIND_AND_STORE_PARENT;
+ int rval = dict->Find(node_key, key_len, mbd);
+ if(rval == MBError::IN_DICT) {
+ node_offset = Get6BInteger(mbd.edge_ptrs.offset_ptr);
+ }
+ return rval;
+ }
+
+ void DestroyDict() {
+ if(dict != NULL) {
+ dict->Destroy();
+ delete dict;
+ dict = NULL;
+ }
+ }
+
+protected:
+ Dict *dict;
+ IndexHeader *header;
+};
+
+TEST_F(DictTest, Constructor_test)
+{
+ int opts = CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW;
+ InitDict(true, opts, 32*ONE_MEGA, 100);
+ DestroyDict();
+
+ int rval = MBError::SUCCESS;
+ try {
+ InitDict(false, CONSTS::ACCESS_MODE_READER, 16*ONE_MEGA, 100);
+ } catch (int err) {
+ rval = err;
+ }
+
+ EXPECT_EQ(rval, MBError::INVALID_SIZE);
+
+ rval = MBError::SUCCESS;
+ try {
+ InitDict(false, CONSTS::ACCESS_MODE_WRITER, 32*ONE_MEGA, 111);
+ } catch (int err) {
+ rval = err;
+ }
+
+ EXPECT_EQ(rval, MBError::INVALID_SIZE);
+
+ InitDict(false, CONSTS::ACCESS_MODE_READER, 32*ONE_MEGA, 100);
+}
+
+TEST_F(DictTest, PrintHeader_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 32*ONE_MEGA, 128);
+ dict->PrintHeader(std::cout);
+}
+
+TEST_F(DictTest, Add_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 4*ONE_MEGA, 10);
+
+ uint8_t key[256];
+ int key_len;
+ MBData mbd;
+ int rval;
+ uint8_t *shm_ptr;
+
+ key_len = 10;
+ memcpy(key, "test-key-1", key_len);
+ mbd.data_len = 100;
+ mbd.Resize(mbd.data_len);
+ memcpy(mbd.buff, FAKE_DATA, mbd.data_len);
+ rval = dict->Add(key, key_len, mbd, false);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = dict->Add(key, key_len, mbd, false);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+ EXPECT_EQ(dict->Count(), 1);
+ shm_ptr = dict->GetShmPtr(header->m_data_offset-mbd.data_len, mbd.data_len);
+ EXPECT_EQ(shm_ptr!=NULL, true);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, mbd.data_len), 0);
+
+ key_len = 10;
+ memcpy(key, "test-key-2", key_len);
+ mbd.data_len = 101;
+ mbd.Resize(mbd.data_len);
+ memcpy(mbd.buff, FAKE_DATA, mbd.data_len);
+ rval = dict->Add(key, key_len, mbd, false);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(dict->Count(), 2);
+ shm_ptr = dict->GetShmPtr(header->m_data_offset-mbd.data_len, mbd.data_len);
+ EXPECT_EQ(shm_ptr!=NULL, true);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, mbd.data_len), 0);
+
+ key_len = 10;
+ memcpy(key, "test-key-3", key_len);
+ mbd.data_len = 105;
+ mbd.Resize(mbd.data_len);
+ memcpy(mbd.buff, FAKE_DATA, mbd.data_len);
+ rval = dict->Add(key, key_len, mbd, false);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(dict->Count(), 3);
+ shm_ptr = dict->GetShmPtr(header->m_data_offset-mbd.data_len, mbd.data_len);
+ EXPECT_EQ(shm_ptr!=NULL, true);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, mbd.data_len), 0);
+
+ key_len = 10;
+ memcpy(key, "test-key-2", key_len);
+ mbd.data_len = 108;
+ mbd.Resize(mbd.data_len);
+ memcpy(mbd.buff, FAKE_DATA, mbd.data_len);
+ rval = dict->Add(key, key_len, mbd, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(dict->Count(), 3);
+ shm_ptr = dict->GetShmPtr(header->m_data_offset-mbd.data_len, mbd.data_len);
+ EXPECT_EQ(shm_ptr!=NULL, true);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, mbd.data_len), 0);
+}
+
+TEST_F(DictTest, Find_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 4*ONE_MEGA, 10);
+ int key_len;
+ int data_len;
+ MBData mbd;
+ int rval;
+
+ key_len = 10;
+ rval = dict->Find((const uint8_t *)FAKE_KEY, key_len, mbd);
+ EXPECT_EQ(rval, MBError::NOT_EXIST);
+
+ data_len = 32;
+ rval = AddKV(key_len, data_len, false);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ dict->Flush();
+ rval = dict->Find((const uint8_t*)FAKE_KEY, key_len, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(mbd.data_len, 32);
+ EXPECT_EQ(memcmp(mbd.buff, FAKE_DATA, mbd.data_len), 0);
+}
+
+TEST_F(DictTest, FindPrefix_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 4*ONE_MEGA, 10);
+ int key_len;
+ int data_len;
+ MBData mbd;
+ int rval;
+
+ key_len = 10;
+ data_len = 32;
+ rval = AddKV(key_len, data_len, false);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key_len = 15;
+ rval = dict->FindPrefix((const uint8_t *)FAKE_KEY, key_len, mbd);
+ EXPECT_EQ(MBError::SUCCESS, rval);
+ EXPECT_EQ(mbd.data_len, 32);
+ EXPECT_EQ(memcmp(mbd.buff, FAKE_DATA, mbd.data_len), 0);
+}
+
+TEST_F(DictTest, Remove_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 4*ONE_MEGA, 10);
+ int key_len;
+ int data_len;
+ int rval;
+
+ key_len = 10;
+ rval = dict->Remove((const uint8_t *)FAKE_KEY, key_len);
+ EXPECT_EQ(rval, MBError::NOT_EXIST);
+
+ key_len = 13;
+ data_len = 28;
+ rval = AddKV(key_len, data_len, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = dict->Remove((const uint8_t *)FAKE_KEY, key_len);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(dict->Count(), 0);
+}
+
+TEST_F(DictTest, RemoveAll_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 8*ONE_MEGA, 12);
+ int rval;
+
+ rval = AddKV(10, 50, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = AddKV(11, 52, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = AddKV(12, 54, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = AddKV(19, 64, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ dict->PrintStats(std::cout);
+
+ rval = dict->RemoveAll();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(dict->Count(), 0);
+ EXPECT_EQ(header->m_data_offset, dict->GetStartDataOffset());
+ EXPECT_EQ(header->pending_data_buff_size, 0);
+ EXPECT_EQ(header->eviction_bucket_index, 0);
+ EXPECT_EQ(header->num_update, 0);
+}
+
+TEST_F(DictTest, ReserveData_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 8*ONE_MEGA, 12);
+ int data_size;
+ size_t offset;
+ uint8_t *shm_ptr;
+
+ data_size = 1;
+ dict->ReserveData((const uint8_t *)FAKE_DATA, data_size, offset);
+ offset += DATA_HDR_BYTE;
+ shm_ptr = dict->GetShmPtr(offset, data_size);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, data_size), 0);
+
+ data_size = 19;
+ dict->ReserveData((const uint8_t *)FAKE_DATA, data_size, offset);
+ offset += DATA_HDR_BYTE;
+ shm_ptr = dict->GetShmPtr(offset, data_size);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, data_size), 0);
+}
+
+TEST_F(DictTest, WriteData_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 8*ONE_MEGA, 12);
+ int data_size;
+ size_t offset;
+ uint8_t *shm_ptr;
+
+ data_size = 14;
+ offset = 111;
+ header->m_data_offset = offset + data_size;
+ dict->Reserve(offset, data_size, shm_ptr);
+ dict->WriteData((const uint8_t *)FAKE_DATA, data_size, offset);
+ shm_ptr = dict->GetShmPtr(offset, data_size);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, data_size), 0);
+
+ data_size = 33;
+ offset = 1234;
+ header->m_data_offset = offset + data_size;
+ dict->WriteData((const uint8_t *)FAKE_DATA, data_size, offset);
+ shm_ptr = dict->GetShmPtr(offset, data_size);
+ EXPECT_EQ(memcmp(shm_ptr, FAKE_DATA, data_size), 0);
+}
+
+TEST_F(DictTest, PrintStats_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 8*ONE_MEGA, 12);
+ dict->PrintStats(std::cout);
+}
+
+TEST_F(DictTest, ReadRootNode_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 8*ONE_MEGA, 12);
+ AddKV(10, 15, true);
+ AddKV(12, 50, true);
+ AddKV(15, 34, true);
+ AddKV(8, 22, true);
+
+ EdgePtrs edge_ptrs;
+ uint8_t buff[256];
+ MBData mbd;
+ int match;
+ int rval;
+
+ rval = dict->ReadRootNode(buff, edge_ptrs, match, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(edge_ptrs.offset, 264u);
+ EXPECT_EQ((int)buff[0], 0);
+ EXPECT_EQ((int)buff[1], 255);
+}
+
+TEST_F(DictTest, ReadNode_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 8*ONE_MEGA, 12);
+ AddKV(8, 22, true);
+ AddKV(10, 15, true);
+ AddKV(12, 50, true);
+ AddKV(15, 34, true);
+
+ int rval;
+ size_t offset = 0;
+ EdgePtrs edge_ptrs;
+ uint8_t buff[256];
+ MBData mbd;
+ int match;
+
+ rval = GetNodeOffset((const uint8_t *)FAKE_KEY, 8, offset);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+ rval = dict->ReadNode(offset, buff, edge_ptrs, match, mbd, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ((int)buff[0], 1);
+ EXPECT_EQ((int)buff[1], 0);
+ EXPECT_EQ(match, 2);
+}
+
+TEST_F(DictTest, ReadNextEdge_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 8*ONE_MEGA, 12);
+ AddKV(10, 15, true);
+ AddKV(12, 50, true);
+ AddKV(15, 34, true);
+ AddKV(8, 22, true);
+
+ EdgePtrs edge_ptrs;
+ uint8_t buff[256];
+ MBData mbd;
+ std::string match_str;
+ size_t offset;
+ int match;
+ int rval;
+
+ memset(&edge_ptrs, 0, sizeof(edge_ptrs));
+ memset(buff, 0, 256);
+
+ edge_ptrs.curr_nt = 15;
+ buff[0] = 14;
+ rval = dict->ReadNextEdge(buff, edge_ptrs, match, mbd, match_str, offset, true);
+ EXPECT_EQ(rval, MBError::OUT_OF_BOUND);
+
+ rval = dict->ReadRootNode(buff, edge_ptrs, match, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = dict->ReadNextEdge(buff, edge_ptrs, match, mbd, match_str, offset, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(match, 0);
+ EXPECT_EQ(offset, 0u);
+}
+
+TEST_F(DictTest, ReadNodeHeader_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 4*ONE_MEGA, 28);
+ AddKV(10, 15, true);
+ AddKV(12, 50, true);
+ AddKV(15, 34, true);
+ AddKV(8, 22, true);
+
+ int rval;
+ size_t offset=0, data_offset, data_link_offset;
+ int node_size;
+ int match;
+ rval = GetNodeOffset((const uint8_t *)FAKE_KEY, 8, offset);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+ dict->ReadNodeHeader(offset, node_size, match, data_offset, data_link_offset);
+ EXPECT_EQ(node_size, 22);
+ EXPECT_EQ(match, 2);
+ EXPECT_EQ(data_offset, 143u);
+ EXPECT_EQ(data_link_offset, 3647u);
+}
+
+/***
+TEST_F(DictTest, CloseDBFiles_test)
+{
+ InitDict(true, CONSTS::ACCESS_MODE_WRITER, 4*ONE_MEGA, 28);
+ AddKV(10, 15, true);
+ AddKV(11, 50, true);
+ AddKV(12, 34, true);
+ AddKV(8, 22, true);
+ EXPECT_EQ(dict->Status(), MBError::DB_CLOSED);
+ EXPECT_TRUE(dict->GetHeaderPtr() == NULL);
+ EXPECT_TRUE(dict->GetMM()->GetHeaderPtr() == NULL);
+
+ int rval;
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(dict->Status(), MBError::SUCCESS);
+ EXPECT_TRUE(dict->GetHeaderPtr() != NULL);
+ EXPECT_TRUE(dict->GetMM()->GetHeaderPtr() != NULL);
+ EXPECT_TRUE(dict->GetMM()->GetHeaderPtr() == dict->GetHeaderPtr());
+
+ MBData mbd;
+ rval = dict->Find((const uint8_t*)FAKE_KEY, 10, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = dict->Find((const uint8_t*)FAKE_KEY, 11, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = dict->Find((const uint8_t*)FAKE_KEY, 12, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = dict->Find((const uint8_t*)FAKE_KEY, 8, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+}***/
+
+}
--- /dev/null
+#include <unistd.h>
+#include <stdlib.h>
+#include <list>
+
+#include <gtest/gtest.h>
+
+#include "../async_writer.h"
+#include "../db.h"
+#include "./test_key.h"
+#include "../resource_pool.h"
+
+using namespace mabain;
+const char *db_dir = "/var/tmp/mabain_test/";
+
+namespace {
+
+class EvictionTest : public ::testing::Test
+{
+public:
+ EvictionTest() {
+ db_async = NULL;
+ db = NULL;
+ memset(&mbconf, 0, sizeof(mbconf));
+ std::string cmd = std::string("rm ") + db_dir + "_mabain_*";
+ if(system(cmd.c_str()) != 0) {
+ }
+ }
+ virtual ~EvictionTest() {}
+ virtual void SetUp() {
+ std::string cmd = std::string("mkdir -p ") + db_dir;
+ if(system(cmd.c_str()) != 0) {
+ }
+ mbconf.mbdir = db_dir;
+ mbconf.memcap_index = 64*1024*1024LL;
+ mbconf.memcap_data = 64*1024*1024LL;
+ mbconf.block_size_index = 8*1024*1024LL;
+ mbconf.block_size_data = 16*1024*1024LL;
+ }
+ void OpenDB(int entry_per_bucket) {
+ mbconf.num_entry_per_bucket = entry_per_bucket;
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::ASYNC_WRITER_MODE;
+ db_async = new DB(mbconf);
+ assert(db_async->is_open());
+
+ mbconf.options = CONSTS::ACCESS_MODE_READER;
+ db = new DB(mbconf);
+ assert(db->is_open());
+#ifndef __SHM_QUEUE__
+ assert(db->SetAsyncWriterPtr(db_async) == MBError::SUCCESS);
+ assert(db->AsyncWriterEnabled());
+#endif
+ }
+ void Insert(int n0, int n) {
+ TestKey tkey(MABAIN_TEST_KEY_TYPE_INT);
+ std::string key;
+ for(int i = 0; i < n; i++) {
+ key = tkey.get_key(i + n0);
+ assert(db->Add(key, key) == MBError::SUCCESS);
+ }
+ }
+ virtual void TearDown() {
+ if(db != NULL) {
+#ifndef __SHM_QUEUE__
+ db->UnsetAsyncWriterPtr(db_async);
+#endif
+ db->Close();
+ delete db;
+ db = NULL;
+ }
+ if(db_async != NULL) {
+ db_async->Close();
+ delete db_async;
+ db_async = NULL;
+ }
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+protected:
+ MBConfig mbconf;
+ DB *db_async;
+ DB *db;
+};
+
+TEST_F(EvictionTest, bucket_256_test)
+{
+ int entry_per_bucket = 256;
+ int num = 10000;
+ MBData mbd;
+ int rval;
+ std::string key;
+
+ OpenDB(entry_per_bucket);
+ Insert(0, num);
+ while(db->AsyncWriterBusy()) {
+ usleep(100);
+ }
+ TestKey tkey(MABAIN_TEST_KEY_TYPE_INT);
+ int index = 0;
+ for(int i = 0; i < num; i++) {
+ key = tkey.get_key(i);
+ assert(db->Find(key, mbd) == MBError::SUCCESS);
+ EXPECT_EQ(mbd.bucket_index, index);
+ if((i+1) % entry_per_bucket == 0)
+ index++;
+ }
+
+ //Prune by db size
+ rval = db->CollectResource(1000000000, 1000000000, 100, 10000000000);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ while(db->AsyncWriterBusy()) {
+ usleep(100);
+ }
+
+ for(int i = 0; i < num; i++) {
+ key = tkey.get_key(i);
+ rval = db->Find(key, mbd);
+ if(i < 5*entry_per_bucket) {
+ EXPECT_EQ(rval, MBError::NOT_EXIST);
+ } else {
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+ }
+
+ Insert(num, 1000);
+ num += 1000;
+ while(db->AsyncWriterBusy()) {
+ usleep(100);
+ }
+ //Prune by db count
+ rval = db->CollectResource(1000000000, 1000000000, 1000000000, 1000);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ while(db->AsyncWriterBusy()) {
+ usleep(100);
+ }
+ for(int i = entry_per_bucket; i < num; i++) {
+ key = tkey.get_key(i);
+ rval = db->Find(key, mbd);
+ if(i < 10*entry_per_bucket) {
+ EXPECT_EQ(rval, MBError::NOT_EXIST);
+ } else {
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+ }
+}
+
+#ifdef __SHM_QUEUE__
+TEST_F(EvictionTest, different_queue_size_test)
+{
+ mbconf.options = CONSTS::ACCESS_MODE_WRITER | CONSTS::ASYNC_WRITER_MODE;
+ db_async = new DB(mbconf);
+ assert(db_async->is_open());
+
+ mbconf.options = CONSTS::ACCESS_MODE_READER;
+ mbconf.queue_size = 99;
+ db = new DB(mbconf);
+ assert(!db->is_open());
+}
+#endif
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <list>
+
+#include <gtest/gtest.h>
+
+#include "../free_list.h"
+#include "../error.h"
+
+using namespace mabain;
+
+namespace {
+
+class FreeListTest : public ::testing::Test
+{
+public:
+ FreeListTest() {}
+ virtual ~FreeListTest() {}
+ virtual void SetUp() {}
+ virtual void TearDown() {}
+
+protected:
+};
+
+TEST_F(FreeListTest, AddBufferByIndex_test)
+{
+ int rval;
+ FreeList flist("./freelist", 4, 1000);
+
+ rval = flist.AddBufferByIndex(0, 100);
+ EXPECT_EQ(rval, 0);
+ EXPECT_EQ(flist.Count(), 1);
+ EXPECT_EQ(flist.GetBufferCountByIndex(0), 1u);
+
+ rval = flist.AddBufferByIndex(111, 128);
+ EXPECT_EQ(rval, 0);
+ EXPECT_EQ(flist.Count(), 2);
+ EXPECT_EQ(flist.GetBufferCountByIndex(111), 1u);
+}
+
+TEST_F(FreeListTest, RemoveBufferByIndex_test)
+{
+ FreeList flist("./freelist", 8, 2000);
+
+ flist.AddBufferByIndex(33, 72);
+ EXPECT_EQ(flist.RemoveBufferByIndex(33), 72u);
+
+
+ flist.AddBufferByIndex(44, 328);
+ flist.AddBufferByIndex(44, 1024);
+ flist.AddBufferByIndex(102, 8);
+ EXPECT_EQ(flist.RemoveBufferByIndex(44), 328u);
+ EXPECT_EQ(flist.RemoveBufferByIndex(102), 8u);
+ EXPECT_EQ(flist.RemoveBufferByIndex(44), 1024u);
+}
+
+TEST_F(FreeListTest, GetAlignmentSize_test)
+{
+ FreeList flist("./freelist", 8, 1222);
+
+ EXPECT_EQ(flist.GetAlignmentSize(128), 128);
+ EXPECT_EQ(flist.GetAlignmentSize(129), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(130), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(131), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(132), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(133), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(134), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(135), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(136), 136);
+ EXPECT_EQ(flist.GetAlignmentSize(137), 144);
+}
+
+TEST_F(FreeListTest, GetBufferIndex_test)
+{
+ FreeList flist("./freelist", 4, 222);
+
+ EXPECT_EQ(flist.GetBufferIndex(1), 0);
+ EXPECT_EQ(flist.GetBufferIndex(2), 0);
+ EXPECT_EQ(flist.GetBufferIndex(3), 0);
+ EXPECT_EQ(flist.GetBufferIndex(4), 0);
+ EXPECT_EQ(flist.GetBufferIndex(5), 1);
+}
+
+TEST_F(FreeListTest, GetBufferCountByIndex_test)
+{
+ FreeList flist("./freelist", 4, 333);
+
+ flist.AddBufferByIndex(7, 4);
+ flist.AddBufferByIndex(7, 24);
+ flist.AddBufferByIndex(7, 8);
+
+ EXPECT_EQ(flist.GetBufferCountByIndex(7), 3u);
+}
+
+TEST_F(FreeListTest, GetBufferSizeByIndex_test)
+{
+ FreeList flist("./freelist", 4, 333);
+
+ flist.AddBufferByIndex(13, 64);
+ flist.AddBufferByIndex(3, 28);
+ flist.AddBufferByIndex(101, 256);
+ flist.AddBufferByIndex(101, 516);
+
+ EXPECT_EQ(flist.GetBufferCountByIndex(13), 1u);
+ EXPECT_EQ(flist.GetBufferCountByIndex(3), 1u);
+ EXPECT_EQ(flist.GetBufferCountByIndex(101), 2u);
+}
+
+TEST_F(FreeListTest, ReleaseBuffer_test)
+{
+ int rval;
+ FreeList flist("./freelist", 4, 444);
+
+ rval = flist.ReleaseBuffer(16, 8);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = flist.ReleaseBuffer(124, 34);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(flist.GetBufferCountByIndex(flist.GetBufferIndex(8)), 1u);
+ EXPECT_EQ(flist.GetBufferCountByIndex(flist.GetBufferIndex(34)), 1u);
+}
+
+TEST_F(FreeListTest, AddBuffer_test)
+{
+ int rval;
+ FreeList flist("./freelist", 4, 555);
+
+ rval = flist.AddBuffer(40, 34);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(flist.GetBufferCountByIndex(flist.GetBufferIndex(34)), 1u);
+ rval =flist. AddBuffer(144, 34);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(flist.GetBufferCountByIndex(flist.GetBufferIndex(34)), 2u);
+ rval = flist.AddBuffer(196, 35);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(flist.GetBufferCountByIndex(flist.GetBufferIndex(34)), 3u);
+
+ rval = flist.AddBuffer(260, 135);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(flist.GetBufferCountByIndex(flist.GetBufferIndex(135)), 1u);
+}
+
+TEST_F(FreeListTest, RemoveBuffer_test)
+{
+ int rval;
+ FreeList flist("./freelist", 4, 555);
+ size_t offset;
+
+ flist.AddBufferByIndex(98, 96);
+ flist.AddBufferByIndex(98, 196);
+ rval = flist.RemoveBuffer(offset, flist.GetBufferSizeByIndex(98));
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 96u);
+ rval = flist.RemoveBuffer(offset, flist.GetBufferSizeByIndex(98));
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 196u);
+ rval = flist.RemoveBuffer(offset, flist.GetBufferSizeByIndex(98));
+ EXPECT_EQ(rval, MBError::NO_MEMORY);
+}
+
+TEST_F(FreeListTest, GetTotSize_test)
+{
+ FreeList flist("./freelist", 4, 555);
+ size_t tot = 0;
+
+ flist.AddBuffer(8, 183);
+ tot += flist.GetAlignmentSize(183);
+ flist.AddBuffer(240, 44);
+ tot += flist.GetAlignmentSize(44);
+ flist.AddBufferByIndex(33, 1024);
+ tot += flist.GetBufferSizeByIndex(33);
+ flist.AddBufferByIndex(133, 2056);
+ tot += flist.GetBufferSizeByIndex(133);
+ EXPECT_EQ(flist.GetTotSize(), tot);
+ EXPECT_EQ(flist.Count(), 4);
+}
+
+TEST_F(FreeListTest, StoreLoadEmpty_test)
+{
+ int rval;
+ FreeList flist("./freelist", 4, 555);
+
+ rval = flist.StoreListOnDisk();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = access("./freelist", R_OK);
+ EXPECT_EQ(rval, -1);
+}
+
+TEST_F(FreeListTest, StoreLoadSmallFilling_test)
+{
+ int rval;
+ size_t offset;
+ FreeList flist("./freelist", 4, 555);
+
+ flist.AddBuffer(32, 17);
+ flist.AddBuffer(84, 11);
+ flist.AddBuffer(144, 23);
+ flist.AddBuffer(444, 3);
+ rval = flist.StoreListOnDisk();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = access("./freelist", F_OK);
+ EXPECT_EQ(rval, 0);
+ EXPECT_EQ(flist.Count(), 0);
+ EXPECT_EQ(flist.GetTotSize(), 0u);
+
+ flist.LoadListFromDisk();
+ rval = access("./freelist", R_OK);
+ EXPECT_EQ(rval, -1);
+ rval = flist.RemoveBuffer(offset, 11);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 84u);
+ rval = flist.RemoveBuffer(offset, 23);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 144u);
+ rval = flist.RemoveBuffer(offset, 17);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 32u);
+ rval = flist.RemoveBuffer(offset, 2);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 444u);
+}
+
+TEST_F(FreeListTest, StoreLoadFullFilling_test)
+{
+ int rval;
+ size_t offset;
+ int size;
+ int num_buff = 1011;
+ FreeList flist("./freelist", 4, num_buff);
+ std::list<size_t> buff_list;
+
+ srand(time(NULL));
+
+ offset = 0;
+ for(int i = 0; i < num_buff; i++)
+ {
+ size = rand() % 111 + 2;
+ flist.AddBuffer(offset, size);
+ buff_list.push_back(offset);
+ buff_list.push_back(size);
+ offset += flist.GetAlignmentSize(size);
+ }
+
+ rval = flist.StoreListOnDisk();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = access("./freelist", F_OK);
+ EXPECT_EQ(rval, 0);
+ EXPECT_EQ(flist.Count(), 0);
+ EXPECT_EQ(flist.GetTotSize(), 0u);
+
+ flist.LoadListFromDisk();
+ rval = access("./freelist", R_OK);
+ EXPECT_EQ(rval, -1);
+
+ for (std::list<size_t>::iterator it = buff_list.begin(); it != buff_list.end(); ++it)
+ {
+ offset = *it;
+ it++;
+ size = *it;
+ rval = flist.RemoveBuffer(offset, size);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+ size = 34;
+ rval = flist.RemoveBuffer(offset, size);
+ EXPECT_EQ(rval, MBError::NO_MEMORY);
+}
+
+TEST_F(FreeListTest, StoreLoadHalfFilling_test)
+{
+ int rval;
+ size_t offset;
+ int size;
+ int num_buff = 1000;
+ FreeList flist("./freelist", 4, num_buff);
+ std::list<size_t> buff_list;
+
+ srand(time(NULL));
+
+ offset = 24;
+ for(int i = 0; i < num_buff; i++)
+ {
+ size = rand() % 98 + 1;
+ if(i % 2 == 0)
+ {
+ flist.AddBuffer(offset, size);
+ buff_list.push_back(offset);
+ buff_list.push_back(size);
+ }
+ offset += flist.GetAlignmentSize(size);
+ }
+
+ rval = flist.StoreListOnDisk();
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = access("./freelist", F_OK);
+ EXPECT_EQ(rval, 0);
+ EXPECT_EQ(flist.Count(), 0);
+ EXPECT_EQ(flist.GetTotSize(), 0u);
+
+ flist.LoadListFromDisk();
+ rval = access("./freelist", R_OK);
+ EXPECT_EQ(rval, -1);
+
+ for (std::list<size_t>::iterator it = buff_list.begin(); it != buff_list.end(); ++it)
+ {
+ offset = *it;
+ it++;
+ size = *it;
+ rval = flist.RemoveBuffer(offset, size);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+ size = 34;
+ rval = flist.RemoveBuffer(offset, size);
+ EXPECT_EQ(rval, MBError::NO_MEMORY);
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "../lock_free.h"
+#include "../mabain_consts.h"
+#include "../error.h"
+#include "../drm_base.h"
+
+using namespace mabain;
+
+namespace {
+
+class LockFreeTest : public ::testing::Test
+{
+public:
+ LockFreeTest() {
+ memset(&lock_free_data, 0, sizeof(lock_free_data));
+ memset(&header, 0, sizeof(header));
+ lfree.LockFreeInit(&lock_free_data, &header, CONSTS::ACCESS_MODE_WRITER);
+ }
+ virtual ~LockFreeTest() {
+ }
+
+ virtual void SetUp() {
+ }
+ virtual void TearDown() {
+ }
+
+protected:
+ LockFreeShmData lock_free_data;
+ LockFree lfree;
+ IndexHeader header;
+};
+
+TEST_F(LockFreeTest, WriterLockFreeStart_test)
+{
+ lfree.WriterLockFreeStart(101);
+ EXPECT_EQ(lock_free_data.counter, 0u);
+ EXPECT_EQ(lock_free_data.offset, 101u);
+}
+
+TEST_F(LockFreeTest, WriterLockFreeStop_test)
+{
+ lock_free_data.counter = 0xFFFFFFFF;
+ lfree.WriterLockFreeStart(102);
+ lfree.WriterLockFreeStop();
+ EXPECT_EQ(lock_free_data.counter, 0u);
+ EXPECT_EQ(lock_free_data.offset_cache[0xFFFFFFFF % MAX_OFFSET_CACHE], 102u);
+}
+
+TEST_F(LockFreeTest, ReaderLockFreeStart_test)
+{
+ lock_free_data.counter = 99;
+ lock_free_data.offset = 12345;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = i*100 + 3;
+ }
+
+ LockFreeData snapshot;
+ lfree.ReaderLockFreeStart(snapshot);
+ EXPECT_EQ(snapshot.counter, 99u);
+}
+
+TEST_F(LockFreeTest, ReaderLockFreeStop_test)
+{
+ lock_free_data.counter = 99;
+ lock_free_data.offset = 12345;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = (i + 1)*1000 + 233;
+ }
+
+ size_t offset;
+ int rval;
+ MBData mbd;
+
+ offset = 100001;
+ LockFreeData snapshot;
+ lfree.WriterLockFreeStart(offset);
+ lfree.WriterLockFreeStop();
+ lfree.ReaderLockFreeStart(snapshot);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset+1000, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+}
+
+TEST_F(LockFreeTest, ReaderLockFreeStop_test_1)
+{
+ lock_free_data.counter = 999;
+ lock_free_data.offset = 12345;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = (i + 1)*1000 + 48213;
+ }
+
+ size_t offset;
+ int rval;
+ MBData mbd;
+
+ offset = 510036;
+ LockFreeData snapshot;
+ lfree.WriterLockFreeStart(offset);
+ lfree.ReaderLockFreeStart(snapshot);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset, mbd);
+ EXPECT_EQ(rval, MBError::TRY_AGAIN);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset+1000, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ lfree.WriterLockFreeStop();
+}
+
+TEST_F(LockFreeTest, ReaderLockFreeStop_test_2)
+{
+ lock_free_data.counter = 999;
+ lock_free_data.offset = 12345;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = (i + 1)*1000 + 48213;
+ }
+
+ size_t offset;
+ int rval;
+ MBData mbd;
+
+ offset = 510036;
+ LockFreeData snapshot;
+ lfree.ReaderLockFreeStart(snapshot);
+ lfree.WriterLockFreeStart(offset);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset,mbd);
+ EXPECT_EQ(rval, MBError::TRY_AGAIN);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset+1000, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ lfree.WriterLockFreeStop();
+}
+
+TEST_F(LockFreeTest, ReaderLockFreeStop_test_3)
+{
+ lock_free_data.counter = 999;
+ lock_free_data.offset = 12345;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = (i + 1)*1000 + 48213;
+ }
+
+ size_t offset;
+ int rval;
+ MBData mbd;
+
+ offset = 510036;
+ LockFreeData snapshot;
+ lfree.ReaderLockFreeStart(snapshot);
+ lfree.WriterLockFreeStart(offset);
+ lfree.WriterLockFreeStop();
+ lfree.WriterLockFreeStart(offset+1000);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset, mbd);
+ EXPECT_EQ(rval, MBError::TRY_AGAIN);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset+1000, mbd);
+ EXPECT_EQ(rval, MBError::TRY_AGAIN);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset+2000, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ lfree.WriterLockFreeStop();
+}
+
+TEST_F(LockFreeTest, ReaderLockFreeStop_test_4)
+{
+ lock_free_data.counter = 999;
+ lock_free_data.offset = 12345;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = (i + 1)*1000 + 48213;
+ }
+
+ size_t offset;
+ int rval;
+ MBData mbd;
+
+ offset = 510036;
+ LockFreeData snapshot;
+ lfree.ReaderLockFreeStart(snapshot);
+
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lfree.WriterLockFreeStart(offset + i*1000);
+ if(i < MAX_OFFSET_CACHE-1) {
+ lfree.WriterLockFreeStop();
+ }
+ }
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ rval = lfree.ReaderLockFreeStop(snapshot, offset + i*1000, mbd);
+ EXPECT_EQ(rval, MBError::TRY_AGAIN);
+ }
+ rval = lfree.ReaderLockFreeStop(snapshot, offset + 1100, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ lfree.WriterLockFreeStop();
+}
+
+TEST_F(LockFreeTest, ReaderLockFreeStop_test_5)
+{
+ lock_free_data.counter = 999;
+ lock_free_data.offset = 12345;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = (i + 1)*1000 + 48213;
+ }
+
+ size_t offset;
+ int rval;
+ MBData mbd;
+
+ offset = 510036;
+ LockFreeData snapshot;
+ lfree.ReaderLockFreeStart(snapshot);
+
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lfree.WriterLockFreeStart(offset + i*1000);
+ lfree.WriterLockFreeStop();
+ }
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ rval = lfree.ReaderLockFreeStop(snapshot, offset + i*1000, mbd);
+ EXPECT_EQ(rval, MBError::TRY_AGAIN);
+ }
+ rval = lfree.ReaderLockFreeStop(snapshot, offset + 1100, mbd);
+ EXPECT_EQ(rval, MBError::TRY_AGAIN);
+ lfree.WriterLockFreeStop();
+}
+
+// Test for https://github.com/chxdeng/mabain/issues/35
+TEST_F(LockFreeTest, ReaderLockFreeStop_test_6)
+{
+ lock_free_data.counter = 232811;
+ lock_free_data.offset = 54321;
+ for(int i = 0; i < MAX_OFFSET_CACHE; i++) {
+ lock_free_data.offset_cache[i] = (i + 1)*1000 + 148213;
+ }
+
+ size_t offset;
+ int rval;
+ MBData mbd;
+
+ offset = 54321;
+ LockFreeData snapshot;
+
+ // edge added
+ lfree.ReaderLockFreeStart(snapshot);
+ header.excep_updating_status = EXCEP_STATUS_ADD_EDGE;
+ lfree.WriterLockFreeStart(offset);
+
+ rval = lfree.ReaderLockFreeStop(snapshot, offset, mbd);
+ EXPECT_EQ(MBError::TRY_AGAIN, rval);
+ EXPECT_TRUE(mbd.options & CONSTS::OPTION_READ_SAVED_EDGE);
+
+ lfree.ReaderLockFreeStart(snapshot);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset, mbd);
+ EXPECT_EQ(MBError::SUCCESS, rval);
+ EXPECT_FALSE(mbd.options & CONSTS::OPTION_READ_SAVED_EDGE);
+
+ // edge remooved
+ header.excep_updating_status = EXCEP_STATUS_REMOVE_EDGE;
+ lfree.WriterLockFreeStart(offset);
+ lfree.ReaderLockFreeStart(snapshot);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset, mbd);
+ EXPECT_EQ(MBError::TRY_AGAIN, rval);
+ EXPECT_TRUE(mbd.options & CONSTS::OPTION_READ_SAVED_EDGE);
+
+ lfree.ReaderLockFreeStart(snapshot);
+ rval = lfree.ReaderLockFreeStop(snapshot, offset, mbd);
+ EXPECT_EQ(MBError::SUCCESS, rval);
+ EXPECT_FALSE(mbd.options & CONSTS::OPTION_READ_SAVED_EDGE);
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <gtest/gtest.h>
+#include <gtest/tap.h>
+
+#include "../db.h"
+#define MB_DIR "/var/tmp/mabain_test/"
+
+GTEST_API_ int main(int argc, char **argv)
+{
+ testing::InitGoogleTest(&argc, argv);
+ testing::TestEventListeners& listeners = testing::UnitTest::GetInstance()->listeners();
+ listeners.Append(new tap::TapListener());
+
+ mode_t mode = 0777;
+ mkdir(MB_DIR, mode);
+
+ mabain::DB::SetLogFile("/var/tmp/mabain_test/mabain.log");
+ int rval = RUN_ALL_TESTS();
+
+ mabain::DB::CloseLogFile();
+ return rval;
+}
+
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <gtest/gtest.h>
+
+#include "../db.h"
+#include "../resource_pool.h"
+
+using namespace mabain;
+
+namespace {
+
+class MemoryOnlyTest : public ::testing::Test
+{
+public:
+ MemoryOnlyTest() {
+ }
+ virtual ~MemoryOnlyTest() {
+ }
+
+ virtual void SetUp() {
+ db = nullptr;
+ }
+ virtual void TearDown() {
+ if(db != nullptr) {
+ db->Close();
+ delete db;
+ }
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+protected:
+ DB *db;
+};
+
+TEST_F(MemoryOnlyTest, MemoryOnlyTest_test_constructor_1)
+{
+ int options = CONSTS::WriterOptions() | CONSTS::MEMORY_ONLY_MODE;
+ db = new DB("test_constructor_1", options);
+ EXPECT_EQ(MBError::SUCCESS, db->Status());
+}
+
+TEST_F(MemoryOnlyTest, MemoryOnlyTest_test_constructor_2)
+{
+ int options = CONSTS::ReaderOptions() | CONSTS::MEMORY_ONLY_MODE;
+ db = new DB("test_constructor_2", options);
+ EXPECT_EQ(MBError::NO_DB, db->Status());
+}
+
+TEST_F(MemoryOnlyTest, MemoryOnlyTest_test_add)
+{
+ int options = CONSTS::WriterOptions() | CONSTS::MEMORY_ONLY_MODE;
+ db = new DB("test_add", options);
+
+ int num = 10000;
+ std::string key;
+ for(int i = 0; i < num; i++) {
+ key = std::string("test_add_") + std::to_string(i);
+ db->Add(key, key);
+ }
+
+ MBData mbd;
+ int rval;
+ std::string value;
+ for(int i = 0; i < num; i++) {
+ key = std::string("test_add_") + std::to_string(i);
+ rval = db->Find(key, mbd);
+ EXPECT_EQ(MBError::SUCCESS, rval);
+ value = std::string((char *)mbd.buff, mbd.data_len);
+ EXPECT_EQ(key, value);
+ }
+}
+
+TEST_F(MemoryOnlyTest, MemoryOnlyTest_test_async)
+{
+ int options = CONSTS::WriterOptions() | CONSTS::MEMORY_ONLY_MODE;
+ options |= CONSTS::ASYNC_WRITER_MODE;
+ db = new DB("test_async", options);
+
+ options = CONSTS::ReaderOptions() | CONSTS::MEMORY_ONLY_MODE;
+ DB db_r = DB("test_async", options);
+
+ db_r.Close();
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <iostream>
+#include <openssl/sha.h>
+#include <sys/time.h>
+
+#include <gtest/gtest.h>
+
+#define TEST_MBSHRINK_FRIENDS \
+ friend class MBShrinkTest;
+
+#include "../db.h"
+#include "../mabain_consts.h"
+#include "../mb_rc.h"
+#include "../error.h"
+#include "./test_key.h"
+#include "../resource_pool.h"
+
+#define DB_DIR "/var/tmp/mabain_test/"
+#define KEY_TYPE_INT 0
+#define KEY_TYPE_SHA256 1
+
+using namespace mabain;
+
+namespace {
+
+class ResourceCollectionTest : public ::testing::Test
+{
+public:
+ ResourceCollectionTest() : db(NULL) {
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ srand(time(NULL));
+ }
+ virtual ~ResourceCollectionTest() {
+ if(db != NULL) delete db;
+ }
+ virtual void SetUp() {
+ std::string cmd = std::string("mkdir -p ") + DB_DIR;
+ if(system(cmd.c_str()) != 0) {
+ }
+ cmd = std::string("rm ") + DB_DIR + "_*";
+ if(system(cmd.c_str()) != 0) {
+ }
+ db = new DB(DB_DIR, CONSTS::ACCESS_MODE_WRITER, 128ULL*1024*1024, 128ULL*1024*1024);
+ if(!db->is_open()) {
+ std::cerr << "failed to open mabain db: " << db->StatusStr() << "\n";
+ abort();
+ }
+ }
+ virtual void TearDown() {
+ db->Close();
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+ void Populate(long num, bool *exist) {
+ TestKey tkey = TestKey(key_type);
+ for(long i = 0; i < num; i++) {
+ std::string key = tkey.get_key(i);
+ std::string value = key;
+ int rval = db->Add(key.c_str(), key.length(), value.c_str(), value.length());
+ assert(rval == MBError::SUCCESS);
+ exist[i] = true;
+ }
+ }
+
+ void DeleteRandom(long num, bool *exist) {
+ long count = 0;
+ int64_t tot_count = db->Count();
+ TestKey tkey = TestKey(key_type);
+ while(true) {
+ long ikey = rand() % tot_count;
+ std::string key = tkey.get_key(ikey);
+ int rval = db->Remove(key.c_str(), key.length());
+ if(rval == MBError::SUCCESS) {
+ exist[ikey] = false;
+ count++;
+ if(count >= num) break;
+ }
+ }
+ }
+
+ void DeleteOdd(long num, bool *exist) {
+ TestKey tkey = TestKey(key_type);
+ for(long i = 0; i < num; i++) {
+ if(i % 2 == 0) continue;
+ std::string key = tkey.get_key(i);
+ std::string value = key;
+ db->Remove(key.c_str(), key.length());
+ exist[i] = false;
+ }
+ }
+
+ void DeleteRange(long start, long end, bool *exist) {
+ TestKey tkey = TestKey(key_type);
+ for(long i = start; i < end; i++) {
+ std::string key = tkey.get_key(i);
+ std::string value = key;
+ db->Remove(key.c_str(), key.length());
+ exist[i] = false;
+ }
+ }
+
+ void VerifyKeyValue(long ikey, bool found) {
+ TestKey tkey = TestKey(key_type);
+ std::string key = tkey.get_key(ikey);
+ std::string value = key;
+ MBData mbd;
+ int rval = db->Find(key.c_str(), key.length(), mbd);
+ if(found) {
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_TRUE(value == std::string(reinterpret_cast<char*>(mbd.buff), mbd.data_len));
+ } else {
+ EXPECT_EQ(rval, MBError::NOT_EXIST);
+ }
+ }
+
+protected:
+ DB *db;
+ int key_type;
+};
+
+TEST_F(ResourceCollectionTest, RC_reorder_index_test)
+{
+ key_type = KEY_TYPE_INT;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_INDEX);
+
+ long tot = 53245;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+TEST_F(ResourceCollectionTest, RC_reorder_data_test)
+{
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_DATA);
+
+ long tot = 35275;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+TEST_F(ResourceCollectionTest, RC_reorder_index_data_test)
+{
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_256;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_INDEX |
+ RESOURCE_COLLECTION_TYPE_DATA);
+
+ long tot = 35275;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+TEST_F(ResourceCollectionTest, RC_delete_odd_collect_index_test)
+{
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_INDEX);
+
+ long tot = 128471;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ DeleteOdd(tot, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+TEST_F(ResourceCollectionTest, RC_delete_random_collect_data_test)
+{
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_DATA);
+
+ long tot = 34521;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ DeleteRandom(tot, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+TEST_F(ResourceCollectionTest, RC_delete_random_collect_index_data_test)
+{
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_256;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_INDEX |
+ RESOURCE_COLLECTION_TYPE_DATA);
+
+ long tot = 34521;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ DeleteRandom(tot, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+TEST_F(ResourceCollectionTest, RC_delete_range_collect_index_data_test)
+{
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_256;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_INDEX |
+ RESOURCE_COLLECTION_TYPE_DATA);
+
+ long tot = 34521;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ DeleteRange(0, 23, exist);
+ DeleteRange(1110, 1118, exist);
+ DeleteRange(29110, 29301, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+TEST_F(ResourceCollectionTest, RC_delete_random_collect_index_data_add_test)
+{
+ key_type = MABAIN_TEST_KEY_TYPE_SHA_128;
+ ResourceCollection rc(*db, RESOURCE_COLLECTION_TYPE_INDEX |
+ RESOURCE_COLLECTION_TYPE_DATA);
+
+ long tot = 55569;
+ bool *exist = new bool[tot];
+ Populate(tot, exist);
+ DeleteRandom(tot, exist);
+ rc.ReclaimResource(0, 0, 10000000000LL, 10000000000LL);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ db->Close();
+ delete db;
+
+ db = new DB(DB_DIR, CONSTS::ACCESS_MODE_WRITER, 128ULL*1024*1024, 128ULL*1024*1024);
+ if(!db->is_open()) {
+ std::cerr << "failed top open db\n";
+ exit(0);
+ }
+
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ key_type = MABAIN_TEST_KEY_TYPE_INT;
+ tot = 12343;
+ Populate(tot, exist);
+ for(long i = 0; i < tot; i++) {
+ VerifyKeyValue(i, exist[i]);
+ }
+
+ delete [] exist;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "../rollable_file.h"
+#include "../mabain_consts.h"
+#include "../error.h"
+#include "../resource_pool.h"
+
+using namespace mabain;
+
+namespace {
+
+#define ROLLABLE_FILE_TEST_DIR "/var/tmp/mabain_test"
+#define FAKE_DATA "dsklckk sldk&&sdijds8990s9090230290399&&^^%%sdhsjdhsjdhsjxnmzn lkvlsdlq;';'a;b; ;;slv; ;;;sdfl; lls;lf;sld;sld;sld;sll;skl;klk;gk;akl;s"
+#define ONE_MEGA 1024*1024ul
+
+class RollableFileTest : public ::testing::Test
+{
+public:
+ RollableFileTest() {
+ rfile = NULL;
+ }
+ virtual ~RollableFileTest() {
+ if(rfile != NULL)
+ delete rfile;
+ }
+
+ virtual void SetUp() {
+ std::string cmd = std::string("mkdir -p ") + ROLLABLE_FILE_TEST_DIR;
+ if(system(cmd.c_str()) != 0) {
+ }
+ }
+ virtual void TearDown() {
+ ResourcePool::getInstance().RemoveAll();
+ std::string cmd = std::string("rm -rf ") + ROLLABLE_FILE_TEST_DIR + "/_*";
+ if(system(cmd.c_str()) != 0) {
+ }
+ }
+
+ void Init() {
+ }
+
+protected:
+ RollableFile *rfile;
+};
+
+TEST_F(RollableFileTest, constructor_test)
+{
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_d",
+ 4*ONE_MEGA, 4*ONE_MEGA, CONSTS::ACCESS_MODE_WRITER, 0);
+ EXPECT_EQ(rfile != NULL, true);
+ delete rfile;
+ ResourcePool::getInstance().RemoveAll();
+
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_d",
+ 4*ONE_MEGA, 4*ONE_MEGA, CONSTS::ACCESS_MODE_READER, 2);
+ EXPECT_EQ(rfile != NULL, true);
+}
+
+TEST_F(RollableFileTest, RandomWrite_test)
+{
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_d",
+ 4*ONE_MEGA, 4*ONE_MEGA, CONSTS::ACCESS_MODE_WRITER, 0);
+
+ EXPECT_EQ(rfile != NULL, true);
+
+ size_t nbytes;
+ size_t offset;
+ uint8_t *ptr;
+
+ nbytes = 5;
+ offset = 0;
+ rfile->Reserve(offset, nbytes, ptr);
+ nbytes = rfile->RandomWrite((const void *)FAKE_DATA, nbytes, offset);
+ EXPECT_EQ(nbytes, 5u);
+
+ uint8_t buff[256];
+ rfile->RandomRead(buff, nbytes, offset);
+ EXPECT_EQ(memcmp(buff, FAKE_DATA, nbytes)==0, true);
+
+ nbytes = 78;
+ offset = ONE_MEGA + 28372;
+ rfile->Reserve(offset, nbytes, ptr);
+ nbytes = rfile->RandomWrite((const void *)FAKE_DATA, nbytes, offset);
+ EXPECT_EQ(nbytes, 78u);
+ rfile->RandomRead(buff, nbytes, offset);
+ EXPECT_EQ(memcmp(buff, FAKE_DATA, nbytes)==0, true);
+}
+
+TEST_F(RollableFileTest, RandomRead_test)
+{
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_i",
+ 4*ONE_MEGA, 4*ONE_MEGA,
+ CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW, 0);
+
+ EXPECT_EQ(rfile != NULL, true);
+ std::atomic<size_t> sliding_addr;
+ rfile->InitShmSlidingAddr(&sliding_addr);
+
+ size_t nbytes;
+ size_t offset;
+ uint8_t *ptr;
+
+ nbytes = 15;
+ offset = 20;
+ rfile->Reserve(offset, nbytes, ptr);
+ nbytes = rfile->RandomWrite((const void *)FAKE_DATA, nbytes, offset);
+ EXPECT_EQ(nbytes, 15u);
+
+ uint8_t buff[256];
+ rfile->RandomRead(buff, nbytes, offset);
+ EXPECT_EQ(memcmp(buff, FAKE_DATA, nbytes)==0, true);
+
+ nbytes = 35;
+ offset = 8*ONE_MEGA + 28372;
+ rfile->Reserve(offset, nbytes, ptr);
+ nbytes = rfile->RandomWrite((const void *)FAKE_DATA, nbytes, offset);
+ EXPECT_EQ(nbytes, 35u);
+ rfile->RandomRead(buff, nbytes, offset);
+ EXPECT_EQ(memcmp(buff, FAKE_DATA, nbytes)==0, true);
+}
+
+TEST_F(RollableFileTest, Reserve_test)
+{
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_i",
+ 4*ONE_MEGA, 4*ONE_MEGA,
+ CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW, 0);
+ EXPECT_EQ(rfile != NULL, true);
+
+ size_t offset;
+ int size;
+ uint8_t *ptr = NULL;
+ int rval;
+
+ offset = 0;
+ size = 34;
+ rval = rfile->Reserve(offset, size, ptr, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 0u);
+ EXPECT_EQ(ptr != NULL, true);
+
+ offset = 12123;
+ size = 19;
+ rval = rfile->Reserve(offset, size, ptr, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 12123u);
+ EXPECT_EQ(ptr != NULL, true);
+}
+
+TEST_F(RollableFileTest, GetShmPtr_test)
+{
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_i",
+ 4*ONE_MEGA, 4*ONE_MEGA,
+ CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW, 0);
+ EXPECT_EQ(rfile != NULL, true);
+
+ size_t offset;
+ int size;
+ uint8_t *ptr = NULL;
+ int rval;
+
+ offset = 42321;
+ size = 49;
+ rval = rfile->Reserve(offset, size, ptr, true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(offset, 42321u);
+ EXPECT_EQ(ptr != NULL, true);
+ EXPECT_EQ(ptr == rfile->GetShmPtr(offset, size), true);
+}
+
+TEST_F(RollableFileTest, CheckAlignment_test)
+{
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_i",
+ 4*ONE_MEGA, 4*ONE_MEGA, CONSTS::ACCESS_MODE_WRITER, 0);
+ EXPECT_EQ(rfile != NULL, true);
+
+ size_t offset;
+ int size;
+
+ offset = 1001;
+ size = 23;
+ offset = rfile->CheckAlignment(offset, size);
+ EXPECT_EQ(offset, 1001u);
+
+ offset = 4*ONE_MEGA-12;
+ size = 55;
+ offset = rfile->CheckAlignment(offset, size);
+ EXPECT_EQ(offset, 4*ONE_MEGA);
+}
+
+TEST_F(RollableFileTest, Flush_test)
+{
+ rfile = new RollableFile(std::string(ROLLABLE_FILE_TEST_DIR) + "/_mabain_i",
+ 4*ONE_MEGA, 4*ONE_MEGA,
+ CONSTS::ACCESS_MODE_WRITER | CONSTS::USE_SLIDING_WINDOW, 0);
+ EXPECT_EQ(rfile != NULL, true);
+ std::atomic<size_t> sliding_addr;
+ rfile->InitShmSlidingAddr(&sliding_addr);
+
+ int nbytes = 78;
+ size_t offset = ONE_MEGA + 28372;
+ uint8_t *ptr;
+ rfile->Reserve(offset, nbytes, ptr);
+ nbytes = rfile->RandomWrite((const void *)FAKE_DATA, nbytes, offset);
+ EXPECT_EQ(nbytes, 78);
+ offset = 4*ONE_MEGA + 233232;
+ nbytes = 101;
+ rfile->Reserve(offset, nbytes, ptr);
+ nbytes = rfile->RandomWrite((const void *)FAKE_DATA, nbytes, offset);
+ EXPECT_EQ(nbytes, 101);
+ rfile->Flush();
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <openssl/sha.h>
+
+#define MABAIN_TEST_KEY_TYPE_INT 0
+#define MABAIN_TEST_KEY_TYPE_SHA_128 1
+#define MABAIN_TEST_KEY_TYPE_SHA_256 2
+#define MABAIN_TEST_KEY_BUFF_SIZE 1024
+
+namespace {
+
+class TestKey
+{
+public:
+ TestKey(int ktype) {
+ key_type = ktype;
+ key_buff[0] = '\0';
+ }
+ ~TestKey() {
+ }
+ const char* get_key(int key) {
+ switch(key_type) {
+ case MABAIN_TEST_KEY_TYPE_INT:
+ snprintf(key_buff, MABAIN_TEST_KEY_BUFF_SIZE, "%d", key);
+ break;
+ case MABAIN_TEST_KEY_TYPE_SHA_128:
+ {
+ unsigned char hash[SHA_DIGEST_LENGTH];
+ SHA_CTX sha1;
+ SHA1_Init(&sha1);
+ SHA1_Update(&sha1, (unsigned char*)&key, 4);
+ SHA1_Final(hash, &sha1);
+ int i = 0;
+ for(i = 0; i < SHA_DIGEST_LENGTH; i++)
+ {
+ sprintf(key_buff + (i * 2), "%02x", hash[i]);
+ }
+ key_buff[32] = 0;
+ }
+ break;
+ case MABAIN_TEST_KEY_TYPE_SHA_256:
+ {
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ SHA256_CTX sha256;
+ SHA256_Init(&sha256);
+ SHA256_Update(&sha256, (unsigned char*)&key, 4);
+ SHA256_Final(hash, &sha256);
+ int i = 0;
+ for(i = 0; i < SHA256_DIGEST_LENGTH; i++)
+ {
+ sprintf(key_buff + (i * 2), "%02x", hash[i]);
+ }
+ key_buff[64] = 0;
+ }
+ break;
+ default:
+ abort();
+ }
+ return (const char *) key_buff;
+ }
+
+private:
+ int key_type;
+ char key_buff[MABAIN_TEST_KEY_BUFF_SIZE];
+};
+
+}
--- /dev/null
+#include <unistd.h>
+#include <stdlib.h>
+#include <list>
+#include <cstdlib>
+
+#include <gtest/gtest.h>
+
+#include "../db.h"
+#include "../mb_data.h"
+#include "../resource_pool.h"
+#include "./test_key.h"
+
+#define MB_DIR "/var/tmp/mabain_test/"
+
+using namespace mabain;
+
+namespace {
+
+class UpdateTest : public ::testing::Test
+{
+public:
+ UpdateTest() {
+ db = NULL;
+ }
+ virtual ~UpdateTest() {
+ if(db != NULL)
+ delete db;
+ }
+ virtual void SetUp() {
+ std::string cmd = std::string("mkdir -p ") + MB_DIR;
+ if(system(cmd.c_str()) != 0) {
+ }
+ cmd = std::string("rm ") + MB_DIR + "_*";
+ if(system(cmd.c_str()) != 0) {
+ }
+ db = new DB(MB_DIR, CONSTS::WriterOptions());
+ }
+ virtual void TearDown() {
+ db->Close();
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+protected:
+ DB *db;
+};
+
+TEST_F(UpdateTest, Update_all)
+{
+ TestKey tkey(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ int num = 1000;
+ std::string key;
+ int rval;
+ for(int i = 0; i < num; i++) {
+ key = tkey.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+ for(int i = 0; i < num; i++) {
+ key = tkey.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+
+ key = tkey.get_key(i);
+ rval = db->Add(key, key+"_new", true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key+"_new", true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+
+ MBData mbd;
+ for(int i = 0; i < num; i++) {
+ key = tkey.get_key(i);
+ rval = db->Find(key, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(std::string((const char*)mbd.buff, mbd.data_len)==key+"_new", true);
+ key = tkey1.get_key(i);
+ rval = db->Find(key, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(std::string((const char*)mbd.buff, mbd.data_len)==key+"_new", true);
+ }
+}
+
+TEST_F(UpdateTest, Update_random)
+{
+ srand(time(NULL));
+ TestKey tkey(MABAIN_TEST_KEY_TYPE_INT);
+ TestKey tkey1(MABAIN_TEST_KEY_TYPE_SHA_128);
+ int num = 3456;
+ std::string key;
+ int rval;
+ bool *added;
+
+ added = new bool[num];
+ for(int i = 0; i < num; i++) {
+ added[i] = false;
+ }
+
+ for(int i = 0; i < num; i++) {
+ if(rand() % 100 < 21)
+ continue;
+
+ key = tkey.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ added[i] = true;
+ }
+ for(int i = 0; i < num; i++) {
+ if(added[i]) {
+ key = tkey.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::IN_DICT);
+
+ key = tkey.get_key(i);
+ rval = db->Add(key, key+"_new", true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key+"_new", true);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ } else {
+ key = tkey.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ key = tkey1.get_key(i);
+ rval = db->Add(key, key);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ }
+ }
+
+ MBData mbd;
+ std::string value;
+ for(int i = 0; i < num; i++) {
+ key = tkey.get_key(i);
+ rval = db->Find(key, mbd);
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ value = key;
+ if(added[i])
+ value += "_new";
+ EXPECT_EQ(std::string((const char*)mbd.buff, mbd.data_len)==value, true);
+ key = tkey1.get_key(i);
+ rval = db->Find(key, mbd);
+ value = key;
+ if(added[i])
+ value += "_new";
+ EXPECT_EQ(rval, MBError::SUCCESS);
+ EXPECT_EQ(std::string((const char*)mbd.buff, mbd.data_len)==value, true);
+ }
+
+ delete [] added;
+}
+
+}
--- /dev/null
+#include <string>
+#include <thread>
+#include <iostream>
+
+#include <gtest/gtest.h>
+
+#include "../db.h"
+#include "../resource_pool.h"
+
+using namespace mabain;
+
+namespace {
+
+#define MB_DIR "/var/tmp/mabain_test/"
+
+class WriterLockTest : public ::testing::Test
+{
+public:
+ WriterLockTest() {
+ }
+ virtual ~WriterLockTest() {
+ }
+ virtual void SetUp() {
+ std::string cmd = std::string("rm ") + MB_DIR + "_*";
+ if(system(cmd.c_str()) != 0) {
+ }
+ }
+ virtual void TearDown() {
+ ResourcePool::getInstance().RemoveAll();
+ }
+
+protected:
+};
+
+TEST_F(WriterLockTest, test_lock)
+{
+ int options = CONSTS::WriterOptions();
+ DB db(MB_DIR, options);
+ EXPECT_TRUE(db.is_open());
+
+ DB db1(MB_DIR, options);
+ EXPECT_TRUE(db1.Status() == MBError::WRITER_EXIST);
+
+ db.Close();
+ DB db2(MB_DIR, options);
+ EXPECT_TRUE(db2.is_open());
+
+ options = CONSTS::ReaderOptions();
+ DB db3(MB_DIR, options);
+ EXPECT_TRUE(db3.is_open());
+
+ DB db4(MB_DIR, options);
+ EXPECT_TRUE(db4.is_open());
+
+ db4 = db3;
+ EXPECT_TRUE(db4.is_open());
+
+ DB db5(db4);
+ EXPECT_TRUE(db5.is_open());
+}
+
+// Multiple process lock test
+/****
+TEST_F(WriterLockTest, test_lock_process)
+{
+#define LOCK_WRITER_SOURCE " \
+#include <unistd.h>\n \
+#include \"../db.h\"\n \
+using namespace mabain;\n \
+int main(int argc, char *argv[]) { \
+ int time_to_run = 30; \
+ const char *db_dir = \"/var/tmp/mabain_test/\"; \
+ if(argc > 1) { \
+ db_dir = argv[1]; \
+ } \
+ if(argc > 2) { \
+ time_to_run = atoi(argv[2]); \
+ } \
+ int options = CONSTS::WriterOptions(); \
+ DB db(db_dir, options); \
+ if(db.is_open()) { \
+ int time_left = time_to_run; \
+ while(time_left-- > 0) { \
+ sleep(1); \
+ } \
+ } else { \
+ std::cerr << \"failed to open db \" << db_dir << std::endl; \
+ } \
+ db.Close(); \
+ return 0; \
+}"
+
+ std::ofstream cpp_source("./lock_writer.cpp");
+ cpp_source << LOCK_WRITER_SOURCE;
+ cpp_source.close();
+ std::string cmd = "g++ -o lock_writer lock_writer.cpp -L.. -lmabain";
+ system(cmd.c_str());
+ assert(access("./lock_writer", F_OK) == 0);
+
+ cmd = std::string("./lock_writer ") + MB_DIR + " 60 &";
+ system(cmd.c_str());
+ sleep(3);
+
+ int options = CONSTS::WriterOptions();
+ DB db(MB_DIR, options);
+ EXPECT_TRUE(db.Status() == MBError::WRITER_EXIST);
+ db.Close();
+
+ system("pkill -9 lock_writer");
+ sleep(3);
+ db = DB(MB_DIR, options);
+ EXPECT_TRUE(db.Status() == MBError::SUCCESS);
+
+ unlink("./lock_writer.cpp");
+ unlink("./lock_writer");
+
+}
+***/
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include "mb_lsq.h"
+#include "error.h"
+
+namespace mabain {
+
+MBlsq::MBlsq(void (*free_fn)(void *)) : FreeFunc(free_fn)
+{
+ head = NULL;
+ tail = NULL;
+ count = 0;
+}
+
+void MBlsq::Clear()
+{
+ LSQ_Node *node;
+ while(head != NULL)
+ {
+ node = head;
+ head = head->next;
+ if(FreeFunc)
+ FreeFunc(node->data.data_ptr);
+ free(node);
+ }
+
+ head = NULL;
+ tail = NULL;
+ count = 0;
+}
+
+MBlsq::~MBlsq()
+{
+ Clear();
+}
+
+uint64_t MBlsq::Count() const
+{
+ return count;
+}
+
+int MBlsq::AddToHead(void *ptr)
+{
+ LSQ_Node *node = (LSQ_Node *) malloc(sizeof(*node));
+ if(node == NULL)
+ return MBError::NO_MEMORY;
+
+ node->data.data_ptr = ptr;
+ node->next = head;
+ head = node;
+ if(count == 0)
+ tail = node;
+
+ count++;
+ return MBError::SUCCESS;
+}
+
+int MBlsq::AddIntToHead(int64_t value)
+{
+ LSQ_Node *node = (LSQ_Node *) malloc(sizeof(*node));
+ if(node == NULL)
+ return MBError::NO_MEMORY;
+
+ node->data.value = value;
+ node->next = head;
+ head = node;
+ if(count == 0)
+ tail = node;
+
+ count++;
+ return MBError::SUCCESS;
+}
+
+int MBlsq::AddIntToTail(int64_t value)
+{
+ LSQ_Node *node = (LSQ_Node *) malloc(sizeof(*node));
+ if(node == NULL)
+ return MBError::NO_MEMORY;
+
+ node->data.value = value;
+ node->next = NULL;
+ if(tail)
+ {
+ tail->next = node;
+ tail = node;
+ }
+ else
+ {
+ head = node;
+ tail = node;
+ }
+
+ count++;
+ return MBError::SUCCESS;
+}
+
+int MBlsq::AddToTail(void *ptr)
+{
+ LSQ_Node *node = (LSQ_Node *) malloc(sizeof(*node));
+ if(node == NULL)
+ return MBError::NO_MEMORY;
+
+ node->data.data_ptr = ptr;
+ node->next = NULL;
+ if(tail != NULL)
+ {
+ tail->next = node;
+ }
+ else
+ {
+ head = node;
+ }
+
+ tail = node;
+ count++;
+ return MBError::SUCCESS;
+}
+
+void* MBlsq::RemoveFromHead()
+{
+ void *data;
+
+ if(count > 1)
+ {
+ LSQ_Node *node = head;
+ data = head->data.data_ptr;
+ head = head->next;
+ count--;
+ free(node);
+ }
+ else if(count == 1)
+ {
+ LSQ_Node *node = head;
+ data = head->data.data_ptr;
+ head = NULL;
+ tail = NULL;
+ count = 0;
+ free(node);
+ }
+ else
+ {
+ data = NULL;
+ }
+
+ return data;
+}
+
+int64_t MBlsq::RemoveIntFromHead()
+{
+ int64_t value;
+
+ if(count > 1)
+ {
+ LSQ_Node *node = head;
+ value = head->data.value;
+ head = head->next;
+ count--;
+ free(node);
+ }
+ else if(count == 1)
+ {
+ LSQ_Node *node = head;
+ value = head->data.value;
+ head = NULL;
+ tail = NULL;
+ count = 0;
+ free(node);
+ }
+ else
+ {
+ value = 0;
+ }
+
+ return value;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __MBLSQ_H__
+#define __MBLSQ_H__
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+namespace mabain {
+
+typedef union _LSQ_Data
+{
+ void *data_ptr;
+ int64_t value;
+} LSQ_Data;
+typedef struct _LSQ_Node
+{
+ LSQ_Data data;
+ struct _LSQ_Node *next;
+} LSQ_Node;
+
+// mabain list/stack/queue class
+class MBlsq
+{
+public:
+ MBlsq(void (*free_fn)(void *));
+ ~MBlsq();
+
+ int AddToHead(void *ptr); // Stack
+ int AddToTail(void *ptr); // Queue
+ void *RemoveFromHead();
+ int AddIntToHead(int64_t value);
+ int AddIntToTail(int64_t value);
+ int64_t RemoveIntFromHead();
+ void Clear();
+ uint64_t Count() const;
+
+private:
+ LSQ_Node *head;
+ LSQ_Node *tail;
+ uint64_t count;
+ void (*FreeFunc)(void *);
+};
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <unistd.h>
+#include "shm_mutex.h"
+#include "../logger.h"
+#include "../error.h"
+
+namespace mabain {
+
+int InitShmMutex(pthread_mutex_t *mutex)
+{
+ if(mutex == NULL)
+ return MBError::INVALID_ARG;
+
+ pthread_mutexattr_t attr;
+ if(pthread_mutexattr_init(&attr))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "pthread_mutexkattr_init failed");
+ return MBError::MUTEX_ERROR;
+ }
+ if(pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "failed to set mutex/PTHREAD_PROCESS_SHARED");
+ pthread_mutexattr_destroy(&attr);
+ return MBError::MUTEX_ERROR;
+ }
+
+ // Cannot set mutex as robust because of the glibc bug
+ if(pthread_mutex_init(mutex, &attr))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "pthread_mutex_init failed");
+ pthread_mutexattr_destroy(&attr);
+ return MBError::MUTEX_ERROR;
+ }
+ pthread_mutexattr_destroy(&attr);
+
+ return MBError::SUCCESS;
+}
+
+int InitShmRWLock(pthread_rwlock_t *lock)
+{
+ if(lock == NULL)
+ return MBError::INVALID_ARG;
+
+ pthread_rwlockattr_t attr;
+ if(pthread_rwlockattr_init(&attr))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "pthread_rwlockattr_init failed");
+ return MBError::MUTEX_ERROR;
+ }
+ if(pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "failed to set rwlock/PTHREAD_PROCESS_SHARED");
+ pthread_rwlockattr_destroy(&attr);
+ return MBError::MUTEX_ERROR;
+ }
+
+ if(pthread_rwlock_init(lock, &attr))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "pthread_rwlock_init failed");
+ pthread_rwlockattr_destroy(&attr);
+ return MBError::MUTEX_ERROR;
+ }
+ pthread_rwlockattr_destroy(&attr);
+
+ return MBError::SUCCESS;
+}
+
+int InitShmCond(pthread_cond_t *cond)
+{
+ if(cond == NULL)
+ return MBError::INVALID_ARG;
+
+ pthread_condattr_t attr;
+ if(pthread_condattr_init(&attr))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "pthread_condattr_init failed");
+ return MBError::MUTEX_ERROR;
+ }
+ if(pthread_condattr_setpshared(&attr, PTHREAD_PROCESS_SHARED))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "failed to set cond/PTHREAD_PROCESS_SHARED");
+ pthread_condattr_destroy(&attr);
+ return MBError::MUTEX_ERROR;
+ }
+
+ if(pthread_cond_init(cond, &attr))
+ {
+ Logger::Log(LOG_LEVEL_WARN, "pthread_cond_init failed");
+ pthread_condattr_destroy(&attr);
+ return MBError::MUTEX_ERROR;
+ }
+ pthread_condattr_destroy(&attr);
+
+ return MBError::SUCCESS;
+}
+
+#ifdef __APPLE__
+// pthread_mutex_timedlock is not supported in Mac OS.
+// Simple implementation using pthread_mutex_trylock
+#define MB_PTHREAD_MUTEX_TIMEDLOCK_SLEEP_INTERVAL 100L // in nano seconds
+int pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abs_timeout)
+{
+ int rval;
+ while(true)
+ {
+ rval = pthread_mutex_trylock(mutex);
+ if(rval != EBUSY) break;
+ if(time(0) >= abs_timeout->tv_sec) break;
+ nanosleep((const struct timespec[]){{0, MB_PTHREAD_MUTEX_TIMEDLOCK_SLEEP_INTERVAL}}, NULL);
+ }
+ if(rval == EBUSY) rval = ETIMEDOUT;
+ return rval;
+}
+#endif
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __SHM_MUTEX_H__
+#define __SHM_MUTEX_H__
+
+#include <pthread.h>
+
+namespace mabain {
+
+int InitShmMutex(pthread_mutex_t *mutex);
+int InitShmRWLock(pthread_rwlock_t *lock);
+int InitShmCond(pthread_cond_t *cond);
+
+#ifdef __APPLE__
+int pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abs_timeout);
+#endif
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#include <string>
+#include <iostream>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "error.h"
+
+namespace mabain {
+
+int acquire_writer_lock(const std::string &lock_file_path)
+{
+ int fd = open(lock_file_path.c_str(), O_WRONLY | O_CREAT,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+ if(fd < 0)
+ {
+ std::cerr << "failed to open lock file " << lock_file_path
+ << " errno: " << errno << std::endl;
+ return fd;
+ }
+
+ struct flock writer_lock;
+ writer_lock.l_type = F_WRLCK;
+ writer_lock.l_start = 0;
+ writer_lock.l_whence = SEEK_SET;
+ writer_lock.l_len = 0;
+ if(fcntl(fd, F_SETLK, &writer_lock) != 0)
+ {
+ std::cerr << "failed to lock file " << lock_file_path
+ << " errno: " << errno << std::endl;
+ close(fd);
+ return -1;
+ }
+
+ return fd;
+}
+
+void release_writer_lock(int &fd)
+{
+ if(fd < 0)
+ return;
+ close(fd);
+ fd = -1;
+}
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2018 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __UTILS_H__
+#define __UTILS_H__
+
+#include <string>
+
+namespace mabain {
+
+int acquire_writer_lock(const std::string &lock_file_path);
+void release_writer_lock(int &fd);
+
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2017 Cisco Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// @author Changxue Deng <chadeng@cisco.com>
+
+#ifndef __VERSION_H__
+#define __VERSION_H__
+
+#include <stdint.h>
+
+namespace mabain {
+
+// version[0] is the major version.
+// version[1] is the minor version.
+// version[2] is the patch version.
+extern uint16_t version[4];
+
+}
+
+#endif
projects / c11concurrency-benchmarks.git / commitdiff