2 * Copyright 2016 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef __STDC_FORMAT_MACROS
18 #define __STDC_FORMAT_MACROS
21 #include <folly/io/async/EventBase.h>
23 #include <folly/ThreadName.h>
24 #include <folly/io/async/NotificationQueue.h>
26 #include <condition_variable>
34 using folly::EventBase;
36 class FunctionLoopCallback : public EventBase::LoopCallback {
38 explicit FunctionLoopCallback(EventBase::Func&& function)
39 : function_(std::move(function)) {}
41 void runLoopCallback() noexcept override {
47 EventBase::Func function_;
54 * EventBase::FunctionRunner
57 class EventBase::FunctionRunner
58 : public NotificationQueue<EventBase::Func>::Consumer {
60 void messageAvailable(Func&& msg) override {
61 // In libevent2, internal events do not break the loop.
62 // Most users would expect loop(), followed by runInEventBaseThread(),
63 // to break the loop and check if it should exit or not.
64 // To have similar bejaviour to libevent1.4, tell the loop to break here.
65 // Note that loop() may still continue to loop, but it will also check the
66 // stop_ flag as well as runInLoop callbacks, etc.
67 event_base_loopbreak(getEventBase()->evb_);
70 // terminateLoopSoon() sends a null message just to
71 // wake up the loop. We can ignore these messages.
75 // The function should never throw an exception, because we have no
76 // way of knowing what sort of error handling to perform.
78 // If it does throw, log a message and abort the program.
81 } catch (const std::exception& ex) {
82 LOG(ERROR) << "runInEventBaseThread() function threw a "
83 << typeid(ex).name() << " exception: " << ex.what();
86 LOG(ERROR) << "runInEventBaseThread() function threw an exception";
93 * EventBase::CobTimeout methods
96 void EventBase::CobTimeout::timeoutExpired() noexcept {
97 // For now, we just swallow any exceptions that the callback threw.
100 } catch (const std::exception& ex) {
101 LOG(ERROR) << "EventBase::runAfterDelay() callback threw "
102 << typeid(ex).name() << " exception: " << ex.what();
104 LOG(ERROR) << "EventBase::runAfterDelay() callback threw non-exception "
108 // The CobTimeout object was allocated on the heap by runAfterDelay(),
109 // so delete it now that the it has fired.
114 // The interface used to libevent is not thread-safe. Calls to
115 // event_init() and event_base_free() directly modify an internal
116 // global 'current_base', so a mutex is required to protect this.
118 // event_init() should only ever be called once. Subsequent calls
119 // should be made to event_base_new(). We can recognise that
120 // event_init() has already been called by simply inspecting current_base.
121 static std::mutex libevent_mutex_;
127 EventBase::EventBase(bool enableTimeMeasurement)
128 : runOnceCallbacks_(nullptr)
134 , avgLoopTime_(2000000)
135 , maxLatencyLoopTime_(avgLoopTime_)
136 , enableTimeMeasurement_(enableTimeMeasurement)
137 , nextLoopCnt_(-40) // Early wrap-around so bugs will manifest soon
138 , latestLoopCnt_(nextLoopCnt_)
141 , observerSampleCount_(0)
142 , executionObserver_(nullptr) {
145 std::lock_guard<std::mutex> lock(libevent_mutex_);
147 // The value 'current_base' (libevent 1) or
148 // 'event_global_current_base_' (libevent 2) is filled in by event_set(),
149 // allowing examination of its value without an explicit reference here.
150 // If ev.ev_base is NULL, then event_init() must be called, otherwise
151 // call event_base_new().
152 event_set(&ev, 0, 0, nullptr, nullptr);
159 evb_ = event_base_new();
162 if (UNLIKELY(evb_ == nullptr)) {
163 LOG(ERROR) << "EventBase(): Failed to init event base.";
164 folly::throwSystemError("error in EventBase::EventBase()");
166 VLOG(5) << "EventBase(): Created.";
167 initNotificationQueue();
168 RequestContext::saveContext();
171 // takes ownership of the event_base
172 EventBase::EventBase(event_base* evb, bool enableTimeMeasurement)
173 : runOnceCallbacks_(nullptr)
180 , avgLoopTime_(2000000)
181 , maxLatencyLoopTime_(avgLoopTime_)
182 , enableTimeMeasurement_(enableTimeMeasurement)
183 , nextLoopCnt_(-40) // Early wrap-around so bugs will manifest soon
184 , latestLoopCnt_(nextLoopCnt_)
187 , observerSampleCount_(0)
188 , executionObserver_(nullptr) {
189 if (UNLIKELY(evb_ == nullptr)) {
190 LOG(ERROR) << "EventBase(): Pass nullptr as event base.";
191 throw std::invalid_argument("EventBase(): event base cannot be nullptr");
193 initNotificationQueue();
194 RequestContext::saveContext();
197 EventBase::~EventBase() {
198 // Keep looping until all keep-alive handles are released. Each keep-alive
199 // handle signals that some external code will still schedule some work on
200 // this EventBase (so it's not safe to destroy it).
201 while (!loopKeepAlive_.unique()) {
202 applyLoopKeepAlive();
206 // Call all destruction callbacks, before we start cleaning up our state.
207 while (!onDestructionCallbacks_.empty()) {
208 LoopCallback* callback = &onDestructionCallbacks_.front();
209 onDestructionCallbacks_.pop_front();
210 callback->runLoopCallback();
213 // Delete any unfired callback objects, so that we don't leak memory
214 // (Note that we don't fire them. The caller is responsible for cleaning up
215 // its own data structures if it destroys the EventBase with unfired events
217 while (!pendingCobTimeouts_.empty()) {
218 CobTimeout* timeout = &pendingCobTimeouts_.front();
222 while (!runBeforeLoopCallbacks_.empty()) {
223 delete &runBeforeLoopCallbacks_.front();
226 (void) runLoopCallbacks(false);
228 if (!fnRunner_->consumeUntilDrained()) {
229 LOG(ERROR) << "~EventBase(): Unable to drain notification queue";
232 // Stop consumer before deleting NotificationQueue
233 fnRunner_->stopConsuming();
235 std::lock_guard<std::mutex> lock(libevent_mutex_);
236 event_base_free(evb_);
239 while (!runAfterDrainCallbacks_.empty()) {
240 LoopCallback* callback = &runAfterDrainCallbacks_.front();
241 runAfterDrainCallbacks_.pop_front();
242 callback->runLoopCallback();
246 std::lock_guard<std::mutex> lock(localStorageMutex_);
247 for (auto storage : localStorageToDtor_) {
248 storage->onEventBaseDestruction(*this);
251 VLOG(5) << "EventBase(): Destroyed.";
254 int EventBase::getNotificationQueueSize() const {
255 return queue_->size();
258 void EventBase::setMaxReadAtOnce(uint32_t maxAtOnce) {
259 fnRunner_->setMaxReadAtOnce(maxAtOnce);
262 // Set smoothing coefficient for loop load average; input is # of milliseconds
263 // for exp(-1) decay.
264 void EventBase::setLoadAvgMsec(uint32_t ms) {
265 assert(enableTimeMeasurement_);
266 uint64_t us = 1000 * ms;
268 maxLatencyLoopTime_.setTimeInterval(us);
269 avgLoopTime_.setTimeInterval(us);
271 LOG(ERROR) << "non-positive arg to setLoadAvgMsec()";
275 void EventBase::resetLoadAvg(double value) {
276 assert(enableTimeMeasurement_);
277 avgLoopTime_.reset(value);
278 maxLatencyLoopTime_.reset(value);
281 static std::chrono::milliseconds
282 getTimeDelta(std::chrono::steady_clock::time_point* prev) {
283 auto result = std::chrono::steady_clock::now() - *prev;
284 *prev = std::chrono::steady_clock::now();
286 return std::chrono::duration_cast<std::chrono::milliseconds>(result);
289 void EventBase::waitUntilRunning() {
290 while (!isRunning()) {
295 // enters the event_base loop -- will only exit when forced to
296 bool EventBase::loop() {
300 bool EventBase::loopOnce(int flags) {
301 return loopBody(flags | EVLOOP_ONCE);
304 bool EventBase::loopBody(int flags) {
305 VLOG(5) << "EventBase(): Starting loop.";
307 bool ranLoopCallbacks;
308 bool blocking = !(flags & EVLOOP_NONBLOCK);
309 bool once = (flags & EVLOOP_ONCE);
311 // time-measurement variables.
312 std::chrono::steady_clock::time_point prev;
313 int64_t idleStart = 0;
317 loopThread_.store(pthread_self(), std::memory_order_release);
319 if (!name_.empty()) {
320 setThreadName(name_);
323 if (enableTimeMeasurement_) {
324 prev = std::chrono::steady_clock::now();
325 idleStart = std::chrono::duration_cast<std::chrono::microseconds>(
326 std::chrono::steady_clock::now().time_since_epoch()).count();
329 while (!stop_.load(std::memory_order_acquire)) {
330 applyLoopKeepAlive();
333 // Run the before loop callbacks
334 LoopCallbackList callbacks;
335 callbacks.swap(runBeforeLoopCallbacks_);
337 while(!callbacks.empty()) {
338 auto* item = &callbacks.front();
339 callbacks.pop_front();
340 item->runLoopCallback();
343 // nobody can add loop callbacks from within this thread if
344 // we don't have to handle anything to start with...
345 if (blocking && loopCallbacks_.empty()) {
346 res = event_base_loop(evb_, EVLOOP_ONCE);
348 res = event_base_loop(evb_, EVLOOP_ONCE | EVLOOP_NONBLOCK);
351 ranLoopCallbacks = runLoopCallbacks();
353 if (enableTimeMeasurement_) {
354 busy = std::chrono::duration_cast<std::chrono::microseconds>(
355 std::chrono::steady_clock::now().time_since_epoch()).count() -
357 idle = startWork_ - idleStart;
359 avgLoopTime_.addSample(idle, busy);
360 maxLatencyLoopTime_.addSample(idle, busy);
363 if (observerSampleCount_++ == observer_->getSampleRate()) {
364 observerSampleCount_ = 0;
365 observer_->loopSample(busy, idle);
369 VLOG(11) << "EventBase " << this << " did not timeout "
370 " loop time guess: " << busy + idle <<
371 " idle time: " << idle <<
372 " busy time: " << busy <<
373 " avgLoopTime: " << avgLoopTime_.get() <<
374 " maxLatencyLoopTime: " << maxLatencyLoopTime_.get() <<
375 " maxLatency_: " << maxLatency_ <<
376 " notificationQueueSize: " << getNotificationQueueSize() <<
377 " nothingHandledYet(): "<< nothingHandledYet();
379 // see if our average loop time has exceeded our limit
380 if ((maxLatency_ > 0) &&
381 (maxLatencyLoopTime_.get() > double(maxLatency_))) {
383 // back off temporarily -- don't keep spamming maxLatencyCob_
384 // if we're only a bit over the limit
385 maxLatencyLoopTime_.dampen(0.9);
388 // Our loop run did real work; reset the idle timer
389 idleStart = std::chrono::duration_cast<std::chrono::microseconds>(
390 std::chrono::steady_clock::now().time_since_epoch()).count();
392 VLOG(11) << "EventBase " << this << " did not timeout";
395 // If the event loop indicate that there were no more events, and
396 // we also didn't have any loop callbacks to run, there is nothing left to
398 if (res != 0 && !ranLoopCallbacks) {
399 // Since Notification Queue is marked 'internal' some events may not have
400 // run. Run them manually if so, and continue looping.
402 if (getNotificationQueueSize() > 0) {
403 fnRunner_->handlerReady(0);
409 if (enableTimeMeasurement_) {
410 VLOG(5) << "EventBase " << this << " loop time: " <<
411 getTimeDelta(&prev).count();
418 // Reset stop_ so loop() can be called again
422 LOG(ERROR) << "EventBase: -- error in event loop, res = " << res;
424 } else if (res == 1) {
425 VLOG(5) << "EventBase: ran out of events (exiting loop)!";
426 } else if (res > 1) {
427 LOG(ERROR) << "EventBase: unknown event loop result = " << res;
431 loopThread_.store({}, std::memory_order_release);
433 VLOG(5) << "EventBase(): Done with loop.";
437 void EventBase::applyLoopKeepAlive() {
438 if (loopKeepAliveActive_ && loopKeepAlive_.unique()) {
439 // Restore the notification queue internal flag
440 fnRunner_->stopConsuming();
441 fnRunner_->startConsumingInternal(this, queue_.get());
442 loopKeepAliveActive_ = false;
443 } else if (!loopKeepAliveActive_ && !loopKeepAlive_.unique()) {
444 // Update the notification queue event to treat it as a normal
445 // (non-internal) event. The notification queue event always remains
446 // installed, and the main loop won't exit with it installed.
447 fnRunner_->stopConsuming();
448 fnRunner_->startConsuming(this, queue_.get());
449 loopKeepAliveActive_ = true;
453 void EventBase::loopForever() {
457 applyLoopKeepAlive();
458 loopForeverActive_ = false;
460 loopForeverActive_ = true;
461 // Make sure notification queue events are treated as normal events.
462 auto loopKeepAlive = loopKeepAlive_;
467 folly::throwSystemError("error in EventBase::loopForever()");
471 void EventBase::bumpHandlingTime() {
472 if (!enableTimeMeasurement_) {
476 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__ <<
477 " (loop) latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
478 if (nothingHandledYet()) {
479 latestLoopCnt_ = nextLoopCnt_;
481 startWork_ = std::chrono::duration_cast<std::chrono::microseconds>(
482 std::chrono::steady_clock::now().time_since_epoch())
485 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__
486 << " (loop) startWork_ " << startWork_;
490 void EventBase::terminateLoopSoon() {
491 VLOG(5) << "EventBase(): Received terminateLoopSoon() command.";
493 // Set stop to true, so the event loop will know to exit.
494 // TODO: We should really use an atomic operation here with a release
498 // Call event_base_loopbreak() so that libevent will exit the next time
500 event_base_loopbreak(evb_);
502 // If terminateLoopSoon() is called from another thread,
503 // the EventBase thread might be stuck waiting for events.
504 // In this case, it won't wake up and notice that stop_ is set until it
505 // receives another event. Send an empty frame to the notification queue
506 // so that the event loop will wake up even if there are no other events.
508 // We don't care about the return value of trySendFrame(). If it fails
509 // this likely means the EventBase already has lots of events waiting
512 queue_->putMessage(nullptr);
514 // We don't care if putMessage() fails. This likely means
515 // the EventBase already has lots of events waiting anyway.
519 void EventBase::runInLoop(LoopCallback* callback, bool thisIteration) {
520 DCHECK(isInEventBaseThread());
521 callback->cancelLoopCallback();
522 callback->context_ = RequestContext::saveContext();
523 if (runOnceCallbacks_ != nullptr && thisIteration) {
524 runOnceCallbacks_->push_back(*callback);
526 loopCallbacks_.push_back(*callback);
530 void EventBase::runInLoop(Func cob, bool thisIteration) {
531 DCHECK(isInEventBaseThread());
532 auto wrapper = new FunctionLoopCallback(std::move(cob));
533 wrapper->context_ = RequestContext::saveContext();
534 if (runOnceCallbacks_ != nullptr && thisIteration) {
535 runOnceCallbacks_->push_back(*wrapper);
537 loopCallbacks_.push_back(*wrapper);
541 void EventBase::runAfterDrain(Func cob) {
542 auto callback = new FunctionLoopCallback(std::move(cob));
543 std::lock_guard<std::mutex> lg(runAfterDrainCallbacksMutex_);
544 callback->cancelLoopCallback();
545 runAfterDrainCallbacks_.push_back(*callback);
548 void EventBase::runOnDestruction(LoopCallback* callback) {
549 std::lock_guard<std::mutex> lg(onDestructionCallbacksMutex_);
550 callback->cancelLoopCallback();
551 onDestructionCallbacks_.push_back(*callback);
554 void EventBase::runBeforeLoop(LoopCallback* callback) {
555 DCHECK(isInEventBaseThread());
556 callback->cancelLoopCallback();
557 runBeforeLoopCallbacks_.push_back(*callback);
560 bool EventBase::runInEventBaseThread(Func fn) {
562 // It will be received by the FunctionRunner in the EventBase's thread.
564 // We try not to schedule nullptr callbacks
566 LOG(ERROR) << "EventBase " << this
567 << ": Scheduling nullptr callbacks is not allowed";
571 // Short-circuit if we are already in our event base
572 if (inRunningEventBaseThread()) {
573 runInLoop(std::move(fn));
579 queue_->putMessage(std::move(fn));
580 } catch (const std::exception& ex) {
581 LOG(ERROR) << "EventBase " << this << ": failed to schedule function "
582 << "for EventBase thread: " << ex.what();
589 bool EventBase::runInEventBaseThreadAndWait(Func fn) {
590 if (inRunningEventBaseThread()) {
591 LOG(ERROR) << "EventBase " << this << ": Waiting in the event loop is not "
598 std::condition_variable cv;
599 runInEventBaseThread([&] {
601 std::unique_lock<std::mutex> l(m);
604 // We cannot release the lock before notify_one, because a spurious
605 // wakeup in the waiting thread may lead to cv and m going out of scope
610 std::unique_lock<std::mutex> l(m);
611 cv.wait(l, [&] { return ready; });
616 bool EventBase::runImmediatelyOrRunInEventBaseThreadAndWait(Func fn) {
617 if (isInEventBaseThread()) {
621 return runInEventBaseThreadAndWait(std::move(fn));
625 void EventBase::runAfterDelay(
627 uint32_t milliseconds,
628 TimeoutManager::InternalEnum in) {
629 if (!tryRunAfterDelay(std::move(cob), milliseconds, in)) {
630 folly::throwSystemError(
631 "error in EventBase::runAfterDelay(), failed to schedule timeout");
635 bool EventBase::tryRunAfterDelay(
637 uint32_t milliseconds,
638 TimeoutManager::InternalEnum in) {
639 CobTimeout* timeout = new CobTimeout(this, std::move(cob), in);
640 if (!timeout->scheduleTimeout(milliseconds)) {
644 pendingCobTimeouts_.push_back(*timeout);
648 bool EventBase::runLoopCallbacks(bool setContext) {
649 if (!loopCallbacks_.empty()) {
651 // Swap the loopCallbacks_ list with a temporary list on our stack.
652 // This way we will only run callbacks scheduled at the time
653 // runLoopCallbacks() was invoked.
655 // If any of these callbacks in turn call runInLoop() to schedule more
656 // callbacks, those new callbacks won't be run until the next iteration
657 // around the event loop. This prevents runInLoop() callbacks from being
658 // able to start file descriptor and timeout based events.
659 LoopCallbackList currentCallbacks;
660 currentCallbacks.swap(loopCallbacks_);
661 runOnceCallbacks_ = ¤tCallbacks;
663 while (!currentCallbacks.empty()) {
664 LoopCallback* callback = ¤tCallbacks.front();
665 currentCallbacks.pop_front();
667 RequestContext::setContext(callback->context_);
669 callback->runLoopCallback();
672 runOnceCallbacks_ = nullptr;
678 void EventBase::initNotificationQueue() {
679 // Infinite size queue
680 queue_.reset(new NotificationQueue<Func>());
682 // We allocate fnRunner_ separately, rather than declaring it directly
683 // as a member of EventBase solely so that we don't need to include
684 // NotificationQueue.h from EventBase.h
685 fnRunner_.reset(new FunctionRunner());
687 // Mark this as an internal event, so event_base_loop() will return if
688 // there are no other events besides this one installed.
690 // Most callers don't care about the internal notification queue used by
691 // EventBase. The queue is always installed, so if we did count the queue as
692 // an active event, loop() would never exit with no more events to process.
693 // Users can use loopForever() if they do care about the notification queue.
694 // (This is useful for EventBase threads that do nothing but process
695 // runInEventBaseThread() notifications.)
696 fnRunner_->startConsumingInternal(this, queue_.get());
699 void EventBase::SmoothLoopTime::setTimeInterval(uint64_t timeInterval) {
700 expCoeff_ = -1.0/timeInterval;
701 VLOG(11) << "expCoeff_ " << expCoeff_ << " " << __PRETTY_FUNCTION__;
704 void EventBase::SmoothLoopTime::reset(double value) {
708 void EventBase::SmoothLoopTime::addSample(int64_t idle, int64_t busy) {
710 * Position at which the busy sample is considered to be taken.
711 * (Allows to quickly skew our average without editing much code)
713 enum BusySamplePosition {
714 RIGHT = 0, // busy sample placed at the end of the iteration
715 CENTER = 1, // busy sample placed at the middle point of the iteration
716 LEFT = 2, // busy sample placed at the beginning of the iteration
719 // See http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
720 // and D676020 for more info on this calculation.
721 VLOG(11) << "idle " << idle << " oldBusyLeftover_ " << oldBusyLeftover_ <<
722 " idle + oldBusyLeftover_ " << idle + oldBusyLeftover_ <<
723 " busy " << busy << " " << __PRETTY_FUNCTION__;
724 idle += oldBusyLeftover_ + busy;
725 oldBusyLeftover_ = (busy * BusySamplePosition::CENTER) / 2;
726 idle -= oldBusyLeftover_;
728 double coeff = exp(idle * expCoeff_);
730 value_ += (1.0 - coeff) * busy;
733 bool EventBase::nothingHandledYet() const noexcept {
734 VLOG(11) << "latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
735 return (nextLoopCnt_ != latestLoopCnt_);
738 void EventBase::attachTimeoutManager(AsyncTimeout* obj,
739 InternalEnum internal) {
741 struct event* ev = obj->getEvent();
742 assert(ev->ev_base == nullptr);
744 event_base_set(getLibeventBase(), ev);
745 if (internal == AsyncTimeout::InternalEnum::INTERNAL) {
746 // Set the EVLIST_INTERNAL flag
747 event_ref_flags(ev) |= EVLIST_INTERNAL;
751 void EventBase::detachTimeoutManager(AsyncTimeout* obj) {
753 struct event* ev = obj->getEvent();
754 ev->ev_base = nullptr;
757 bool EventBase::scheduleTimeout(AsyncTimeout* obj,
758 TimeoutManager::timeout_type timeout) {
759 assert(isInEventBaseThread());
760 // Set up the timeval and add the event
762 tv.tv_sec = timeout.count() / 1000LL;
763 tv.tv_usec = (timeout.count() % 1000LL) * 1000LL;
765 struct event* ev = obj->getEvent();
766 if (event_add(ev, &tv) < 0) {
767 LOG(ERROR) << "EventBase: failed to schedule timeout: " << strerror(errno);
774 void EventBase::cancelTimeout(AsyncTimeout* obj) {
775 assert(isInEventBaseThread());
776 struct event* ev = obj->getEvent();
777 if (EventUtil::isEventRegistered(ev)) {
782 void EventBase::setName(const std::string& name) {
783 assert(isInEventBaseThread());
787 setThreadName(loopThread_.load(std::memory_order_relaxed),
792 const std::string& EventBase::getName() {
793 assert(isInEventBaseThread());
797 const char* EventBase::getLibeventVersion() { return event_get_version(); }
798 const char* EventBase::getLibeventMethod() { return event_get_method(); }