2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef __STDC_FORMAT_MACROS
18 #define __STDC_FORMAT_MACROS
21 #include <folly/io/async/EventBase.h>
23 #include <folly/ThreadName.h>
24 #include <folly/io/async/NotificationQueue.h>
25 #include <folly/portability/Unistd.h>
27 #include <condition_variable>
35 * EventBase::FunctionRunner
38 class EventBase::FunctionRunner
39 : public NotificationQueue<EventBase::Func>::Consumer {
41 void messageAvailable(Func&& msg) override {
42 // In libevent2, internal events do not break the loop.
43 // Most users would expect loop(), followed by runInEventBaseThread(),
44 // to break the loop and check if it should exit or not.
45 // To have similar bejaviour to libevent1.4, tell the loop to break here.
46 // Note that loop() may still continue to loop, but it will also check the
47 // stop_ flag as well as runInLoop callbacks, etc.
48 event_base_loopbreak(getEventBase()->evb_);
51 // terminateLoopSoon() sends a null message just to
52 // wake up the loop. We can ignore these messages.
56 // The function should never throw an exception, because we have no
57 // way of knowing what sort of error handling to perform.
59 // If it does throw, log a message and abort the program.
62 } catch (const std::exception& ex) {
63 LOG(ERROR) << "runInEventBaseThread() function threw a "
64 << typeid(ex).name() << " exception: " << ex.what();
67 LOG(ERROR) << "runInEventBaseThread() function threw an exception";
73 // The interface used to libevent is not thread-safe. Calls to
74 // event_init() and event_base_free() directly modify an internal
75 // global 'current_base', so a mutex is required to protect this.
77 // event_init() should only ever be called once. Subsequent calls
78 // should be made to event_base_new(). We can recognise that
79 // event_init() has already been called by simply inspecting current_base.
80 static std::mutex libevent_mutex_;
86 EventBase::EventBase(bool enableTimeMeasurement)
87 : runOnceCallbacks_(nullptr)
93 , avgLoopTime_(std::chrono::milliseconds(2000000))
94 , maxLatencyLoopTime_(avgLoopTime_)
95 , enableTimeMeasurement_(enableTimeMeasurement)
96 , nextLoopCnt_(uint64_t(-40)) // Early wrap-around so bugs will manifest soon
97 , latestLoopCnt_(nextLoopCnt_)
100 , observerSampleCount_(0)
101 , executionObserver_(nullptr) {
104 std::lock_guard<std::mutex> lock(libevent_mutex_);
106 // The value 'current_base' (libevent 1) or
107 // 'event_global_current_base_' (libevent 2) is filled in by event_set(),
108 // allowing examination of its value without an explicit reference here.
109 // If ev.ev_base is NULL, then event_init() must be called, otherwise
110 // call event_base_new().
111 event_set(&ev, 0, 0, nullptr, nullptr);
118 evb_ = event_base_new();
121 if (UNLIKELY(evb_ == nullptr)) {
122 LOG(ERROR) << "EventBase(): Failed to init event base.";
123 folly::throwSystemError("error in EventBase::EventBase()");
125 VLOG(5) << "EventBase(): Created.";
126 initNotificationQueue();
127 RequestContext::saveContext();
130 // takes ownership of the event_base
131 EventBase::EventBase(event_base* evb, bool enableTimeMeasurement)
132 : runOnceCallbacks_(nullptr)
139 , avgLoopTime_(std::chrono::milliseconds(2000000))
140 , maxLatencyLoopTime_(avgLoopTime_)
141 , enableTimeMeasurement_(enableTimeMeasurement)
142 , nextLoopCnt_(uint64_t(-40)) // Early wrap-around so bugs will manifest soon
143 , latestLoopCnt_(nextLoopCnt_)
146 , observerSampleCount_(0)
147 , executionObserver_(nullptr) {
148 if (UNLIKELY(evb_ == nullptr)) {
149 LOG(ERROR) << "EventBase(): Pass nullptr as event base.";
150 throw std::invalid_argument("EventBase(): event base cannot be nullptr");
152 initNotificationQueue();
153 RequestContext::saveContext();
156 EventBase::~EventBase() {
157 // Keep looping until all keep-alive handles are released. Each keep-alive
158 // handle signals that some external code will still schedule some work on
159 // this EventBase (so it's not safe to destroy it).
160 while (loopKeepAliveCount() > 0) {
161 applyLoopKeepAlive();
165 // Call all destruction callbacks, before we start cleaning up our state.
166 while (!onDestructionCallbacks_.empty()) {
167 LoopCallback* callback = &onDestructionCallbacks_.front();
168 onDestructionCallbacks_.pop_front();
169 callback->runLoopCallback();
174 DCHECK_EQ(0, runBeforeLoopCallbacks_.size());
176 (void)runLoopCallbacks();
178 if (!fnRunner_->consumeUntilDrained()) {
179 LOG(ERROR) << "~EventBase(): Unable to drain notification queue";
182 // Stop consumer before deleting NotificationQueue
183 fnRunner_->stopConsuming();
185 std::lock_guard<std::mutex> lock(libevent_mutex_);
186 event_base_free(evb_);
190 std::lock_guard<std::mutex> lock(localStorageMutex_);
191 for (auto storage : localStorageToDtor_) {
192 storage->onEventBaseDestruction(*this);
195 VLOG(5) << "EventBase(): Destroyed.";
198 size_t EventBase::getNotificationQueueSize() const {
199 return queue_->size();
202 void EventBase::setMaxReadAtOnce(uint32_t maxAtOnce) {
203 fnRunner_->setMaxReadAtOnce(maxAtOnce);
206 // Set smoothing coefficient for loop load average; input is # of milliseconds
207 // for exp(-1) decay.
208 void EventBase::setLoadAvgMsec(std::chrono::milliseconds ms) {
209 assert(enableTimeMeasurement_);
210 std::chrono::microseconds us = std::chrono::milliseconds(ms);
211 if (ms > std::chrono::milliseconds::zero()) {
212 maxLatencyLoopTime_.setTimeInterval(us);
213 avgLoopTime_.setTimeInterval(us);
215 LOG(ERROR) << "non-positive arg to setLoadAvgMsec()";
219 void EventBase::resetLoadAvg(double value) {
220 assert(enableTimeMeasurement_);
221 avgLoopTime_.reset(value);
222 maxLatencyLoopTime_.reset(value);
225 static std::chrono::milliseconds
226 getTimeDelta(std::chrono::steady_clock::time_point* prev) {
227 auto result = std::chrono::steady_clock::now() - *prev;
228 *prev = std::chrono::steady_clock::now();
230 return std::chrono::duration_cast<std::chrono::milliseconds>(result);
233 void EventBase::waitUntilRunning() {
234 while (!isRunning()) {
239 // enters the event_base loop -- will only exit when forced to
240 bool EventBase::loop() {
244 bool EventBase::loopOnce(int flags) {
245 return loopBody(flags | EVLOOP_ONCE);
248 bool EventBase::loopBody(int flags) {
249 VLOG(5) << "EventBase(): Starting loop.";
251 DCHECK(!invokingLoop_)
252 << "Your code just tried to loop over an event base from inside another "
253 << "event base loop. Since libevent is not reentrant, this leads to "
254 << "undefined behavior in opt builds. Please fix immediately. For the "
255 << "common case of an inner function that needs to do some synchronous "
256 << "computation on an event-base, replace getEventBase() by a new, "
257 << "stack-allocated EvenBase.";
258 invokingLoop_ = true;
260 invokingLoop_ = false;
264 bool ranLoopCallbacks;
265 bool blocking = !(flags & EVLOOP_NONBLOCK);
266 bool once = (flags & EVLOOP_ONCE);
268 // time-measurement variables.
269 std::chrono::steady_clock::time_point prev;
270 int64_t idleStart = 0;
274 loopThread_.store(pthread_self(), std::memory_order_release);
276 if (!name_.empty()) {
277 setThreadName(name_);
280 if (enableTimeMeasurement_) {
281 prev = std::chrono::steady_clock::now();
282 idleStart = std::chrono::duration_cast<std::chrono::microseconds>(
283 std::chrono::steady_clock::now().time_since_epoch()).count();
286 while (!stop_.load(std::memory_order_acquire)) {
287 applyLoopKeepAlive();
290 // Run the before loop callbacks
291 LoopCallbackList callbacks;
292 callbacks.swap(runBeforeLoopCallbacks_);
294 while(!callbacks.empty()) {
295 auto* item = &callbacks.front();
296 callbacks.pop_front();
297 item->runLoopCallback();
300 // nobody can add loop callbacks from within this thread if
301 // we don't have to handle anything to start with...
302 if (blocking && loopCallbacks_.empty()) {
303 res = event_base_loop(evb_, EVLOOP_ONCE);
305 res = event_base_loop(evb_, EVLOOP_ONCE | EVLOOP_NONBLOCK);
308 ranLoopCallbacks = runLoopCallbacks();
310 if (enableTimeMeasurement_) {
311 busy = std::chrono::duration_cast<std::chrono::microseconds>(
312 std::chrono::steady_clock::now().time_since_epoch()).count() -
314 idle = startWork_ - idleStart;
316 avgLoopTime_.addSample(std::chrono::microseconds(idle),
317 std::chrono::microseconds(busy));
318 maxLatencyLoopTime_.addSample(std::chrono::microseconds(idle),
319 std::chrono::microseconds(busy));
322 if (observerSampleCount_++ == observer_->getSampleRate()) {
323 observerSampleCount_ = 0;
324 observer_->loopSample(busy, idle);
328 VLOG(11) << "EventBase " << this << " did not timeout "
329 " loop time guess: " << busy + idle <<
330 " idle time: " << idle <<
331 " busy time: " << busy <<
332 " avgLoopTime: " << avgLoopTime_.get() <<
333 " maxLatencyLoopTime: " << maxLatencyLoopTime_.get() <<
334 " maxLatency_: " << maxLatency_ <<
335 " notificationQueueSize: " << getNotificationQueueSize() <<
336 " nothingHandledYet(): "<< nothingHandledYet();
338 // see if our average loop time has exceeded our limit
339 if ((maxLatency_ > 0) &&
340 (maxLatencyLoopTime_.get() > double(maxLatency_))) {
342 // back off temporarily -- don't keep spamming maxLatencyCob_
343 // if we're only a bit over the limit
344 maxLatencyLoopTime_.dampen(0.9);
347 // Our loop run did real work; reset the idle timer
348 idleStart = std::chrono::duration_cast<std::chrono::microseconds>(
349 std::chrono::steady_clock::now().time_since_epoch()).count();
351 VLOG(11) << "EventBase " << this << " did not timeout";
354 // If the event loop indicate that there were no more events, and
355 // we also didn't have any loop callbacks to run, there is nothing left to
357 if (res != 0 && !ranLoopCallbacks) {
358 // Since Notification Queue is marked 'internal' some events may not have
359 // run. Run them manually if so, and continue looping.
361 if (getNotificationQueueSize() > 0) {
362 fnRunner_->handlerReady(0);
368 if (enableTimeMeasurement_) {
369 VLOG(5) << "EventBase " << this << " loop time: " <<
370 getTimeDelta(&prev).count();
377 // Reset stop_ so loop() can be called again
381 LOG(ERROR) << "EventBase: -- error in event loop, res = " << res;
383 } else if (res == 1) {
384 VLOG(5) << "EventBase: ran out of events (exiting loop)!";
385 } else if (res > 1) {
386 LOG(ERROR) << "EventBase: unknown event loop result = " << res;
390 loopThread_.store({}, std::memory_order_release);
392 VLOG(5) << "EventBase(): Done with loop.";
396 ssize_t EventBase::loopKeepAliveCount() {
397 if (loopKeepAliveCountAtomic_.load(std::memory_order_relaxed)) {
398 loopKeepAliveCount_ +=
399 loopKeepAliveCountAtomic_.exchange(0, std::memory_order_relaxed);
401 DCHECK_GE(loopKeepAliveCount_, 0);
402 return loopKeepAliveCount_;
405 void EventBase::applyLoopKeepAlive() {
406 if (loopKeepAliveActive_ && loopKeepAliveCount() == 0) {
407 // Restore the notification queue internal flag
408 fnRunner_->stopConsuming();
409 fnRunner_->startConsumingInternal(this, queue_.get());
410 loopKeepAliveActive_ = false;
411 } else if (!loopKeepAliveActive_ && loopKeepAliveCount() > 0) {
412 // Update the notification queue event to treat it as a normal
413 // (non-internal) event. The notification queue event always remains
414 // installed, and the main loop won't exit with it installed.
415 fnRunner_->stopConsuming();
416 fnRunner_->startConsuming(this, queue_.get());
417 loopKeepAliveActive_ = true;
421 void EventBase::loopForever() {
425 applyLoopKeepAlive();
427 // Make sure notification queue events are treated as normal events.
428 // We can't use loopKeepAlive() here since LoopKeepAlive token can only be
429 // released inside a loop.
430 ++loopKeepAliveCount_;
432 --loopKeepAliveCount_;
438 folly::throwSystemError("error in EventBase::loopForever()");
442 void EventBase::bumpHandlingTime() {
443 if (!enableTimeMeasurement_) {
447 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__ <<
448 " (loop) latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
449 if (nothingHandledYet()) {
450 latestLoopCnt_ = nextLoopCnt_;
452 startWork_ = std::chrono::duration_cast<std::chrono::microseconds>(
453 std::chrono::steady_clock::now().time_since_epoch())
456 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__
457 << " (loop) startWork_ " << startWork_;
461 void EventBase::terminateLoopSoon() {
462 VLOG(5) << "EventBase(): Received terminateLoopSoon() command.";
464 // Set stop to true, so the event loop will know to exit.
465 // TODO: We should really use an atomic operation here with a release
469 // Call event_base_loopbreak() so that libevent will exit the next time
471 event_base_loopbreak(evb_);
473 // If terminateLoopSoon() is called from another thread,
474 // the EventBase thread might be stuck waiting for events.
475 // In this case, it won't wake up and notice that stop_ is set until it
476 // receives another event. Send an empty frame to the notification queue
477 // so that the event loop will wake up even if there are no other events.
479 // We don't care about the return value of trySendFrame(). If it fails
480 // this likely means the EventBase already has lots of events waiting
483 queue_->putMessage(nullptr);
485 // We don't care if putMessage() fails. This likely means
486 // the EventBase already has lots of events waiting anyway.
490 void EventBase::runInLoop(LoopCallback* callback, bool thisIteration) {
491 DCHECK(isInEventBaseThread());
492 callback->cancelLoopCallback();
493 callback->context_ = RequestContext::saveContext();
494 if (runOnceCallbacks_ != nullptr && thisIteration) {
495 runOnceCallbacks_->push_back(*callback);
497 loopCallbacks_.push_back(*callback);
501 void EventBase::runInLoop(Func cob, bool thisIteration) {
502 DCHECK(isInEventBaseThread());
503 auto wrapper = new FunctionLoopCallback(std::move(cob));
504 wrapper->context_ = RequestContext::saveContext();
505 if (runOnceCallbacks_ != nullptr && thisIteration) {
506 runOnceCallbacks_->push_back(*wrapper);
508 loopCallbacks_.push_back(*wrapper);
512 void EventBase::runOnDestruction(LoopCallback* callback) {
513 std::lock_guard<std::mutex> lg(onDestructionCallbacksMutex_);
514 callback->cancelLoopCallback();
515 onDestructionCallbacks_.push_back(*callback);
518 void EventBase::runBeforeLoop(LoopCallback* callback) {
519 DCHECK(isInEventBaseThread());
520 callback->cancelLoopCallback();
521 runBeforeLoopCallbacks_.push_back(*callback);
524 bool EventBase::runInEventBaseThread(Func fn) {
526 // It will be received by the FunctionRunner in the EventBase's thread.
528 // We try not to schedule nullptr callbacks
530 LOG(ERROR) << "EventBase " << this
531 << ": Scheduling nullptr callbacks is not allowed";
535 // Short-circuit if we are already in our event base
536 if (inRunningEventBaseThread()) {
537 runInLoop(std::move(fn));
543 queue_->putMessage(std::move(fn));
544 } catch (const std::exception& ex) {
545 LOG(ERROR) << "EventBase " << this << ": failed to schedule function "
546 << "for EventBase thread: " << ex.what();
553 bool EventBase::runInEventBaseThreadAndWait(FuncRef fn) {
554 if (inRunningEventBaseThread()) {
555 LOG(ERROR) << "EventBase " << this << ": Waiting in the event loop is not "
562 std::condition_variable cv;
563 runInEventBaseThread([&] {
565 std::unique_lock<std::mutex> l(m);
568 // We cannot release the lock before notify_one, because a spurious
569 // wakeup in the waiting thread may lead to cv and m going out of scope
574 std::unique_lock<std::mutex> l(m);
575 cv.wait(l, [&] { return ready; });
580 bool EventBase::runImmediatelyOrRunInEventBaseThreadAndWait(FuncRef fn) {
581 if (isInEventBaseThread()) {
585 return runInEventBaseThreadAndWait(std::move(fn));
589 bool EventBase::runLoopCallbacks() {
590 if (!loopCallbacks_.empty()) {
592 // Swap the loopCallbacks_ list with a temporary list on our stack.
593 // This way we will only run callbacks scheduled at the time
594 // runLoopCallbacks() was invoked.
596 // If any of these callbacks in turn call runInLoop() to schedule more
597 // callbacks, those new callbacks won't be run until the next iteration
598 // around the event loop. This prevents runInLoop() callbacks from being
599 // able to start file descriptor and timeout based events.
600 LoopCallbackList currentCallbacks;
601 currentCallbacks.swap(loopCallbacks_);
602 runOnceCallbacks_ = ¤tCallbacks;
604 while (!currentCallbacks.empty()) {
605 LoopCallback* callback = ¤tCallbacks.front();
606 currentCallbacks.pop_front();
607 folly::RequestContextScopeGuard rctx(callback->context_);
608 callback->runLoopCallback();
611 runOnceCallbacks_ = nullptr;
617 void EventBase::initNotificationQueue() {
618 // Infinite size queue
619 queue_.reset(new NotificationQueue<Func>());
621 // We allocate fnRunner_ separately, rather than declaring it directly
622 // as a member of EventBase solely so that we don't need to include
623 // NotificationQueue.h from EventBase.h
624 fnRunner_.reset(new FunctionRunner());
626 // Mark this as an internal event, so event_base_loop() will return if
627 // there are no other events besides this one installed.
629 // Most callers don't care about the internal notification queue used by
630 // EventBase. The queue is always installed, so if we did count the queue as
631 // an active event, loop() would never exit with no more events to process.
632 // Users can use loopForever() if they do care about the notification queue.
633 // (This is useful for EventBase threads that do nothing but process
634 // runInEventBaseThread() notifications.)
635 fnRunner_->startConsumingInternal(this, queue_.get());
638 void EventBase::SmoothLoopTime::setTimeInterval(
639 std::chrono::microseconds timeInterval) {
640 expCoeff_ = -1.0 / timeInterval.count();
641 VLOG(11) << "expCoeff_ " << expCoeff_ << " " << __PRETTY_FUNCTION__;
644 void EventBase::SmoothLoopTime::reset(double value) {
648 void EventBase::SmoothLoopTime::addSample(
649 std::chrono::microseconds idle,
650 std::chrono::microseconds busy) {
652 * Position at which the busy sample is considered to be taken.
653 * (Allows to quickly skew our average without editing much code)
655 enum BusySamplePosition {
656 RIGHT = 0, // busy sample placed at the end of the iteration
657 CENTER = 1, // busy sample placed at the middle point of the iteration
658 LEFT = 2, // busy sample placed at the beginning of the iteration
661 // See http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
662 // and D676020 for more info on this calculation.
663 VLOG(11) << "idle " << idle.count() << " oldBusyLeftover_ "
664 << oldBusyLeftover_.count() << " idle + oldBusyLeftover_ "
665 << (idle + oldBusyLeftover_).count() << " busy " << busy.count()
666 << " " << __PRETTY_FUNCTION__;
667 idle += oldBusyLeftover_ + busy;
668 oldBusyLeftover_ = (busy * BusySamplePosition::CENTER) / 2;
669 idle -= oldBusyLeftover_;
671 double coeff = exp(idle.count() * expCoeff_);
673 value_ += (1.0 - coeff) * busy.count();
676 bool EventBase::nothingHandledYet() const noexcept {
677 VLOG(11) << "latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
678 return (nextLoopCnt_ != latestLoopCnt_);
681 void EventBase::attachTimeoutManager(AsyncTimeout* obj,
682 InternalEnum internal) {
684 struct event* ev = obj->getEvent();
685 assert(ev->ev_base == nullptr);
687 event_base_set(getLibeventBase(), ev);
688 if (internal == AsyncTimeout::InternalEnum::INTERNAL) {
689 // Set the EVLIST_INTERNAL flag
690 event_ref_flags(ev) |= EVLIST_INTERNAL;
694 void EventBase::detachTimeoutManager(AsyncTimeout* obj) {
696 struct event* ev = obj->getEvent();
697 ev->ev_base = nullptr;
700 bool EventBase::scheduleTimeout(AsyncTimeout* obj,
701 TimeoutManager::timeout_type timeout) {
702 assert(isInEventBaseThread());
703 // Set up the timeval and add the event
705 tv.tv_sec = long(timeout.count() / 1000LL);
706 tv.tv_usec = long((timeout.count() % 1000LL) * 1000LL);
708 struct event* ev = obj->getEvent();
709 if (event_add(ev, &tv) < 0) {
710 LOG(ERROR) << "EventBase: failed to schedule timeout: " << strerror(errno);
717 void EventBase::cancelTimeout(AsyncTimeout* obj) {
718 assert(isInEventBaseThread());
719 struct event* ev = obj->getEvent();
720 if (EventUtil::isEventRegistered(ev)) {
725 void EventBase::setName(const std::string& name) {
726 assert(isInEventBaseThread());
730 setThreadName(loopThread_.load(std::memory_order_relaxed),
735 const std::string& EventBase::getName() {
736 assert(isInEventBaseThread());
740 const char* EventBase::getLibeventVersion() { return event_get_version(); }
741 const char* EventBase::getLibeventMethod() { return event_get_method(); }