2 * Copyright 2004-present Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
16 #include <folly/experimental/logging/AsyncFileWriter.h>
18 #include <folly/Exception.h>
19 #include <folly/FileUtil.h>
20 #include <folly/experimental/logging/LoggerDB.h>
23 using folly::StringPiece;
27 AsyncFileWriter::AsyncFileWriter(StringPiece path)
28 : AsyncFileWriter{File{path.str(), O_WRONLY | O_APPEND | O_CREAT}} {}
30 AsyncFileWriter::AsyncFileWriter(folly::File&& file)
31 : file_{std::move(file)}, ioThread_([this] { ioThread(); }) {}
33 AsyncFileWriter::~AsyncFileWriter() {
35 messageReady_.notify_one();
39 void AsyncFileWriter::writeMessage(StringPiece buffer, uint32_t flags) {
40 return writeMessage(buffer.str(), flags);
43 void AsyncFileWriter::writeMessage(std::string&& buffer, uint32_t flags) {
44 auto data = data_.lock();
45 if ((data->currentBufferSize >= data->maxBufferBytes) &&
46 !(flags & NEVER_DISCARD)) {
51 data->currentBufferSize += buffer.size();
52 auto* queue = data->getCurrentQueue();
53 queue->emplace_back(std::move(buffer));
54 messageReady_.notify_one();
57 void AsyncFileWriter::flush() {
58 auto data = data_.lock();
59 auto start = data->ioThreadCounter;
61 // Wait until ioThreadCounter increments by at least two.
62 // Waiting for a single increment is not sufficient, as this happens after
63 // the I/O thread has swapped the queues, which is before it has actually
65 while (data->ioThreadCounter < start + 2) {
66 if (data->ioThreadDone) {
70 // Enqueue an empty string and wake the I/O thread.
71 // The empty string ensures that the I/O thread will break out of its wait
72 // loop and increment the ioThreadCounter, even if there is no other work
74 data->getCurrentQueue()->emplace_back();
75 messageReady_.notify_one();
77 // Wait for notification from the I/O thread that it has done work.
78 ioCV_.wait(data.getUniqueLock());
82 void AsyncFileWriter::ioThread() {
84 // With the lock held, grab a pointer to the current queue, then increment
85 // the ioThreadCounter index so that other threads will write into the
86 // other queue as we process this one.
87 std::vector<std::string>* ioQueue;
91 auto data = data_.lock();
92 ioQueue = data->getCurrentQueue();
93 while (ioQueue->empty() && !data->stop) {
94 messageReady_.wait(data.getUniqueLock());
97 ++data->ioThreadCounter;
98 numDiscarded = data->numDiscarded;
99 data->numDiscarded = 0;
100 data->currentBufferSize = 0;
105 // Write the log messages now that we have released the lock
108 } catch (const std::exception& ex) {
112 // clear() empties the vector, but the allocated capacity remains so we can
113 // just reuse it without having to re-allocate in most cases.
116 if (numDiscarded > 0) {
117 auto msg = getNumDiscardedMsg(numDiscarded);
119 auto ret = folly::writeFull(file_.fd(), msg.data(), msg.size());
120 // We currently ignore errors from writeFull() here.
121 // There's not much we can really do.
127 data_->ioThreadDone = true;
133 void AsyncFileWriter::performIO(std::vector<std::string>* ioQueue) {
134 // kNumIovecs controls the maximum number of strings we write at once in a
135 // single writev() call.
136 constexpr int kNumIovecs = 64;
137 std::array<iovec, kNumIovecs> iovecs;
140 while (idx < ioQueue->size()) {
142 while (numIovecs < kNumIovecs && idx < ioQueue->size()) {
143 const auto& str = (*ioQueue)[idx];
144 iovecs[numIovecs].iov_base = const_cast<char*>(str.data());
145 iovecs[numIovecs].iov_len = str.size();
150 auto ret = folly::writevFull(file_.fd(), iovecs.data(), numIovecs);
151 folly::checkUnixError(ret, "writeFull() failed");
155 void AsyncFileWriter::onIoError(const std::exception& ex) {
156 LoggerDB::internalWarning(
159 "error writing to log file ",
161 " in AsyncFileWriter: ",
162 folly::exceptionStr(ex));
165 std::string AsyncFileWriter::getNumDiscardedMsg(size_t numDiscarded) {
166 // We may want to make this customizable in the future (e.g., to allow it to
167 // conform to the LogFormatter style being used).
168 // For now just return a simple fixed message.
169 return folly::to<std::string>(
171 " log messages discarded: logging faster than we can write\n");