2 * Copyright 2016 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef __STDC_LIMIT_MACROS
18 #define __STDC_LIMIT_MACROS
21 #include <folly/io/IOBuf.h>
23 #include <folly/Conv.h>
24 #include <folly/Likely.h>
25 #include <folly/Malloc.h>
26 #include <folly/Memory.h>
27 #include <folly/ScopeGuard.h>
28 #include <folly/SpookyHashV2.h>
29 #include <folly/io/Cursor.h>
36 using std::unique_ptr;
42 // This memory segment contains an IOBuf that is still in use
44 // This memory segment contains buffer data that is still in use
49 // When create() is called for buffers less than kDefaultCombinedBufSize,
50 // we allocate a single combined memory segment for the IOBuf and the data
51 // together. See the comments for createCombined()/createSeparate() for more
54 // (The size of 1k is largely just a guess here. We could could probably do
55 // benchmarks of real applications to see if adjusting this number makes a
56 // difference. Callers that know their exact use case can also explicitly
57 // call createCombined() or createSeparate().)
58 kDefaultCombinedBufSize = 1024
61 // Helper function for IOBuf::takeOwnership()
62 void takeOwnershipError(bool freeOnError, void* buf,
63 folly::IOBuf::FreeFunction freeFn,
73 freeFn(buf, userData);
75 // The user's free function is not allowed to throw.
76 // (We are already in the middle of throwing an exception, so
77 // we cannot let this exception go unhandled.)
82 } // unnamed namespace
86 struct IOBuf::HeapPrefix {
87 HeapPrefix(uint16_t flg)
91 // Reset magic to 0 on destruction. This is solely for debugging purposes
92 // to help catch bugs where someone tries to use HeapStorage after it has
98 std::atomic<uint16_t> flags;
101 struct IOBuf::HeapStorage {
103 // The IOBuf is last in the HeapStorage object.
104 // This way operator new will work even if allocating a subclass of IOBuf
105 // that requires more space.
109 struct IOBuf::HeapFullStorage {
110 // Make sure jemalloc allocates from the 64-byte class. Putting this here
111 // because HeapStorage is private so it can't be at namespace level.
112 static_assert(sizeof(HeapStorage) <= 64,
113 "IOBuf may not grow over 56 bytes!");
117 std::max_align_t align;
120 IOBuf::SharedInfo::SharedInfo()
123 // Use relaxed memory ordering here. Since we are creating a new SharedInfo,
124 // no other threads should be referring to it yet.
125 refcount.store(1, std::memory_order_relaxed);
128 IOBuf::SharedInfo::SharedInfo(FreeFunction fn, void* arg)
131 // Use relaxed memory ordering here. Since we are creating a new SharedInfo,
132 // no other threads should be referring to it yet.
133 refcount.store(1, std::memory_order_relaxed);
136 void* IOBuf::operator new(size_t size) {
137 size_t fullSize = offsetof(HeapStorage, buf) + size;
138 auto* storage = static_cast<HeapStorage*>(malloc(fullSize));
139 // operator new is not allowed to return NULL
140 if (UNLIKELY(storage == nullptr)) {
141 throw std::bad_alloc();
144 new (&storage->prefix) HeapPrefix(kIOBufInUse);
145 return &(storage->buf);
148 void* IOBuf::operator new(size_t /* size */, void* ptr) { return ptr; }
150 void IOBuf::operator delete(void* ptr) {
151 auto* storageAddr = static_cast<uint8_t*>(ptr) - offsetof(HeapStorage, buf);
152 auto* storage = reinterpret_cast<HeapStorage*>(storageAddr);
153 releaseStorage(storage, kIOBufInUse);
156 void IOBuf::releaseStorage(HeapStorage* storage, uint16_t freeFlags) {
157 CHECK_EQ(storage->prefix.magic, static_cast<uint16_t>(kHeapMagic));
159 // Use relaxed memory order here. If we are unlucky and happen to get
160 // out-of-date data the compare_exchange_weak() call below will catch
161 // it and load new data with memory_order_acq_rel.
162 auto flags = storage->prefix.flags.load(std::memory_order_acquire);
163 DCHECK_EQ((flags & freeFlags), freeFlags);
166 uint16_t newFlags = (flags & ~freeFlags);
168 // The storage space is now unused. Free it.
169 storage->prefix.HeapPrefix::~HeapPrefix();
174 // This storage segment still contains portions that are in use.
175 // Just clear the flags specified in freeFlags for now.
176 auto ret = storage->prefix.flags.compare_exchange_weak(
177 flags, newFlags, std::memory_order_acq_rel);
179 // We successfully updated the flags.
183 // We failed to update the flags. Some other thread probably updated them
184 // and cleared some of the other bits. Continue around the loop to see if
185 // we are the last user now, or if we need to try updating the flags again.
189 void IOBuf::freeInternalBuf(void* /* buf */, void* userData) {
190 auto* storage = static_cast<HeapStorage*>(userData);
191 releaseStorage(storage, kDataInUse);
194 IOBuf::IOBuf(CreateOp, uint64_t capacity)
199 flagsAndSharedInfo_(0) {
201 allocExtBuffer(capacity, &buf_, &info, &capacity_);
206 IOBuf::IOBuf(CopyBufferOp /* op */,
210 uint64_t minTailroom)
211 : IOBuf(CREATE, headroom + size + minTailroom) {
213 memcpy(writableData(), buf, size);
217 IOBuf::IOBuf(CopyBufferOp op, ByteRange br,
218 uint64_t headroom, uint64_t minTailroom)
219 : IOBuf(op, br.data(), br.size(), headroom, minTailroom) {
222 unique_ptr<IOBuf> IOBuf::create(uint64_t capacity) {
223 // For smaller-sized buffers, allocate the IOBuf, SharedInfo, and the buffer
224 // all with a single allocation.
226 // We don't do this for larger buffers since it can be wasteful if the user
227 // needs to reallocate the buffer but keeps using the same IOBuf object.
228 // In this case we can't free the data space until the IOBuf is also
229 // destroyed. Callers can explicitly call createCombined() or
230 // createSeparate() if they know their use case better, and know if they are
231 // likely to reallocate the buffer later.
232 if (capacity <= kDefaultCombinedBufSize) {
233 return createCombined(capacity);
235 return createSeparate(capacity);
238 unique_ptr<IOBuf> IOBuf::createCombined(uint64_t capacity) {
239 // To save a memory allocation, allocate space for the IOBuf object, the
240 // SharedInfo struct, and the data itself all with a single call to malloc().
241 size_t requiredStorage = offsetof(HeapFullStorage, align) + capacity;
242 size_t mallocSize = goodMallocSize(requiredStorage);
243 auto* storage = static_cast<HeapFullStorage*>(malloc(mallocSize));
245 new (&storage->hs.prefix) HeapPrefix(kIOBufInUse | kDataInUse);
246 new (&storage->shared) SharedInfo(freeInternalBuf, storage);
248 uint8_t* bufAddr = reinterpret_cast<uint8_t*>(&storage->align);
249 uint8_t* storageEnd = reinterpret_cast<uint8_t*>(storage) + mallocSize;
250 size_t actualCapacity = storageEnd - bufAddr;
251 unique_ptr<IOBuf> ret(new (&storage->hs.buf) IOBuf(
252 InternalConstructor(), packFlagsAndSharedInfo(0, &storage->shared),
253 bufAddr, actualCapacity, bufAddr, 0));
257 unique_ptr<IOBuf> IOBuf::createSeparate(uint64_t capacity) {
258 return make_unique<IOBuf>(CREATE, capacity);
261 unique_ptr<IOBuf> IOBuf::createChain(
262 size_t totalCapacity, uint64_t maxBufCapacity) {
263 unique_ptr<IOBuf> out = create(
264 std::min(totalCapacity, size_t(maxBufCapacity)));
265 size_t allocatedCapacity = out->capacity();
267 while (allocatedCapacity < totalCapacity) {
268 unique_ptr<IOBuf> newBuf = create(
269 std::min(totalCapacity - allocatedCapacity, size_t(maxBufCapacity)));
270 allocatedCapacity += newBuf->capacity();
271 out->prependChain(std::move(newBuf));
277 IOBuf::IOBuf(TakeOwnershipOp, void* buf, uint64_t capacity, uint64_t length,
278 FreeFunction freeFn, void* userData,
282 data_(static_cast<uint8_t*>(buf)),
283 buf_(static_cast<uint8_t*>(buf)),
286 flagsAndSharedInfo_(packFlagsAndSharedInfo(kFlagFreeSharedInfo, nullptr)) {
288 setSharedInfo(new SharedInfo(freeFn, userData));
290 takeOwnershipError(freeOnError, buf, freeFn, userData);
295 unique_ptr<IOBuf> IOBuf::takeOwnership(void* buf, uint64_t capacity,
301 // TODO: We could allocate the IOBuf object and SharedInfo all in a single
302 // memory allocation. We could use the existing HeapStorage class, and
303 // define a new kSharedInfoInUse flag. We could change our code to call
304 // releaseStorage(kFlagFreeSharedInfo) when this kFlagFreeSharedInfo,
305 // rather than directly calling delete.
307 // Note that we always pass freeOnError as false to the constructor.
308 // If the constructor throws we'll handle it below. (We have to handle
309 // allocation failures from make_unique too.)
310 return make_unique<IOBuf>(TAKE_OWNERSHIP, buf, capacity, length,
311 freeFn, userData, false);
313 takeOwnershipError(freeOnError, buf, freeFn, userData);
318 IOBuf::IOBuf(WrapBufferOp, const void* buf, uint64_t capacity)
319 : IOBuf(InternalConstructor(), 0,
320 // We cast away the const-ness of the buffer here.
321 // This is okay since IOBuf users must use unshare() to create a copy
322 // of this buffer before writing to the buffer.
323 static_cast<uint8_t*>(const_cast<void*>(buf)), capacity,
324 static_cast<uint8_t*>(const_cast<void*>(buf)), capacity) {
327 IOBuf::IOBuf(WrapBufferOp op, ByteRange br)
328 : IOBuf(op, br.data(), br.size()) {
331 unique_ptr<IOBuf> IOBuf::wrapBuffer(const void* buf, uint64_t capacity) {
332 return make_unique<IOBuf>(WRAP_BUFFER, buf, capacity);
335 IOBuf::IOBuf() noexcept {
338 IOBuf::IOBuf(IOBuf&& other) noexcept
339 : data_(other.data_),
341 length_(other.length_),
342 capacity_(other.capacity_),
343 flagsAndSharedInfo_(other.flagsAndSharedInfo_) {
344 // Reset other so it is a clean state to be destroyed.
345 other.data_ = nullptr;
346 other.buf_ = nullptr;
349 other.flagsAndSharedInfo_ = 0;
351 // If other was part of the chain, assume ownership of the rest of its chain.
352 // (It's only valid to perform move assignment on the head of a chain.)
353 if (other.next_ != &other) {
356 other.next_ = &other;
360 other.prev_ = &other;
363 // Sanity check to make sure that other is in a valid state to be destroyed.
364 DCHECK_EQ(other.prev_, &other);
365 DCHECK_EQ(other.next_, &other);
368 IOBuf::IOBuf(const IOBuf& other) {
369 *this = other.cloneAsValue();
372 IOBuf::IOBuf(InternalConstructor,
373 uintptr_t flagsAndSharedInfo,
384 flagsAndSharedInfo_(flagsAndSharedInfo) {
386 assert(data + length <= buf + capacity);
390 // Destroying an IOBuf destroys the entire chain.
391 // Users of IOBuf should only explicitly delete the head of any chain.
392 // The other elements in the chain will be automatically destroyed.
393 while (next_ != this) {
394 // Since unlink() returns unique_ptr() and we don't store it,
395 // it will automatically delete the unlinked element.
396 (void)next_->unlink();
402 IOBuf& IOBuf::operator=(IOBuf&& other) noexcept {
403 if (this == &other) {
407 // If we are part of a chain, delete the rest of the chain.
408 while (next_ != this) {
409 // Since unlink() returns unique_ptr() and we don't store it,
410 // it will automatically delete the unlinked element.
411 (void)next_->unlink();
414 // Decrement our refcount on the current buffer
417 // Take ownership of the other buffer's data
420 length_ = other.length_;
421 capacity_ = other.capacity_;
422 flagsAndSharedInfo_ = other.flagsAndSharedInfo_;
423 // Reset other so it is a clean state to be destroyed.
424 other.data_ = nullptr;
425 other.buf_ = nullptr;
428 other.flagsAndSharedInfo_ = 0;
430 // If other was part of the chain, assume ownership of the rest of its chain.
431 // (It's only valid to perform move assignment on the head of a chain.)
432 if (other.next_ != &other) {
435 other.next_ = &other;
439 other.prev_ = &other;
442 // Sanity check to make sure that other is in a valid state to be destroyed.
443 DCHECK_EQ(other.prev_, &other);
444 DCHECK_EQ(other.next_, &other);
449 IOBuf& IOBuf::operator=(const IOBuf& other) {
450 if (this != &other) {
451 *this = IOBuf(other);
456 bool IOBuf::empty() const {
457 const IOBuf* current = this;
459 if (current->length() != 0) {
462 current = current->next_;
463 } while (current != this);
467 size_t IOBuf::countChainElements() const {
468 size_t numElements = 1;
469 for (IOBuf* current = next_; current != this; current = current->next_) {
475 uint64_t IOBuf::computeChainDataLength() const {
476 uint64_t fullLength = length_;
477 for (IOBuf* current = next_; current != this; current = current->next_) {
478 fullLength += current->length_;
483 void IOBuf::prependChain(unique_ptr<IOBuf>&& iobuf) {
484 // Take ownership of the specified IOBuf
485 IOBuf* other = iobuf.release();
487 // Remember the pointer to the tail of the other chain
488 IOBuf* otherTail = other->prev_;
490 // Hook up prev_->next_ to point at the start of the other chain,
491 // and other->prev_ to point at prev_
492 prev_->next_ = other;
493 other->prev_ = prev_;
495 // Hook up otherTail->next_ to point at us,
496 // and prev_ to point back at otherTail,
497 otherTail->next_ = this;
501 unique_ptr<IOBuf> IOBuf::clone() const {
502 return make_unique<IOBuf>(cloneAsValue());
505 unique_ptr<IOBuf> IOBuf::cloneOne() const {
506 return make_unique<IOBuf>(cloneOneAsValue());
509 IOBuf IOBuf::cloneAsValue() const {
510 auto tmp = cloneOneAsValue();
512 for (IOBuf* current = next_; current != this; current = current->next_) {
513 tmp.prependChain(current->cloneOne());
519 IOBuf IOBuf::cloneOneAsValue() const {
520 if (SharedInfo* info = sharedInfo()) {
521 setFlags(kFlagMaybeShared);
522 info->refcount.fetch_add(1, std::memory_order_acq_rel);
525 InternalConstructor(),
533 void IOBuf::unshareOneSlow() {
534 // Allocate a new buffer for the data
536 SharedInfo* sharedInfo;
537 uint64_t actualCapacity;
538 allocExtBuffer(capacity_, &buf, &sharedInfo, &actualCapacity);
541 // Maintain the same amount of headroom. Since we maintained the same
542 // minimum capacity we also maintain at least the same amount of tailroom.
543 uint64_t headlen = headroom();
544 memcpy(buf + headlen, data_, length_);
546 // Release our reference on the old buffer
548 // Make sure kFlagMaybeShared and kFlagFreeSharedInfo are all cleared.
549 setFlagsAndSharedInfo(0, sharedInfo);
551 // Update the buffer pointers to point to the new buffer
552 data_ = buf + headlen;
556 void IOBuf::unshareChained() {
557 // unshareChained() should only be called if we are part of a chain of
558 // multiple IOBufs. The caller should have already verified this.
561 IOBuf* current = this;
563 if (current->isSharedOne()) {
564 // we have to unshare
568 current = current->next_;
569 if (current == this) {
570 // None of the IOBufs in the chain are shared,
571 // so return without doing anything
576 // We have to unshare. Let coalesceSlow() do the work.
580 void IOBuf::makeManagedChained() {
583 IOBuf* current = this;
585 current->makeManagedOne();
586 current = current->next_;
587 if (current == this) {
593 void IOBuf::coalesceSlow() {
594 // coalesceSlow() should only be called if we are part of a chain of multiple
595 // IOBufs. The caller should have already verified this.
598 // Compute the length of the entire chain
599 uint64_t newLength = 0;
602 newLength += end->length_;
604 } while (end != this);
606 coalesceAndReallocate(newLength, end);
607 // We should be only element left in the chain now
608 DCHECK(!isChained());
611 void IOBuf::coalesceSlow(size_t maxLength) {
612 // coalesceSlow() should only be called if we are part of a chain of multiple
613 // IOBufs. The caller should have already verified this.
615 DCHECK_LT(length_, maxLength);
617 // Compute the length of the entire chain
618 uint64_t newLength = 0;
621 newLength += end->length_;
623 if (newLength >= maxLength) {
627 throw std::overflow_error("attempted to coalesce more data than "
632 coalesceAndReallocate(newLength, end);
633 // We should have the requested length now
634 DCHECK_GE(length_, maxLength);
637 void IOBuf::coalesceAndReallocate(size_t newHeadroom,
640 size_t newTailroom) {
641 uint64_t newCapacity = newLength + newHeadroom + newTailroom;
643 // Allocate space for the coalesced buffer.
644 // We always convert to an external buffer, even if we happened to be an
645 // internal buffer before.
648 uint64_t actualCapacity;
649 allocExtBuffer(newCapacity, &newBuf, &newInfo, &actualCapacity);
651 // Copy the data into the new buffer
652 uint8_t* newData = newBuf + newHeadroom;
653 uint8_t* p = newData;
654 IOBuf* current = this;
655 size_t remaining = newLength;
657 assert(current->length_ <= remaining);
658 remaining -= current->length_;
659 memcpy(p, current->data_, current->length_);
660 p += current->length_;
661 current = current->next_;
662 } while (current != end);
663 assert(remaining == 0);
665 // Point at the new buffer
668 // Make sure kFlagMaybeShared and kFlagFreeSharedInfo are all cleared.
669 setFlagsAndSharedInfo(0, newInfo);
671 capacity_ = actualCapacity;
676 // Separate from the rest of our chain.
677 // Since we don't store the unique_ptr returned by separateChain(),
678 // this will immediately delete the returned subchain.
680 (void)separateChain(next_, current->prev_);
684 void IOBuf::decrementRefcount() {
685 // Externally owned buffers don't have a SharedInfo object and aren't managed
686 // by the reference count
687 SharedInfo* info = sharedInfo();
692 // Decrement the refcount
693 uint32_t newcnt = info->refcount.fetch_sub(
694 1, std::memory_order_acq_rel);
695 // Note that fetch_sub() returns the value before we decremented.
696 // If it is 1, we were the only remaining user; if it is greater there are
697 // still other users.
702 // We were the last user. Free the buffer
705 // Free the SharedInfo if it was allocated separately.
707 // This is only used by takeOwnership().
709 // To avoid this special case handling in decrementRefcount(), we could have
710 // takeOwnership() set a custom freeFn() that calls the user's free function
711 // then frees the SharedInfo object. (This would require that
712 // takeOwnership() store the user's free function with its allocated
713 // SharedInfo object.) However, handling this specially with a flag seems
714 // like it shouldn't be problematic.
715 if (flags() & kFlagFreeSharedInfo) {
720 void IOBuf::reserveSlow(uint64_t minHeadroom, uint64_t minTailroom) {
721 size_t newCapacity = (size_t)length_ + minHeadroom + minTailroom;
722 DCHECK_LT(newCapacity, UINT32_MAX);
724 // reserveSlow() is dangerous if anyone else is sharing the buffer, as we may
725 // reallocate and free the original buffer. It should only ever be called if
726 // we are the only user of the buffer.
727 DCHECK(!isSharedOne());
729 // We'll need to reallocate the buffer.
730 // There are a few options.
731 // - If we have enough total room, move the data around in the buffer
732 // and adjust the data_ pointer.
733 // - If we're using an internal buffer, we'll switch to an external
734 // buffer with enough headroom and tailroom.
735 // - If we have enough headroom (headroom() >= minHeadroom) but not too much
736 // (so we don't waste memory), we can try one of two things, depending on
737 // whether we use jemalloc or not:
738 // - If using jemalloc, we can try to expand in place, avoiding a memcpy()
739 // - If not using jemalloc and we don't have too much to copy,
740 // we'll use realloc() (note that realloc might have to copy
741 // headroom + data + tailroom, see smartRealloc in folly/Malloc.h)
742 // - Otherwise, bite the bullet and reallocate.
743 if (headroom() + tailroom() >= minHeadroom + minTailroom) {
744 uint8_t* newData = writableBuffer() + minHeadroom;
745 memmove(newData, data_, length_);
750 size_t newAllocatedCapacity = 0;
751 uint8_t* newBuffer = nullptr;
752 uint64_t newHeadroom = 0;
753 uint64_t oldHeadroom = headroom();
755 // If we have a buffer allocated with malloc and we just need more tailroom,
756 // try to use realloc()/xallocx() to grow the buffer in place.
757 SharedInfo* info = sharedInfo();
758 if (info && (info->freeFn == nullptr) && length_ != 0 &&
759 oldHeadroom >= minHeadroom) {
760 size_t headSlack = oldHeadroom - minHeadroom;
761 newAllocatedCapacity = goodExtBufferSize(newCapacity + headSlack);
762 if (usingJEMalloc()) {
763 // We assume that tailroom is more useful and more important than
764 // headroom (not least because realloc / xallocx allow us to grow the
765 // buffer at the tail, but not at the head) So, if we have more headroom
766 // than we need, we consider that "wasted". We arbitrarily define "too
767 // much" headroom to be 25% of the capacity.
768 if (headSlack * 4 <= newCapacity) {
769 size_t allocatedCapacity = capacity() + sizeof(SharedInfo);
771 if (allocatedCapacity >= jemallocMinInPlaceExpandable) {
772 if (xallocx(p, newAllocatedCapacity, 0, 0) == newAllocatedCapacity) {
773 newBuffer = static_cast<uint8_t*>(p);
774 newHeadroom = oldHeadroom;
776 // if xallocx failed, do nothing, fall back to malloc/memcpy/free
779 } else { // Not using jemalloc
780 size_t copySlack = capacity() - length_;
781 if (copySlack * 2 <= length_) {
782 void* p = realloc(buf_, newAllocatedCapacity);
783 if (UNLIKELY(p == nullptr)) {
784 throw std::bad_alloc();
786 newBuffer = static_cast<uint8_t*>(p);
787 newHeadroom = oldHeadroom;
792 // None of the previous reallocation strategies worked (or we're using
793 // an internal buffer). malloc/copy/free.
794 if (newBuffer == nullptr) {
795 newAllocatedCapacity = goodExtBufferSize(newCapacity);
796 void* p = malloc(newAllocatedCapacity);
797 if (UNLIKELY(p == nullptr)) {
798 throw std::bad_alloc();
800 newBuffer = static_cast<uint8_t*>(p);
801 memcpy(newBuffer + minHeadroom, data_, length_);
805 newHeadroom = minHeadroom;
809 initExtBuffer(newBuffer, newAllocatedCapacity, &info, &cap);
811 if (flags() & kFlagFreeSharedInfo) {
815 setFlagsAndSharedInfo(0, info);
818 data_ = newBuffer + newHeadroom;
819 // length_ is unchanged
822 void IOBuf::freeExtBuffer() {
823 SharedInfo* info = sharedInfo();
828 info->freeFn(buf_, info->userData);
830 // The user's free function should never throw. Otherwise we might
831 // throw from the IOBuf destructor. Other code paths like coalesce()
832 // also assume that decrementRefcount() cannot throw.
840 void IOBuf::allocExtBuffer(uint64_t minCapacity,
842 SharedInfo** infoReturn,
843 uint64_t* capacityReturn) {
844 size_t mallocSize = goodExtBufferSize(minCapacity);
845 uint8_t* buf = static_cast<uint8_t*>(malloc(mallocSize));
846 if (UNLIKELY(buf == nullptr)) {
847 throw std::bad_alloc();
849 initExtBuffer(buf, mallocSize, infoReturn, capacityReturn);
853 size_t IOBuf::goodExtBufferSize(uint64_t minCapacity) {
854 // Determine how much space we should allocate. We'll store the SharedInfo
855 // for the external buffer just after the buffer itself. (We store it just
856 // after the buffer rather than just before so that the code can still just
857 // use free(buf_) to free the buffer.)
858 size_t minSize = static_cast<size_t>(minCapacity) + sizeof(SharedInfo);
859 // Add room for padding so that the SharedInfo will be aligned on an 8-byte
861 minSize = (minSize + 7) & ~7;
863 // Use goodMallocSize() to bump up the capacity to a decent size to request
864 // from malloc, so we can use all of the space that malloc will probably give
866 return goodMallocSize(minSize);
869 void IOBuf::initExtBuffer(uint8_t* buf, size_t mallocSize,
870 SharedInfo** infoReturn,
871 uint64_t* capacityReturn) {
872 // Find the SharedInfo storage at the end of the buffer
873 // and construct the SharedInfo.
874 uint8_t* infoStart = (buf + mallocSize) - sizeof(SharedInfo);
875 SharedInfo* sharedInfo = new(infoStart) SharedInfo;
877 *capacityReturn = infoStart - buf;
878 *infoReturn = sharedInfo;
881 fbstring IOBuf::moveToFbString() {
882 // malloc-allocated buffers are just fine, everything else needs
883 // to be turned into one.
884 if (!sharedInfo() || // user owned, not ours to give up
885 sharedInfo()->freeFn || // not malloc()-ed
886 headroom() != 0 || // malloc()-ed block doesn't start at beginning
887 tailroom() == 0 || // no room for NUL terminator
888 isShared() || // shared
889 isChained()) { // chained
890 // We might as well get rid of all head and tailroom if we're going
891 // to reallocate; we need 1 byte for NUL terminator.
892 coalesceAndReallocate(0, computeChainDataLength(), this, 1);
895 // Ensure NUL terminated
897 fbstring str(reinterpret_cast<char*>(writableData()),
898 length(), capacity(),
899 AcquireMallocatedString());
901 if (flags() & kFlagFreeSharedInfo) {
905 // Reset to a state where we can be deleted cleanly
906 flagsAndSharedInfo_ = 0;
912 IOBuf::Iterator IOBuf::cbegin() const {
913 return Iterator(this, this);
916 IOBuf::Iterator IOBuf::cend() const {
917 return Iterator(nullptr, nullptr);
920 folly::fbvector<struct iovec> IOBuf::getIov() const {
921 folly::fbvector<struct iovec> iov;
922 iov.reserve(countChainElements());
927 void IOBuf::appendToIov(folly::fbvector<struct iovec>* iov) const {
928 IOBuf const* p = this;
930 // some code can get confused by empty iovs, so skip them
931 if (p->length() > 0) {
932 iov->push_back({(void*)p->data(), folly::to<size_t>(p->length())});
938 size_t IOBuf::fillIov(struct iovec* iov, size_t len) const {
939 IOBuf const* p = this;
942 // some code can get confused by empty iovs, so skip them
943 if (p->length() > 0) {
944 iov[i].iov_base = const_cast<uint8_t*>(p->data());
945 iov[i].iov_len = p->length();
956 size_t IOBufHash::operator()(const IOBuf& buf) const {
957 folly::hash::SpookyHashV2 hasher;
959 io::Cursor cursor(&buf);
961 auto p = cursor.peek();
965 hasher.Update(p.first, p.second);
966 cursor.skip(p.second);
970 hasher.Final(&h1, &h2);
974 bool IOBufEqual::operator()(const IOBuf& a, const IOBuf& b) const {
980 if (pa.second == 0 && pb.second == 0) {
982 } else if (pa.second == 0 || pb.second == 0) {
985 size_t n = std::min(pa.second, pb.second);
987 if (memcmp(pa.first, pb.first, n)) {