X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=folly%2Fio%2FIOBuf.h;h=1727b6e46bcb516054a39a04a4a4cc573932eba6;hb=156f70334627e5a100283b5334c0a2121d5a45a2;hp=ebba90a8b9884a32876f08bcb5bb5dc9f268859d;hpb=cb5272724fbe18e937bbe8bb8a0a279027255032;p=folly.git diff --git a/folly/io/IOBuf.h b/folly/io/IOBuf.h index ebba90a8..1727b6e4 100644 --- a/folly/io/IOBuf.h +++ b/folly/io/IOBuf.h @@ -1,5 +1,5 @@ /* - * Copyright 2013 Facebook, Inc. + * Copyright 2016 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ * limitations under the License. */ -#ifndef FOLLY_IO_IOBUF_H_ -#define FOLLY_IO_IOBUF_H_ +#pragma once #include #include @@ -25,14 +24,14 @@ #include #include #include -#include #include #include -#include "folly/FBString.h" -#include "folly/Range.h" -#include "folly/FBVector.h" +#include +#include +#include +#include // Ignore shadowing warnings within this file, so includers can use -Wshadow. #pragma GCC diagnostic push @@ -165,27 +164,49 @@ namespace folly { * accessed by multiple threads. * * - * IOBuf Object Allocation/Sharing - * ------------------------------- + * IOBuf Object Allocation + * ----------------------- * - * IOBuf objects themselves are always allocated on the heap. The IOBuf - * constructors are private, so IOBuf objects may not be created on the stack. - * In part this is done since some IOBuf objects use small-buffer optimization - * and contain the buffer data immediately after the IOBuf object itself. The - * coalesce() and unshare() methods also expect to be able to delete subsequent - * IOBuf objects in the chain if they are no longer needed due to coalescing. + * IOBuf objects themselves exist separately from the data buffer they point + * to. Therefore one must also consider how to allocate and manage the IOBuf + * objects. * - * The IOBuf structure also does not provide room for an intrusive refcount on - * the IOBuf object itself, only the underlying data buffer is reference - * counted. If users want to share the same IOBuf object between multiple - * parts of the code, they are responsible for managing this sharing on their - * own. (For example, by using a shared_ptr. Alternatively, users always have - * the option of using clone() to create a second IOBuf that points to the same - * underlying buffer.) + * It is more common to allocate IOBuf objects on the heap, using the create(), + * takeOwnership(), or wrapBuffer() factory functions. The clone()/cloneOne() + * functions also return new heap-allocated IOBufs. The createCombined() + * function allocates the IOBuf object and data storage space together, in a + * single memory allocation. This can improve performance, particularly if you + * know that the data buffer and the IOBuf itself will have similar lifetimes. * - * With jemalloc, allocating small objects like IOBuf objects should be - * relatively fast, and the cost of allocating IOBuf objects on the heap and - * cloning new IOBufs should be relatively cheap. + * That said, it is also possible to allocate IOBufs on the stack or inline + * inside another object as well. This is useful for cases where the IOBuf is + * short-lived, or when the overhead of allocating the IOBuf on the heap is + * undesirable. + * + * However, note that stack-allocated IOBufs may only be used as the head of a + * chain (or standalone as the only IOBuf in a chain). All non-head members of + * an IOBuf chain must be heap allocated. (All functions to add nodes to a + * chain require a std::unique_ptr, which enforces this requrement.) + * + * Copying IOBufs is only meaningful for the head of a chain. The entire chain + * is cloned; the IOBufs will become shared, and the old and new IOBufs will + * refer to the same underlying memory. + * + * IOBuf Sharing + * ------------- + * + * The IOBuf class manages sharing of the underlying buffer that it points to, + * maintaining a reference count if multiple IOBufs are pointing at the same + * buffer. + * + * However, it is the callers responsibility to manage sharing and ownership of + * IOBuf objects themselves. The IOBuf structure does not provide room for an + * intrusive refcount on the IOBuf object itself, only the underlying data + * buffer is reference counted. If users want to share the same IOBuf object + * between multiple parts of the code, they are responsible for managing this + * sharing on their own. (For example, by using a shared_ptr. Alternatively, + * users always have the option of using clone() to create a second IOBuf that + * points to the same underlying buffer.) */ namespace detail { // Is T a unique_ptr<> to a standard-layout type? @@ -202,6 +223,11 @@ class IOBuf { public: class Iterator; + enum CreateOp { CREATE }; + enum WrapBufferOp { WRAP_BUFFER }; + enum TakeOwnershipOp { TAKE_OWNERSHIP }; + enum CopyBufferOp { COPY_BUFFER }; + typedef ByteRange value_type; typedef Iterator iterator; typedef Iterator const_iterator; @@ -220,7 +246,8 @@ class IOBuf { * * Throws std::bad_alloc on error. */ - static std::unique_ptr create(uint32_t capacity); + static std::unique_ptr create(uint64_t capacity); + IOBuf(CreateOp, uint64_t capacity); /** * Create a new IOBuf, using a single memory allocation to allocate space @@ -232,7 +259,7 @@ class IOBuf { * IOBuf object itself is also freed. (It can also be slightly wasteful in * some cases where you clone this IOBuf and then free the original IOBuf.) */ - static std::unique_ptr createCombined(uint32_t capacity); + static std::unique_ptr createCombined(uint64_t capacity); /** * Create a new IOBuf, using separate memory allocations for the IOBuf object @@ -241,14 +268,14 @@ class IOBuf { * This requires two memory allocations, but saves space in the long run * if you know that you will need to reallocate the data buffer later. */ - static std::unique_ptr createSeparate(uint32_t capacity); + static std::unique_ptr createSeparate(uint64_t capacity); /** * Allocate a new IOBuf chain with the requested total capacity, allocating * no more than maxBufCapacity to each buffer. */ static std::unique_ptr createChain( - size_t totalCapacity, uint32_t maxBufCapacity); + size_t totalCapacity, uint64_t maxBufCapacity); /** * Create a new IOBuf pointing to an existing data buffer. @@ -274,19 +301,26 @@ class IOBuf { * On error, std::bad_alloc will be thrown. If freeOnError is true (the * default) the buffer will be freed before throwing the error. */ - static std::unique_ptr takeOwnership(void* buf, uint32_t capacity, - FreeFunction freeFn = NULL, - void* userData = NULL, + static std::unique_ptr takeOwnership(void* buf, uint64_t capacity, + FreeFunction freeFn = nullptr, + void* userData = nullptr, bool freeOnError = true) { return takeOwnership(buf, capacity, capacity, freeFn, userData, freeOnError); } - - static std::unique_ptr takeOwnership(void* buf, uint32_t capacity, - uint32_t length, - FreeFunction freeFn = NULL, - void* userData = NULL, + IOBuf(TakeOwnershipOp op, void* buf, uint64_t capacity, + FreeFunction freeFn = nullptr, void* userData = nullptr, + bool freeOnError = true) + : IOBuf(op, buf, capacity, capacity, freeFn, userData, freeOnError) {} + + static std::unique_ptr takeOwnership(void* buf, uint64_t capacity, + uint64_t length, + FreeFunction freeFn = nullptr, + void* userData = nullptr, bool freeOnError = true); + IOBuf(TakeOwnershipOp, void* buf, uint64_t capacity, uint64_t length, + FreeFunction freeFn = nullptr, void* userData = nullptr, + bool freeOnError = true); /** * Create a new IOBuf pointing to an existing data buffer made up of @@ -333,26 +367,40 @@ class IOBuf { * * On error, std::bad_alloc will be thrown. */ - static std::unique_ptr wrapBuffer(const void* buf, uint32_t capacity); + static std::unique_ptr wrapBuffer(const void* buf, uint64_t capacity); static std::unique_ptr wrapBuffer(ByteRange br) { - CHECK_LE(br.size(), std::numeric_limits::max()); return wrapBuffer(br.data(), br.size()); } + /** + * Similar to wrapBuffer(), but returns IOBuf by value rather than + * heap-allocating it. + */ + static IOBuf wrapBufferAsValue(const void* buf, uint64_t capacity); + static IOBuf wrapBufferAsValue(ByteRange br) { + return wrapBufferAsValue(br.data(), br.size()); + } + + IOBuf(WrapBufferOp op, const void* buf, uint64_t capacity); + IOBuf(WrapBufferOp op, ByteRange br); + /** * Convenience function to create a new IOBuf object that copies data from a * user-supplied buffer, optionally allocating a given amount of * headroom and tailroom. */ - static std::unique_ptr copyBuffer(const void* buf, uint32_t size, - uint32_t headroom=0, - uint32_t minTailroom=0); + static std::unique_ptr copyBuffer(const void* buf, uint64_t size, + uint64_t headroom=0, + uint64_t minTailroom=0); static std::unique_ptr copyBuffer(ByteRange br, - uint32_t headroom=0, - uint32_t minTailroom=0) { - CHECK_LE(br.size(), std::numeric_limits::max()); + uint64_t headroom=0, + uint64_t minTailroom=0) { return copyBuffer(br.data(), br.size(), headroom, minTailroom); } + IOBuf(CopyBufferOp op, const void* buf, uint64_t size, + uint64_t headroom=0, uint64_t minTailroom=0); + IOBuf(CopyBufferOp op, ByteRange br, + uint64_t headroom=0, uint64_t minTailroom=0); /** * Convenience function to create a new IOBuf object that copies data from a @@ -366,16 +414,19 @@ class IOBuf { * copyBuffer() above, with the size argument of 3. */ static std::unique_ptr copyBuffer(const std::string& buf, - uint32_t headroom=0, - uint32_t minTailroom=0); + uint64_t headroom=0, + uint64_t minTailroom=0); + IOBuf(CopyBufferOp op, const std::string& buf, + uint64_t headroom=0, uint64_t minTailroom=0) + : IOBuf(op, buf.data(), buf.size(), headroom, minTailroom) {} /** * A version of copyBuffer() that returns a null pointer if the input string * is empty. */ static std::unique_ptr maybeCopyBuffer(const std::string& buf, - uint32_t headroom=0, - uint32_t minTailroom=0); + uint64_t headroom=0, + uint64_t minTailroom=0); /** * Convenience function to free a chain of IOBufs held by a unique_ptr. @@ -446,7 +497,7 @@ class IOBuf { /** * Get the data length. */ - uint32_t length() const { + uint64_t length() const { return length_; } @@ -455,7 +506,7 @@ class IOBuf { * * Returns the number of bytes in the buffer before the start of the data. */ - uint32_t headroom() const { + uint64_t headroom() const { return data_ - buffer(); } @@ -464,7 +515,7 @@ class IOBuf { * * Returns the number of bytes in the buffer after the end of the data. */ - uint32_t tailroom() const { + uint64_t tailroom() const { return bufferEnd() - tail(); } @@ -506,7 +557,7 @@ class IOBuf { * This returns the total usable length of the buffer. Use the length() * method to get the length of the actual valid data in this IOBuf. */ - uint32_t capacity() const { + uint64_t capacity() const { return capacity_; } @@ -545,7 +596,7 @@ class IOBuf { * for making sure the buffer is unshared, so it will not affect other IOBufs * that may be sharing the same underlying buffer. */ - void advance(uint32_t amount) { + void advance(uint64_t amount) { // In debug builds, assert if there is a problem. assert(amount <= tailroom()); @@ -566,7 +617,7 @@ class IOBuf { * for making sure the buffer is unshared, so it will not affect other IOBufs * that may be sharing the same underlying buffer. */ - void retreat(uint32_t amount) { + void retreat(uint64_t amount) { // In debug builds, assert if there is a problem. assert(amount <= headroom()); @@ -586,7 +637,7 @@ class IOBuf { * * This does not modify any actual data in the buffer. */ - void prepend(uint32_t amount) { + void prepend(uint64_t amount) { DCHECK_LE(amount, headroom()); data_ -= amount; length_ += amount; @@ -602,7 +653,7 @@ class IOBuf { * * This does not modify any actual data in the buffer. */ - void append(uint32_t amount) { + void append(uint64_t amount) { DCHECK_LE(amount, tailroom()); length_ += amount; } @@ -616,7 +667,7 @@ class IOBuf { * * This does not modify any actual data in the buffer. */ - void trimStart(uint32_t amount) { + void trimStart(uint64_t amount) { DCHECK_LE(amount, length_); data_ += amount; length_ -= amount; @@ -631,7 +682,7 @@ class IOBuf { * * This does not modify any actual data in the buffer. */ - void trimEnd(uint32_t amount) { + void trimEnd(uint64_t amount) { DCHECK_LE(amount, length_); length_ -= amount; } @@ -654,7 +705,7 @@ class IOBuf { * Postcondition: headroom() >= minHeadroom, tailroom() >= minTailroom, * the data (between data() and data() + length()) is preserved. */ - void reserve(uint32_t minHeadroom, uint32_t minTailroom) { + void reserve(uint64_t minHeadroom, uint64_t minTailroom) { // Maybe we don't need to do anything. if (headroom() >= minHeadroom && tailroom() >= minTailroom) { return; @@ -686,7 +737,7 @@ class IOBuf { * Use isChained() if you just want to check if this IOBuf is part of a chain * or not. */ - uint32_t countChainElements() const; + size_t countChainElements() const; /** * Get the length of all the data in this IOBuf chain. @@ -779,7 +830,7 @@ class IOBuf { prev_->next_ = next_; prev_ = this; next_ = this; - return std::unique_ptr((next == this) ? NULL : next); + return std::unique_ptr((next == this) ? nullptr : next); } /** @@ -829,6 +880,33 @@ class IOBuf { } } + /** + * Return true if all IOBufs in this chain are managed by the usual + * refcounting mechanism (and so the lifetime of the underlying memory + * can be extended by clone()). + */ + bool isManaged() const { + const IOBuf* current = this; + while (true) { + if (!current->isManagedOne()) { + return false; + } + current = current->next_; + if (current == this) { + return true; + } + } + } + + /** + * Return true if this IOBuf is managed by the usual refcounting mechanism + * (and so the lifetime of the underlying memory can be extended by + * cloneOne()). + */ + bool isManagedOne() const { + return sharedInfo(); + } + /** * Return true if other IOBufs are also pointing to the buffer used by this * IOBuf, and false otherwise. @@ -840,23 +918,26 @@ class IOBuf { * This only checks the current IOBuf, and not other IOBufs in the chain. */ bool isSharedOne() const { - if (LIKELY(flags_ & (kFlagUserOwned | kFlagMaybeShared)) == 0) { - return false; + // If this is a user-owned buffer, it is always considered shared + if (UNLIKELY(!sharedInfo())) { + return true; } - // If this is a user-owned buffer, it is always considered shared - if (flags_ & kFlagUserOwned) { + if (UNLIKELY(sharedInfo()->externallyShared)) { return true; } + if (LIKELY(!(flags() & kFlagMaybeShared))) { + return false; + } + // kFlagMaybeShared is set, so we need to check the reference count. // (Checking the reference count requires an atomic operation, which is why // we prefer to only check kFlagMaybeShared if possible.) - DCHECK(flags_ & kFlagMaybeShared); - bool shared = sharedInfo_->refcount.load(std::memory_order_acquire) > 1; + bool shared = sharedInfo()->refcount.load(std::memory_order_acquire) > 1; if (!shared) { // we're the last one left - flags_ &= ~kFlagMaybeShared; + clearFlags(kFlagMaybeShared); } return shared; } @@ -905,6 +986,63 @@ class IOBuf { } } + /** + * Mark the underlying buffers in this chain as shared with external memory + * management mechanism. This will make isShared() always returns true. + * + * This function is not thread-safe, and only safe to call immediately after + * creating an IOBuf, before it has been shared with other threads. + */ + void markExternallyShared(); + + /** + * Mark the underlying buffer that this IOBuf refers to as shared with + * external memory management mechanism. This will make isSharedOne() always + * returns true. + * + * This function is not thread-safe, and only safe to call immediately after + * creating an IOBuf, before it has been shared with other threads. + */ + void markExternallySharedOne() { + SharedInfo* info = sharedInfo(); + if (info) { + info->externallyShared = true; + } + } + + /** + * Ensure that the memory that IOBufs in this chain refer to will continue to + * be allocated for as long as the IOBufs of the chain (or any clone()s + * created from this point onwards) is alive. + * + * This only has an effect for user-owned buffers (created with the + * WRAP_BUFFER constructor or wrapBuffer factory function), in which case + * those buffers are unshared. + */ + void makeManaged() { + if (isChained()) { + makeManagedChained(); + } else { + makeManagedOne(); + } + } + + /** + * Ensure that the memory that this IOBuf refers to will continue to be + * allocated for as long as this IOBuf (or any clone()s created from this + * point onwards) is alive. + * + * This only has an effect for user-owned buffers (created with the + * WRAP_BUFFER constructor or wrapBuffer factory function), in which case + * those buffers are unshared. + */ + void makeManagedOne() { + if (!isManagedOne()) { + // We can call the internal function directly; unmanaged implies shared. + unshareOneSlow(); + } + } + /** * Coalesce this IOBuf chain into a single buffer. * @@ -918,8 +1056,7 @@ class IOBuf { * in the chain. * * Throws std::bad_alloc on error. On error the IOBuf chain will be - * unmodified. Throws std::overflow_error if the length of the entire chain - * larger than can be described by a uint32_t capacity. + * unmodified. * * Returns ByteRange that points to the data IOBuf stores. */ @@ -941,18 +1078,15 @@ class IOBuf { * first IOBuf in the chain, and at least as much tailroom as the last IOBuf * that was coalesced. * - * Throws std::bad_alloc on error. On error the IOBuf chain will be - * unmodified. Throws std::overflow_error if the length of the coalesced - * portion of the chain is larger than can be described by a uint32_t - * capacity. (Although maxLength is uint32_t, gather() doesn't split - * buffers, so coalescing whole buffers may result in a capacity that can't - * be described in uint32_t. + * Throws std::bad_alloc or std::overflow_error on error. On error the IOBuf + * chain will be unmodified. Throws std::overflow_error if maxLength is + * longer than the total chain length. * * Upon return, either enough of the chain was coalesced into a contiguous * region, or the entire chain was coalesced. That is, * length() >= maxLength || !isChained() is true. */ - void gather(uint32_t maxLength) { + void gather(uint64_t maxLength) { if (!isChained() || length_ >= maxLength) { return; } @@ -969,6 +1103,12 @@ class IOBuf { */ std::unique_ptr clone() const; + /** + * Similar to clone(). But returns IOBuf by value rather than heap-allocating + * it. + */ + IOBuf cloneAsValue() const; + /** * Return a new IOBuf with the same data as this IOBuf. * @@ -977,6 +1117,28 @@ class IOBuf { */ std::unique_ptr cloneOne() const; + /** + * Similar to cloneOne(). But returns IOBuf by value rather than + * heap-allocating it. + */ + IOBuf cloneOneAsValue() const; + + /** + * Similar to Clone(). But use other as the head node. Other nodes in the + * chain (if any) will be allocted on heap. + */ + void cloneInto(IOBuf& other) const { + other = cloneAsValue(); + } + + /** + * Similar to CloneOne(). But to fill an existing IOBuf instead of a new + * IOBuf. + */ + void cloneOneInto(IOBuf& other) const { + other = cloneOneAsValue(); + } + /** * Return an iovector suitable for e.g. writev() * @@ -988,6 +1150,29 @@ class IOBuf { */ folly::fbvector getIov() const; + /** + * Update an existing iovec array with the IOBuf data. + * + * New iovecs will be appended to the existing vector; anything already + * present in the vector will be left unchanged. + * + * Naturally, the returned iovec data will be invalid if you modify the + * buffer chain. + */ + void appendToIov(folly::fbvector* iov) const; + + /** + * Fill an iovec array with the IOBuf data. + * + * Returns the number of iovec filled. If there are more buffer than + * iovec, returns 0. This version is suitable to use with stack iovec + * arrays. + * + * Naturally, the filled iovec data will be invalid if you modify the + * buffer chain. + */ + size_t fillIov(struct iovec* iov, size_t len) const; + /* * Overridden operator new and delete. * These perform specialized memory management to help support @@ -1015,22 +1200,46 @@ class IOBuf { Iterator begin() const; Iterator end() const; - private: - enum FlagsEnum : uint32_t { - kFlagUserOwned = 0x1, - kFlagFreeSharedInfo = 0x2, - kFlagMaybeShared = 0x4, - }; + /** + * Allocate a new null buffer. + * + * This can be used to allocate an empty IOBuf on the stack. It will have no + * space allocated for it. This is generally useful only to later use move + * assignment to fill out the IOBuf. + */ + IOBuf() noexcept; + + /** + * Move constructor and assignment operator. + * + * In general, you should only ever move the head of an IOBuf chain. + * Internal nodes in an IOBuf chain are owned by the head of the chain, and + * should not be moved from. (Technically, nothing prevents you from moving + * a non-head node, but the moved-to node will replace the moved-from node in + * the chain. This has implications for ownership, since non-head nodes are + * owned by the chain head. You are then responsible for relinquishing + * ownership of the moved-to node, and manually deleting the moved-from + * node.) + * + * With the move assignment operator, the destination of the move should be + * the head of an IOBuf chain or a solitary IOBuf not part of a chain. If + * the move destination is part of a chain, all other IOBufs in the chain + * will be deleted. + */ + IOBuf(IOBuf&& other) noexcept; + IOBuf& operator=(IOBuf&& other) noexcept; + + IOBuf(const IOBuf& other); + IOBuf& operator=(const IOBuf& other); - // Values for the type_ field. - // We currently don't really use this for anything, other than to have it - // around for debugging purposes. We store it at the moment just because we - // have the 4 extra bytes that would just be padding otherwise. - enum ExtBufTypeEnum { - kExtAllocated = 0, - kExtUserSupplied = 1, - kExtUserOwned = 2, - kCombinedAlloc = 3, + private: + enum FlagsEnum : uintptr_t { + // Adding any more flags would not work on 32-bit architectures, + // as these flags are stashed in the least significant 2 bits of a + // max-align-aligned pointer. + kFlagFreeSharedInfo = 0x1, + kFlagMaybeShared = 0x2, + kFlagMask = kFlagFreeSharedInfo | kFlagMaybeShared }; struct SharedInfo { @@ -1038,20 +1247,17 @@ class IOBuf { SharedInfo(FreeFunction fn, void* arg); // A pointer to a function to call to free the buffer when the refcount - // hits 0. If this is NULL, free() will be used instead. + // hits 0. If this is null, free() will be used instead. FreeFunction freeFn; void* userData; std::atomic refcount; + bool externallyShared{false}; }; // Helper structs for use by operator new and delete struct HeapPrefix; struct HeapStorage; struct HeapFullStorage; - // Forbidden copy constructor and assignment opererator - IOBuf(IOBuf const &); - IOBuf& operator=(IOBuf const &); - /** * Create a new IOBuf pointing to an external buffer. * @@ -1059,14 +1265,16 @@ class IOBuf { * IOBuf. The IOBuf constructor does not automatically increment the * reference count. */ - IOBuf(ExtBufTypeEnum type, uint32_t flags, - uint8_t* buf, uint32_t capacity, - uint8_t* data, uint32_t length, - SharedInfo* sharedInfo); + struct InternalConstructor {}; // avoid conflicts + IOBuf(InternalConstructor, uintptr_t flagsAndSharedInfo, + uint8_t* buf, uint64_t capacity, + uint8_t* data, uint64_t length); void unshareOneSlow(); void unshareChained(); - void coalesceSlow(size_t maxLength=std::numeric_limits::max()); + void makeManagedChained(); + void coalesceSlow(); + void coalesceSlow(size_t maxLength); // newLength must be the entire length of the buffers between this and // end (no truncation) void coalesceAndReallocate( @@ -1074,18 +1282,21 @@ class IOBuf { size_t newLength, IOBuf* end, size_t newTailroom); + void coalesceAndReallocate(size_t newLength, IOBuf* end) { + coalesceAndReallocate(headroom(), newLength, end, end->prev_->tailroom()); + } void decrementRefcount(); - void reserveSlow(uint32_t minHeadroom, uint32_t minTailroom); + void reserveSlow(uint64_t minHeadroom, uint64_t minTailroom); void freeExtBuffer(); - static size_t goodExtBufferSize(uint32_t minCapacity); + static size_t goodExtBufferSize(uint64_t minCapacity); static void initExtBuffer(uint8_t* buf, size_t mallocSize, SharedInfo** infoReturn, - uint32_t* capacityReturn); - static void allocExtBuffer(uint32_t minCapacity, + uint64_t* capacityReturn); + static void allocExtBuffer(uint64_t minCapacity, uint8_t** bufReturn, SharedInfo** infoReturn, - uint32_t* capacityReturn); + uint64_t* capacityReturn); static void releaseStorage(HeapStorage* storage, uint16_t freeFlags); static void freeInternalBuf(void* buf, void* userData); @@ -1097,11 +1308,11 @@ class IOBuf { * Links to the next and the previous IOBuf in this chain. * * The chain is circularly linked (the last element in the chain points back - * at the head), and next_ and prev_ can never be NULL. If this IOBuf is the + * at the head), and next_ and prev_ can never be null. If this IOBuf is the * only element in the chain, next_ and prev_ will both point to this. */ - IOBuf* next_; - IOBuf* prev_; + IOBuf* next_{this}; + IOBuf* prev_{this}; /* * A pointer to the start of the data referenced by this IOBuf, and the @@ -1109,15 +1320,50 @@ class IOBuf { * * This may refer to any subsection of the actual buffer capacity. */ - uint8_t* data_; - uint8_t* buf_; - uint32_t length_; - uint32_t capacity_; - mutable uint32_t flags_; - uint32_t type_; - // SharedInfo may be NULL if kFlagUserOwned is set. It is non-NULL - // in all other cases. - SharedInfo* sharedInfo_; + uint8_t* data_{nullptr}; + uint8_t* buf_{nullptr}; + uint64_t length_{0}; + uint64_t capacity_{0}; + + // Pack flags in least significant 2 bits, sharedInfo in the rest + mutable uintptr_t flagsAndSharedInfo_{0}; + + static inline uintptr_t packFlagsAndSharedInfo(uintptr_t flags, + SharedInfo* info) { + uintptr_t uinfo = reinterpret_cast(info); + DCHECK_EQ(flags & ~kFlagMask, 0); + DCHECK_EQ(uinfo & kFlagMask, 0); + return flags | uinfo; + } + + inline SharedInfo* sharedInfo() const { + return reinterpret_cast(flagsAndSharedInfo_ & ~kFlagMask); + } + + inline void setSharedInfo(SharedInfo* info) { + uintptr_t uinfo = reinterpret_cast(info); + DCHECK_EQ(uinfo & kFlagMask, 0); + flagsAndSharedInfo_ = (flagsAndSharedInfo_ & kFlagMask) | uinfo; + } + + inline uintptr_t flags() const { + return flagsAndSharedInfo_ & kFlagMask; + } + + // flags_ are changed from const methods + inline void setFlags(uintptr_t flags) const { + DCHECK_EQ(flags & ~kFlagMask, 0); + flagsAndSharedInfo_ |= flags; + } + + inline void clearFlags(uintptr_t flags) const { + DCHECK_EQ(flags & ~kFlagMask, 0); + flagsAndSharedInfo_ &= ~flags; + } + + inline void setFlagsAndSharedInfo(uintptr_t flags, SharedInfo* info) { + flagsAndSharedInfo_ = packFlagsAndSharedInfo(flags, info); + } struct DeleterBase { virtual ~DeleterBase() { } @@ -1148,12 +1394,38 @@ class IOBuf { } }; +/** + * Hasher for IOBuf objects. Hashes the entire chain using SpookyHashV2. + */ +struct IOBufHash { + size_t operator()(const IOBuf& buf) const; + size_t operator()(const std::unique_ptr& buf) const { + return buf ? (*this)(*buf) : 0; + } +}; + +/** + * Equality predicate for IOBuf objects. Compares data in the entire chain. + */ +struct IOBufEqual { + bool operator()(const IOBuf& a, const IOBuf& b) const; + bool operator()(const std::unique_ptr& a, + const std::unique_ptr& b) const { + if (!a && !b) { + return true; + } else if (!a || !b) { + return false; + } else { + return (*this)(*a, *b); + } + } +}; + template typename std::enable_if::value, std::unique_ptr>::type IOBuf::takeOwnership(UniquePtr&& buf, size_t count) { size_t size = count * sizeof(typename UniquePtr::element_type); - DCHECK_LT(size, size_t(std::numeric_limits::max())); auto deleter = new UniquePtrDeleter(buf.get_deleter()); return takeOwnership(buf.release(), size, @@ -1162,9 +1434,9 @@ IOBuf::takeOwnership(UniquePtr&& buf, size_t count) { } inline std::unique_ptr IOBuf::copyBuffer( - const void* data, uint32_t size, uint32_t headroom, - uint32_t minTailroom) { - uint32_t capacity = headroom + size + minTailroom; + const void* data, uint64_t size, uint64_t headroom, + uint64_t minTailroom) { + uint64_t capacity = headroom + size + minTailroom; std::unique_ptr buf = create(capacity); buf->advance(headroom); memcpy(buf->writableData(), data, size); @@ -1173,14 +1445,14 @@ inline std::unique_ptr IOBuf::copyBuffer( } inline std::unique_ptr IOBuf::copyBuffer(const std::string& buf, - uint32_t headroom, - uint32_t minTailroom) { + uint64_t headroom, + uint64_t minTailroom) { return copyBuffer(buf.data(), buf.size(), headroom, minTailroom); } inline std::unique_ptr IOBuf::maybeCopyBuffer(const std::string& buf, - uint32_t headroom, - uint32_t minTailroom) { + uint64_t headroom, + uint64_t minTailroom) { if (buf.empty()) { return nullptr; } @@ -1249,5 +1521,3 @@ inline IOBuf::Iterator IOBuf::end() const { return cend(); } } // folly #pragma GCC diagnostic pop - -#endif // FOLLY_IO_IOBUF_H_