From: Rafael Espindola Date: Wed, 12 Nov 2014 03:55:46 +0000 (+0000) Subject: Merge StreamableMemoryObject into MemoryObject. X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=d0518569ecfe74ad7d90b26cee2d8ebebc1bdb93;p=oota-llvm.git Merge StreamableMemoryObject into MemoryObject. Every MemoryObject is a StreamableMemoryObject since the removal of StringRefMemoryObject, so just merge the two. I will clean up the MemoryObject interface in the upcoming commits. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@221766 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/Bitcode/BitstreamReader.h b/include/llvm/Bitcode/BitstreamReader.h index b42f559e13b..6965da36ee2 100644 --- a/include/llvm/Bitcode/BitstreamReader.h +++ b/include/llvm/Bitcode/BitstreamReader.h @@ -17,7 +17,7 @@ #include "llvm/Bitcode/BitCodes.h" #include "llvm/Support/Endian.h" -#include "llvm/Support/StreamableMemoryObject.h" +#include "llvm/Support/StreamingMemoryObject.h" #include #include #include @@ -42,7 +42,7 @@ public: std::vector > RecordNames; }; private: - std::unique_ptr BitcodeBytes; + std::unique_ptr BitcodeBytes; std::vector BlockInfoRecords; @@ -61,7 +61,7 @@ public: init(Start, End); } - BitstreamReader(StreamableMemoryObject *bytes) : IgnoreBlockInfoNames(true) { + BitstreamReader(MemoryObject *bytes) : IgnoreBlockInfoNames(true) { BitcodeBytes.reset(bytes); } @@ -82,7 +82,7 @@ public: BitcodeBytes.reset(getNonStreamedMemoryObject(Start, End)); } - StreamableMemoryObject &getBitcodeBytes() { return *BitcodeBytes; } + MemoryObject &getBitcodeBytes() { return *BitcodeBytes; } /// This is called by clients that want block/record name information. void CollectBlockInfoNames() { IgnoreBlockInfoNames = false; } diff --git a/include/llvm/Support/MemoryObject.h b/include/llvm/Support/MemoryObject.h index 6208772532c..584a2c55c04 100644 --- a/include/llvm/Support/MemoryObject.h +++ b/include/llvm/Support/MemoryObject.h @@ -14,10 +14,17 @@ namespace llvm { -/// Abstract base class for contiguous addressable memory. Necessary for cases -/// in which the memory is in another process, in a file, or on a remote -/// machine. All size and offset parameters are uint64_ts, to allow 32-bit -/// processes access to 64-bit address spaces. +/// Interface to data which might be streamed. Streamability has 2 important +/// implications/restrictions. First, the data might not yet exist in memory +/// when the request is made. This just means that readByte/readBytes might have +/// to block or do some work to get it. More significantly, the exact size of +/// the object might not be known until it has all been fetched. This means that +/// to return the right result, getExtent must also wait for all the data to +/// arrive; therefore it should not be called on objects which are actually +/// streamed (this would defeat the purpose of streaming). Instead, +/// isValidAddress and isObjectEnd can be used to test addresses without knowing +/// the exact size of the stream. Finally, getPointer can be used instead of +/// readBytes to avoid extra copying. class MemoryObject { public: virtual ~MemoryObject(); @@ -41,6 +48,28 @@ public: /// bounds violation or an implementation-specific error. virtual int readBytes(uint64_t address, uint64_t size, uint8_t *buf) const = 0; + + /// Ensures that the requested data is in memory, and returns a pointer to it. + /// More efficient than using readBytes if the data is already in memory. May + /// block until (address - base + size) bytes have been read + /// @param address - address of the byte, in the same space as getBase() + /// @param size - amount of data that must be available on return + /// @result - valid pointer to the requested data + virtual const uint8_t *getPointer(uint64_t address, uint64_t size) const = 0; + + /// Returns true if the address is within the object (i.e. between base and + /// base + extent - 1 inclusive). May block until (address - base) bytes have + /// been read + /// @param address - address of the byte, in the same space as getBase() + /// @result - true if the address may be read with readByte() + virtual bool isValidAddress(uint64_t address) const = 0; + + /// Returns true if the address is one past the end of the object (i.e. if it + /// is equal to base + extent). May block until (address - base) bytes have + /// been read + /// @param address - address of the byte, in the same space as getBase() + /// @result - true if the address is equal to base + extent + virtual bool isObjectEnd(uint64_t address) const = 0; }; } diff --git a/include/llvm/Support/StreamableMemoryObject.h b/include/llvm/Support/StreamableMemoryObject.h deleted file mode 100644 index 618610d084c..00000000000 --- a/include/llvm/Support/StreamableMemoryObject.h +++ /dev/null @@ -1,135 +0,0 @@ -//===- StreamableMemoryObject.h - Streamable data interface -----*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - - -#ifndef LLVM_SUPPORT_STREAMABLEMEMORYOBJECT_H -#define LLVM_SUPPORT_STREAMABLEMEMORYOBJECT_H - -#include "llvm/Support/Compiler.h" -#include "llvm/Support/DataStream.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MemoryObject.h" -#include -#include -#include - -namespace llvm { - -/// Interface to data which might be streamed. Streamability has 2 important -/// implications/restrictions. First, the data might not yet exist in memory -/// when the request is made. This just means that readByte/readBytes might have -/// to block or do some work to get it. More significantly, the exact size of -/// the object might not be known until it has all been fetched. This means that -/// to return the right result, getExtent must also wait for all the data to -/// arrive; therefore it should not be called on objects which are actually -/// streamed (this would defeat the purpose of streaming). Instead, -/// isValidAddress and isObjectEnd can be used to test addresses without knowing -/// the exact size of the stream. Finally, getPointer can be used instead of -/// readBytes to avoid extra copying. -class StreamableMemoryObject : public MemoryObject { - public: - virtual ~StreamableMemoryObject(); - - /// Ensures that the requested data is in memory, and returns a pointer to it. - /// More efficient than using readBytes if the data is already in memory. May - /// block until (address - base + size) bytes have been read - /// @param address - address of the byte, in the same space as getBase() - /// @param size - amount of data that must be available on return - /// @result - valid pointer to the requested data - virtual const uint8_t *getPointer(uint64_t address, uint64_t size) const = 0; - - /// Returns true if the address is within the object (i.e. between base and - /// base + extent - 1 inclusive). May block until (address - base) bytes have - /// been read - /// @param address - address of the byte, in the same space as getBase() - /// @result - true if the address may be read with readByte() - virtual bool isValidAddress(uint64_t address) const = 0; - - /// Returns true if the address is one past the end of the object (i.e. if it - /// is equal to base + extent). May block until (address - base) bytes have - /// been read - /// @param address - address of the byte, in the same space as getBase() - /// @result - true if the address is equal to base + extent - virtual bool isObjectEnd(uint64_t address) const = 0; -}; - -/// Interface to data which is actually streamed from a DataStreamer. In -/// addition to inherited members, it has the dropLeadingBytes and -/// setKnownObjectSize methods which are not applicable to non-streamed objects. -class StreamingMemoryObject : public StreamableMemoryObject { -public: - StreamingMemoryObject(DataStreamer *streamer); - uint64_t getExtent() const override; - int readBytes(uint64_t address, uint64_t size, - uint8_t *buf) const override; - const uint8_t *getPointer(uint64_t address, uint64_t size) const override { - // This could be fixed by ensuring the bytes are fetched and making a copy, - // requiring that the bitcode size be known, or otherwise ensuring that - // the memory doesn't go away/get reallocated, but it's - // not currently necessary. Users that need the pointer don't stream. - llvm_unreachable("getPointer in streaming memory objects not allowed"); - return nullptr; - } - bool isValidAddress(uint64_t address) const override; - bool isObjectEnd(uint64_t address) const override; - - /// Drop s bytes from the front of the stream, pushing the positions of the - /// remaining bytes down by s. This is used to skip past the bitcode header, - /// since we don't know a priori if it's present, and we can't put bytes - /// back into the stream once we've read them. - bool dropLeadingBytes(size_t s); - - /// If the data object size is known in advance, many of the operations can - /// be made more efficient, so this method should be called before reading - /// starts (although it can be called anytime). - void setKnownObjectSize(size_t size); - -private: - const static uint32_t kChunkSize = 4096 * 4; - mutable std::vector Bytes; - std::unique_ptr Streamer; - mutable size_t BytesRead; // Bytes read from stream - size_t BytesSkipped;// Bytes skipped at start of stream (e.g. wrapper/header) - mutable size_t ObjectSize; // 0 if unknown, set if wrapper seen or EOF reached - mutable bool EOFReached; - - // Fetch enough bytes such that Pos can be read or EOF is reached - // (i.e. BytesRead > Pos). Return true if Pos can be read. - // Unlike most of the functions in BitcodeReader, returns true on success. - // Most of the requests will be small, but we fetch at kChunkSize bytes - // at a time to avoid making too many potentially expensive GetBytes calls - bool fetchToPos(size_t Pos) const { - if (EOFReached) return Pos < ObjectSize; - while (Pos >= BytesRead) { - Bytes.resize(BytesRead + BytesSkipped + kChunkSize); - size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped], - kChunkSize); - BytesRead += bytes; - if (bytes < kChunkSize) { - assert((!ObjectSize || BytesRead >= Pos) && - "Unexpected short read fetching bitcode"); - if (BytesRead <= Pos) { // reached EOF/ran out of bytes - ObjectSize = BytesRead; - EOFReached = true; - return false; - } - } - } - return true; - } - - StreamingMemoryObject(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION; - void operator=(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION; -}; - -StreamableMemoryObject *getNonStreamedMemoryObject( - const unsigned char *Start, const unsigned char *End); - -} -#endif // STREAMABLEMEMORYOBJECT_H_ diff --git a/include/llvm/Support/StreamingMemoryObject.h b/include/llvm/Support/StreamingMemoryObject.h new file mode 100644 index 00000000000..da1b88738d0 --- /dev/null +++ b/include/llvm/Support/StreamingMemoryObject.h @@ -0,0 +1,96 @@ +//===- StreamingMemoryObject.h - Streamable data interface -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_STREAMINGMEMORYOBJECT_H +#define LLVM_SUPPORT_STREAMINGMEMORYOBJECT_H + +#include "llvm/Support/Compiler.h" +#include "llvm/Support/DataStream.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MemoryObject.h" +#include +#include +#include + +namespace llvm { + +/// Interface to data which is actually streamed from a DataStreamer. In +/// addition to inherited members, it has the dropLeadingBytes and +/// setKnownObjectSize methods which are not applicable to non-streamed objects. +class StreamingMemoryObject : public MemoryObject { +public: + StreamingMemoryObject(DataStreamer *streamer); + uint64_t getExtent() const override; + int readBytes(uint64_t address, uint64_t size, + uint8_t *buf) const override; + const uint8_t *getPointer(uint64_t address, uint64_t size) const override { + // This could be fixed by ensuring the bytes are fetched and making a copy, + // requiring that the bitcode size be known, or otherwise ensuring that + // the memory doesn't go away/get reallocated, but it's + // not currently necessary. Users that need the pointer don't stream. + llvm_unreachable("getPointer in streaming memory objects not allowed"); + return nullptr; + } + bool isValidAddress(uint64_t address) const override; + bool isObjectEnd(uint64_t address) const override; + + /// Drop s bytes from the front of the stream, pushing the positions of the + /// remaining bytes down by s. This is used to skip past the bitcode header, + /// since we don't know a priori if it's present, and we can't put bytes + /// back into the stream once we've read them. + bool dropLeadingBytes(size_t s); + + /// If the data object size is known in advance, many of the operations can + /// be made more efficient, so this method should be called before reading + /// starts (although it can be called anytime). + void setKnownObjectSize(size_t size); + +private: + const static uint32_t kChunkSize = 4096 * 4; + mutable std::vector Bytes; + std::unique_ptr Streamer; + mutable size_t BytesRead; // Bytes read from stream + size_t BytesSkipped;// Bytes skipped at start of stream (e.g. wrapper/header) + mutable size_t ObjectSize; // 0 if unknown, set if wrapper seen or EOF reached + mutable bool EOFReached; + + // Fetch enough bytes such that Pos can be read or EOF is reached + // (i.e. BytesRead > Pos). Return true if Pos can be read. + // Unlike most of the functions in BitcodeReader, returns true on success. + // Most of the requests will be small, but we fetch at kChunkSize bytes + // at a time to avoid making too many potentially expensive GetBytes calls + bool fetchToPos(size_t Pos) const { + if (EOFReached) return Pos < ObjectSize; + while (Pos >= BytesRead) { + Bytes.resize(BytesRead + BytesSkipped + kChunkSize); + size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped], + kChunkSize); + BytesRead += bytes; + if (bytes < kChunkSize) { + assert((!ObjectSize || BytesRead >= Pos) && + "Unexpected short read fetching bitcode"); + if (BytesRead <= Pos) { // reached EOF/ran out of bytes + ObjectSize = BytesRead; + EOFReached = true; + return false; + } + } + } + return true; + } + + StreamingMemoryObject(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION; + void operator=(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION; +}; + +MemoryObject *getNonStreamedMemoryObject( + const unsigned char *Start, const unsigned char *End); + +} +#endif // STREAMINGMEMORYOBJECT_H_ diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt index 794372d172c..fa62591191d 100644 --- a/lib/Support/CMakeLists.txt +++ b/lib/Support/CMakeLists.txt @@ -80,7 +80,7 @@ add_llvm_library(LLVMSupport SourceMgr.cpp SpecialCaseList.cpp Statistic.cpp - StreamableMemoryObject.cpp + StreamingMemoryObject.cpp StringExtras.cpp StringMap.cpp StringPool.cpp diff --git a/lib/Support/DataStream.cpp b/lib/Support/DataStream.cpp index 32653de5194..dbf6465189d 100644 --- a/lib/Support/DataStream.cpp +++ b/lib/Support/DataStream.cpp @@ -32,12 +32,12 @@ using namespace llvm; #define DEBUG_TYPE "Data-stream" // Interface goals: -// * StreamableMemoryObject doesn't care about complexities like using +// * StreamingMemoryObject doesn't care about complexities like using // threads/async callbacks to actually overlap download+compile // * Don't want to duplicate Data in memory // * Don't need to know total Data len in advance // Non-goals: -// StreamableMemoryObject already has random access so this interface only does +// StreamingMemoryObject already has random access so this interface only does // in-order streaming (no arbitrary seeking, else we'd have to buffer all the // Data here in addition to MemoryObject). This also means that if we want // to be able to to free Data, BitstreamBytes/BitcodeReader will implement it diff --git a/lib/Support/StreamableMemoryObject.cpp b/lib/Support/StreamableMemoryObject.cpp deleted file mode 100644 index 1a11f5b0cde..00000000000 --- a/lib/Support/StreamableMemoryObject.cpp +++ /dev/null @@ -1,126 +0,0 @@ -//===- StreamableMemoryObject.cpp - Streamable data interface -------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Support/StreamableMemoryObject.h" -#include "llvm/Support/Compiler.h" -#include -#include -#include - - -using namespace llvm; - -namespace { - -class RawMemoryObject : public StreamableMemoryObject { -public: - RawMemoryObject(const unsigned char *Start, const unsigned char *End) : - FirstChar(Start), LastChar(End) { - assert(LastChar >= FirstChar && "Invalid start/end range"); - } - - uint64_t getExtent() const override { - return LastChar - FirstChar; - } - int readBytes(uint64_t address, uint64_t size, - uint8_t *buf) const override; - const uint8_t *getPointer(uint64_t address, uint64_t size) const override; - bool isValidAddress(uint64_t address) const override { - return validAddress(address); - } - bool isObjectEnd(uint64_t address) const override { - return objectEnd(address); - } - -private: - const uint8_t* const FirstChar; - const uint8_t* const LastChar; - - // These are implemented as inline functions here to avoid multiple virtual - // calls per public function - bool validAddress(uint64_t address) const { - return static_cast(address) < LastChar - FirstChar; - } - bool objectEnd(uint64_t address) const { - return static_cast(address) == LastChar - FirstChar; - } - - RawMemoryObject(const RawMemoryObject&) LLVM_DELETED_FUNCTION; - void operator=(const RawMemoryObject&) LLVM_DELETED_FUNCTION; -}; - -int RawMemoryObject::readBytes(uint64_t address, - uint64_t size, - uint8_t *buf) const { - if (!validAddress(address) || !validAddress(address + size - 1)) return -1; - memcpy(buf, (uint8_t *)(uintptr_t)(address + FirstChar), size); - return size; -} - -const uint8_t *RawMemoryObject::getPointer(uint64_t address, - uint64_t size) const { - return FirstChar + address; -} -} // anonymous namespace - -namespace llvm { -// If the bitcode has a header, then its size is known, and we don't have to -// block until we actually want to read it. -bool StreamingMemoryObject::isValidAddress(uint64_t address) const { - if (ObjectSize && address < ObjectSize) return true; - return fetchToPos(address); -} - -bool StreamingMemoryObject::isObjectEnd(uint64_t address) const { - if (ObjectSize) return address == ObjectSize; - fetchToPos(address); - return address == ObjectSize && address != 0; -} - -uint64_t StreamingMemoryObject::getExtent() const { - if (ObjectSize) return ObjectSize; - size_t pos = BytesRead + kChunkSize; - // keep fetching until we run out of bytes - while (fetchToPos(pos)) pos += kChunkSize; - return ObjectSize; -} - -int StreamingMemoryObject::readBytes(uint64_t address, - uint64_t size, - uint8_t *buf) const { - if (!fetchToPos(address + size - 1)) return -1; - memcpy(buf, &Bytes[address + BytesSkipped], size); - return 0; -} - -bool StreamingMemoryObject::dropLeadingBytes(size_t s) { - if (BytesRead < s) return true; - BytesSkipped = s; - BytesRead -= s; - return false; -} - -void StreamingMemoryObject::setKnownObjectSize(size_t size) { - ObjectSize = size; - Bytes.reserve(size); -} - -StreamableMemoryObject *getNonStreamedMemoryObject( - const unsigned char *Start, const unsigned char *End) { - return new RawMemoryObject(Start, End); -} - -StreamableMemoryObject::~StreamableMemoryObject() { } - -StreamingMemoryObject::StreamingMemoryObject(DataStreamer *streamer) : - Bytes(kChunkSize), Streamer(streamer), BytesRead(0), BytesSkipped(0), - ObjectSize(0), EOFReached(false) { - BytesRead = streamer->GetBytes(&Bytes[0], kChunkSize); -} -} diff --git a/lib/Support/StreamingMemoryObject.cpp b/lib/Support/StreamingMemoryObject.cpp new file mode 100644 index 00000000000..7187ce013fa --- /dev/null +++ b/lib/Support/StreamingMemoryObject.cpp @@ -0,0 +1,124 @@ +//===- StreamingMemoryObject.cpp - Streamable data interface -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/StreamingMemoryObject.h" +#include "llvm/Support/Compiler.h" +#include +#include +#include + + +using namespace llvm; + +namespace { + +class RawMemoryObject : public MemoryObject { +public: + RawMemoryObject(const unsigned char *Start, const unsigned char *End) : + FirstChar(Start), LastChar(End) { + assert(LastChar >= FirstChar && "Invalid start/end range"); + } + + uint64_t getExtent() const override { + return LastChar - FirstChar; + } + int readBytes(uint64_t address, uint64_t size, + uint8_t *buf) const override; + const uint8_t *getPointer(uint64_t address, uint64_t size) const override; + bool isValidAddress(uint64_t address) const override { + return validAddress(address); + } + bool isObjectEnd(uint64_t address) const override { + return objectEnd(address); + } + +private: + const uint8_t* const FirstChar; + const uint8_t* const LastChar; + + // These are implemented as inline functions here to avoid multiple virtual + // calls per public function + bool validAddress(uint64_t address) const { + return static_cast(address) < LastChar - FirstChar; + } + bool objectEnd(uint64_t address) const { + return static_cast(address) == LastChar - FirstChar; + } + + RawMemoryObject(const RawMemoryObject&) LLVM_DELETED_FUNCTION; + void operator=(const RawMemoryObject&) LLVM_DELETED_FUNCTION; +}; + +int RawMemoryObject::readBytes(uint64_t address, + uint64_t size, + uint8_t *buf) const { + if (!validAddress(address) || !validAddress(address + size - 1)) return -1; + memcpy(buf, (uint8_t *)(uintptr_t)(address + FirstChar), size); + return size; +} + +const uint8_t *RawMemoryObject::getPointer(uint64_t address, + uint64_t size) const { + return FirstChar + address; +} +} // anonymous namespace + +namespace llvm { +// If the bitcode has a header, then its size is known, and we don't have to +// block until we actually want to read it. +bool StreamingMemoryObject::isValidAddress(uint64_t address) const { + if (ObjectSize && address < ObjectSize) return true; + return fetchToPos(address); +} + +bool StreamingMemoryObject::isObjectEnd(uint64_t address) const { + if (ObjectSize) return address == ObjectSize; + fetchToPos(address); + return address == ObjectSize && address != 0; +} + +uint64_t StreamingMemoryObject::getExtent() const { + if (ObjectSize) return ObjectSize; + size_t pos = BytesRead + kChunkSize; + // keep fetching until we run out of bytes + while (fetchToPos(pos)) pos += kChunkSize; + return ObjectSize; +} + +int StreamingMemoryObject::readBytes(uint64_t address, + uint64_t size, + uint8_t *buf) const { + if (!fetchToPos(address + size - 1)) return -1; + memcpy(buf, &Bytes[address + BytesSkipped], size); + return 0; +} + +bool StreamingMemoryObject::dropLeadingBytes(size_t s) { + if (BytesRead < s) return true; + BytesSkipped = s; + BytesRead -= s; + return false; +} + +void StreamingMemoryObject::setKnownObjectSize(size_t size) { + ObjectSize = size; + Bytes.reserve(size); +} + +MemoryObject *getNonStreamedMemoryObject(const unsigned char *Start, + const unsigned char *End) { + return new RawMemoryObject(Start, End); +} + +StreamingMemoryObject::StreamingMemoryObject(DataStreamer *streamer) : + Bytes(kChunkSize), Streamer(streamer), BytesRead(0), BytesSkipped(0), + ObjectSize(0), EOFReached(false) { + BytesRead = streamer->GetBytes(&Bytes[0], kChunkSize); +} +}