2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * RecordIO: self-synchronizing stream of variable length records
20 * RecordIO gives you the ability to write a stream of variable length records
21 * and read them later even in the face of data corruption -- randomly inserted
22 * or deleted chunks of the file, or modified data. When reading, you may lose
23 * corrupted records, but the stream will resynchronize automatically.
25 #ifndef FOLLY_IO_RECORDIO_H_
26 #define FOLLY_IO_RECORDIO_H_
32 #include <folly/File.h>
33 #include <folly/Range.h>
34 #include <folly/MemoryMapping.h>
35 #include <folly/io/IOBuf.h>
40 * Class to write a stream of RecordIO records to a file.
42 * RecordIOWriter is thread-safe
44 class RecordIOWriter {
47 * Create a RecordIOWriter around a file; will append to the end of
50 * Each file must have a non-zero file id, which is embedded in all
51 * record headers. Readers will only return records with the requested
52 * file id (or, if the reader is created with fileId=0 in the constructor,
53 * the reader will return all records). File ids are only used to allow
54 * resynchronization if you store RecordIO records (with headers) inside
55 * other RecordIO records (for example, if a record consists of a fragment
56 * from another RecordIO file). If you're not planning to do that,
57 * the defaults are fine.
59 explicit RecordIOWriter(File file, uint32_t fileId = 1);
62 * Write a record. We will use at most headerSize() bytes of headroom,
63 * you might want to arrange that before copying your data into it.
65 void write(std::unique_ptr<IOBuf> buf);
68 * Return the position in the file where the next byte will be written.
69 * Conservative, as stuff can be written at any time from another thread.
71 off_t filePos() const { return filePos_; }
76 std::unique_lock<File> writeLock_;
77 std::atomic<off_t> filePos_;
81 * Class to read from a RecordIO file. Will skip invalid records.
83 class RecordIOReader {
88 * RecordIOReader is iterable, returning pairs of ByteRange (record content)
89 * and position in file where the record (including header) begins.
90 * Note that the position includes the header, that is, it can be passed back
93 typedef Iterator iterator;
94 typedef Iterator const_iterator;
95 typedef std::pair<ByteRange, off_t> value_type;
96 typedef value_type& reference;
97 typedef const value_type& const_reference;
100 * A record reader with a fileId of 0 will return all records.
101 * A record reader with a non-zero fileId will only return records where
102 * the fileId matches.
104 explicit RecordIOReader(File file, uint32_t fileId = 0);
106 Iterator cbegin() const;
107 Iterator begin() const;
108 Iterator cend() const;
109 Iterator end() const;
112 * Create an iterator to the first valid record after pos.
114 Iterator seek(off_t pos) const;
121 namespace recordio_helpers {
123 // We're exposing the guts of the RecordIO implementation for two reasons:
124 // 1. It makes unit testing easier, and
125 // 2. It allows you to build different RecordIO readers / writers that use
126 // different storage systems underneath (not standard files)
131 constexpr size_t headerSize(); // defined in RecordIO-inl.h
134 * Write a header in the buffer. We will prepend the header to the front
135 * of the chain. Do not write the buffer if empty (we don't allow empty
136 * records). Returns the total length, including header (0 if empty)
137 * (same as buf->computeChainDataLength(), but likely faster)
139 * The fileId should be unique per stream and allows you to have RecordIO
140 * headers stored inside the data (for example, have an entire RecordIO
141 * file stored as a record inside another RecordIO file). The fileId may
144 size_t prependHeader(std::unique_ptr<IOBuf>& buf, uint32_t fileId = 1);
147 * Search for the first valid record that begins in searchRange (which must be
148 * a subrange of wholeRange). Returns the record data (not the header) if
149 * found, ByteRange() otherwise.
151 * The fileId may be 0, in which case we'll return the first valid record for
152 * *any* fileId, or non-zero, in which case we'll only look for records with
153 * the requested fileId.
159 RecordInfo findRecord(ByteRange searchRange,
160 ByteRange wholeRange,
164 * Search for the first valid record in range.
166 RecordInfo findRecord(ByteRange range, uint32_t fileId);
169 * Check if there is a valid record at the beginning of range. Returns the
170 * record data (not the header) if the record is valid, ByteRange() otherwise.
172 RecordInfo validateRecord(ByteRange range, uint32_t fileId);
174 } // namespace recordio_helpers
178 #include <folly/io/RecordIO-inl.h>
180 #endif /* FOLLY_IO_RECORDIO_H_ */