cannam@62: // Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors cannam@62: // Licensed under the MIT License: cannam@62: // cannam@62: // Permission is hereby granted, free of charge, to any person obtaining a copy cannam@62: // of this software and associated documentation files (the "Software"), to deal cannam@62: // in the Software without restriction, including without limitation the rights cannam@62: // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cannam@62: // copies of the Software, and to permit persons to whom the Software is cannam@62: // furnished to do so, subject to the following conditions: cannam@62: // cannam@62: // The above copyright notice and this permission notice shall be included in cannam@62: // all copies or substantial portions of the Software. cannam@62: // cannam@62: // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR cannam@62: // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, cannam@62: // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE cannam@62: // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER cannam@62: // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, cannam@62: // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN cannam@62: // THE SOFTWARE. cannam@62: cannam@62: // This file implements a simple serialization format for Cap'n Proto messages. The format cannam@62: // is as follows: cannam@62: // cannam@62: // * 32-bit little-endian segment count (4 bytes). cannam@62: // * 32-bit little-endian size of each segment (4*(segment count) bytes). cannam@62: // * Padding so that subsequent data is 64-bit-aligned (0 or 4 bytes). (I.e., if there are an even cannam@62: // number of segments, there are 4 bytes of zeros here, otherwise there is no padding.) cannam@62: // * Data from each segment, in order (8*sum(segment sizes) bytes) cannam@62: // cannam@62: // This format has some important properties: cannam@62: // - It is self-delimiting, so multiple messages may be written to a stream without any external cannam@62: // delimiter. cannam@62: // - The total size and position of each segment can be determined by reading only the first part cannam@62: // of the message, allowing lazy and random-access reading of the segment data. cannam@62: // - A message is always at least 8 bytes. cannam@62: // - A single-segment message can be read entirely in two system calls with no buffering. cannam@62: // - A multi-segment message can be read entirely in three system calls with no buffering. cannam@62: // - The format is appropriate for mmap()ing since all data is aligned. cannam@62: cannam@62: #ifndef CAPNP_SERIALIZE_H_ cannam@62: #define CAPNP_SERIALIZE_H_ cannam@62: cannam@62: #if defined(__GNUC__) && !defined(CAPNP_HEADER_WARNINGS) cannam@62: #pragma GCC system_header cannam@62: #endif cannam@62: cannam@62: #include "message.h" cannam@62: #include cannam@62: cannam@62: namespace capnp { cannam@62: cannam@62: class FlatArrayMessageReader: public MessageReader { cannam@62: // Parses a message from a flat array. Note that it makes sense to use this together with mmap() cannam@62: // for extremely fast parsing. cannam@62: cannam@62: public: cannam@62: FlatArrayMessageReader(kj::ArrayPtr array, ReaderOptions options = ReaderOptions()); cannam@62: // The array must remain valid until the MessageReader is destroyed. cannam@62: cannam@62: kj::ArrayPtr getSegment(uint id) override; cannam@62: cannam@62: const word* getEnd() const { return end; } cannam@62: // Get a pointer just past the end of the message as determined by reading the message header. cannam@62: // This could actually be before the end of the input array. This pointer is useful e.g. if cannam@62: // you know that the input array has extra stuff appended after the message and you want to cannam@62: // get at it. cannam@62: cannam@62: private: cannam@62: // Optimize for single-segment case. cannam@62: kj::ArrayPtr segment0; cannam@62: kj::Array> moreSegments; cannam@62: const word* end; cannam@62: }; cannam@62: cannam@62: kj::ArrayPtr initMessageBuilderFromFlatArrayCopy( cannam@62: kj::ArrayPtr array, MessageBuilder& target, cannam@62: ReaderOptions options = ReaderOptions()); cannam@62: // Convenience function which reads a message using `FlatArrayMessageReader` then copies the cannam@62: // content into the target `MessageBuilder`, verifying that the message structure is valid cannam@62: // (although not necessarily that it matches the desired schema). cannam@62: // cannam@62: // Returns an ArrayPtr containing any words left over in the array after consuming the whole cannam@62: // message. This is useful when reading multiple messages that have been concatenated. See also cannam@62: // FlatArrayMessageReader::getEnd(). cannam@62: // cannam@62: // (Note that it's also possible to initialize a `MessageBuilder` directly without a copy using one cannam@62: // of `MessageBuilder`'s constructors. However, this approach skips the validation step and is not cannam@62: // safe to use on untrusted input. Therefore, we do not provide a convenience method for it.) cannam@62: cannam@62: kj::Array messageToFlatArray(MessageBuilder& builder); cannam@62: // Constructs a flat array containing the entire content of the given message. cannam@62: // cannam@62: // To output the message as bytes, use `.asBytes()` on the returned word array. Keep in mind that cannam@62: // `asBytes()` returns an ArrayPtr, so you have to save the Array as well to prevent it from being cannam@62: // deleted. For example: cannam@62: // cannam@62: // kj::Array words = messageToFlatArray(myMessage); cannam@62: // kj::ArrayPtr bytes = words.asBytes(); cannam@62: // write(fd, bytes.begin(), bytes.size()); cannam@62: cannam@62: kj::Array messageToFlatArray(kj::ArrayPtr> segments); cannam@62: // Version of messageToFlatArray that takes a raw segment array. cannam@62: cannam@62: size_t computeSerializedSizeInWords(MessageBuilder& builder); cannam@62: // Returns the size, in words, that will be needed to serialize the message, including the header. cannam@62: cannam@62: size_t computeSerializedSizeInWords(kj::ArrayPtr> segments); cannam@62: // Version of computeSerializedSizeInWords that takes a raw segment array. cannam@62: cannam@62: size_t expectedSizeInWordsFromPrefix(kj::ArrayPtr messagePrefix); cannam@62: // Given a prefix of a serialized message, try to determine the expected total size of the message, cannam@62: // in words. The returned size is based on the information known so far; it may be an underestimate cannam@62: // if the prefix doesn't contain the full segment table. cannam@62: // cannam@62: // If the returned value is greater than `messagePrefix.size()`, then the message is not yet cannam@62: // complete and the app cannot parse it yet. If the returned value is less than or equal to cannam@62: // `messagePrefix.size()`, then the returned value is the exact total size of the message; any cannam@62: // remaining bytes are part of the next message. cannam@62: // cannam@62: // This function is useful when reading messages from a stream in an asynchronous way, but when cannam@62: // using the full KJ async infrastructure would be too difficult. Each time bytes are received, cannam@62: // use this function to determine if an entire message is ready to be parsed. cannam@62: cannam@62: // ======================================================================================= cannam@62: cannam@62: class InputStreamMessageReader: public MessageReader { cannam@62: // A MessageReader that reads from an abstract kj::InputStream. See also StreamFdMessageReader cannam@62: // for a subclass specific to file descriptors. cannam@62: cannam@62: public: cannam@62: InputStreamMessageReader(kj::InputStream& inputStream, cannam@62: ReaderOptions options = ReaderOptions(), cannam@62: kj::ArrayPtr scratchSpace = nullptr); cannam@62: ~InputStreamMessageReader() noexcept(false); cannam@62: cannam@62: // implements MessageReader ---------------------------------------- cannam@62: kj::ArrayPtr getSegment(uint id) override; cannam@62: cannam@62: private: cannam@62: kj::InputStream& inputStream; cannam@62: byte* readPos; cannam@62: cannam@62: // Optimize for single-segment case. cannam@62: kj::ArrayPtr segment0; cannam@62: kj::Array> moreSegments; cannam@62: cannam@62: kj::Array ownedSpace; cannam@62: // Only if scratchSpace wasn't big enough. cannam@62: cannam@62: kj::UnwindDetector unwindDetector; cannam@62: }; cannam@62: cannam@62: void readMessageCopy(kj::InputStream& input, MessageBuilder& target, cannam@62: ReaderOptions options = ReaderOptions(), cannam@62: kj::ArrayPtr scratchSpace = nullptr); cannam@62: // Convenience function which reads a message using `InputStreamMessageReader` then copies the cannam@62: // content into the target `MessageBuilder`, verifying that the message structure is valid cannam@62: // (although not necessarily that it matches the desired schema). cannam@62: // cannam@62: // (Note that it's also possible to initialize a `MessageBuilder` directly without a copy using one cannam@62: // of `MessageBuilder`'s constructors. However, this approach skips the validation step and is not cannam@62: // safe to use on untrusted input. Therefore, we do not provide a convenience method for it.) cannam@62: cannam@62: void writeMessage(kj::OutputStream& output, MessageBuilder& builder); cannam@62: // Write the message to the given output stream. cannam@62: cannam@62: void writeMessage(kj::OutputStream& output, kj::ArrayPtr> segments); cannam@62: // Write the segment array to the given output stream. cannam@62: cannam@62: // ======================================================================================= cannam@62: // Specializations for reading from / writing to file descriptors. cannam@62: cannam@62: class StreamFdMessageReader: private kj::FdInputStream, public InputStreamMessageReader { cannam@62: // A MessageReader that reads from a steam-based file descriptor. cannam@62: cannam@62: public: cannam@62: StreamFdMessageReader(int fd, ReaderOptions options = ReaderOptions(), cannam@62: kj::ArrayPtr scratchSpace = nullptr) cannam@62: : FdInputStream(fd), InputStreamMessageReader(*this, options, scratchSpace) {} cannam@62: // Read message from a file descriptor, without taking ownership of the descriptor. cannam@62: cannam@62: StreamFdMessageReader(kj::AutoCloseFd fd, ReaderOptions options = ReaderOptions(), cannam@62: kj::ArrayPtr scratchSpace = nullptr) cannam@62: : FdInputStream(kj::mv(fd)), InputStreamMessageReader(*this, options, scratchSpace) {} cannam@62: // Read a message from a file descriptor, taking ownership of the descriptor. cannam@62: cannam@62: ~StreamFdMessageReader() noexcept(false); cannam@62: }; cannam@62: cannam@62: void readMessageCopyFromFd(int fd, MessageBuilder& target, cannam@62: ReaderOptions options = ReaderOptions(), cannam@62: kj::ArrayPtr scratchSpace = nullptr); cannam@62: // Convenience function which reads a message using `StreamFdMessageReader` then copies the cannam@62: // content into the target `MessageBuilder`, verifying that the message structure is valid cannam@62: // (although not necessarily that it matches the desired schema). cannam@62: // cannam@62: // (Note that it's also possible to initialize a `MessageBuilder` directly without a copy using one cannam@62: // of `MessageBuilder`'s constructors. However, this approach skips the validation step and is not cannam@62: // safe to use on untrusted input. Therefore, we do not provide a convenience method for it.) cannam@62: cannam@62: void writeMessageToFd(int fd, MessageBuilder& builder); cannam@62: // Write the message to the given file descriptor. cannam@62: // cannam@62: // This function throws an exception on any I/O error. If your code is not exception-safe, be sure cannam@62: // you catch this exception at the call site. If throwing an exception is not acceptable, you cannam@62: // can implement your own OutputStream with arbitrary error handling and then use writeMessage(). cannam@62: cannam@62: void writeMessageToFd(int fd, kj::ArrayPtr> segments); cannam@62: // Write the segment array to the given file descriptor. cannam@62: // cannam@62: // This function throws an exception on any I/O error. If your code is not exception-safe, be sure cannam@62: // you catch this exception at the call site. If throwing an exception is not acceptable, you cannam@62: // can implement your own OutputStream with arbitrary error handling and then use writeMessage(). cannam@62: cannam@62: // ======================================================================================= cannam@62: // inline stuff cannam@62: cannam@62: inline kj::Array messageToFlatArray(MessageBuilder& builder) { cannam@62: return messageToFlatArray(builder.getSegmentsForOutput()); cannam@62: } cannam@62: cannam@62: inline size_t computeSerializedSizeInWords(MessageBuilder& builder) { cannam@62: return computeSerializedSizeInWords(builder.getSegmentsForOutput()); cannam@62: } cannam@62: cannam@62: inline void writeMessage(kj::OutputStream& output, MessageBuilder& builder) { cannam@62: writeMessage(output, builder.getSegmentsForOutput()); cannam@62: } cannam@62: cannam@62: inline void writeMessageToFd(int fd, MessageBuilder& builder) { cannam@62: writeMessageToFd(fd, builder.getSegmentsForOutput()); cannam@62: } cannam@62: cannam@62: } // namespace capnp cannam@62: cannam@62: #endif // SERIALIZE_H_