1 //===--- YAMLParser.cpp - Simple YAML parser ------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements a YAML parser.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Support/YAMLParser.h"
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/ADT/Twine.h"
18 #include "llvm/ADT/ilist.h"
19 #include "llvm/ADT/ilist_node.h"
20 #include "llvm/Support/ErrorHandling.h"
21 #include "llvm/Support/MemoryBuffer.h"
22 #include "llvm/Support/SourceMgr.h"
23 #include "llvm/Support/raw_ostream.h"
28 enum UnicodeEncodingForm {
29 UEF_UTF32_LE, ///< UTF-32 Little Endian
30 UEF_UTF32_BE, ///< UTF-32 Big Endian
31 UEF_UTF16_LE, ///< UTF-16 Little Endian
32 UEF_UTF16_BE, ///< UTF-16 Big Endian
33 UEF_UTF8, ///< UTF-8 or ascii.
34 UEF_Unknown ///< Not a valid Unicode encoding.
37 /// EncodingInfo - Holds the encoding type and length of the byte order mark if
38 /// it exists. Length is in {0, 2, 3, 4}.
39 typedef std::pair<UnicodeEncodingForm, unsigned> EncodingInfo;
41 /// getUnicodeEncoding - Reads up to the first 4 bytes to determine the Unicode
42 /// encoding form of \a Input.
44 /// @param Input A string of length 0 or more.
45 /// @returns An EncodingInfo indicating the Unicode encoding form of the input
46 /// and how long the byte order mark is if one exists.
47 static EncodingInfo getUnicodeEncoding(StringRef Input) {
48 if (Input.size() == 0)
49 return std::make_pair(UEF_Unknown, 0);
51 switch (uint8_t(Input[0])) {
53 if (Input.size() >= 4) {
55 && uint8_t(Input[2]) == 0xFE
56 && uint8_t(Input[3]) == 0xFF)
57 return std::make_pair(UEF_UTF32_BE, 4);
58 if (Input[1] == 0 && Input[2] == 0 && Input[3] != 0)
59 return std::make_pair(UEF_UTF32_BE, 0);
62 if (Input.size() >= 2 && Input[1] != 0)
63 return std::make_pair(UEF_UTF16_BE, 0);
64 return std::make_pair(UEF_Unknown, 0);
66 if ( Input.size() >= 4
67 && uint8_t(Input[1]) == 0xFE
70 return std::make_pair(UEF_UTF32_LE, 4);
72 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFE)
73 return std::make_pair(UEF_UTF16_LE, 2);
74 return std::make_pair(UEF_Unknown, 0);
76 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFF)
77 return std::make_pair(UEF_UTF16_BE, 2);
78 return std::make_pair(UEF_Unknown, 0);
80 if ( Input.size() >= 3
81 && uint8_t(Input[1]) == 0xBB
82 && uint8_t(Input[2]) == 0xBF)
83 return std::make_pair(UEF_UTF8, 3);
84 return std::make_pair(UEF_Unknown, 0);
87 // It could still be utf-32 or utf-16.
88 if (Input.size() >= 4 && Input[1] == 0 && Input[2] == 0 && Input[3] == 0)
89 return std::make_pair(UEF_UTF32_LE, 0);
91 if (Input.size() >= 2 && Input[1] == 0)
92 return std::make_pair(UEF_UTF16_LE, 0);
94 return std::make_pair(UEF_UTF8, 0);
99 /// Token - A single YAML token.
100 struct Token : ilist_node<Token> {
102 TK_Error, // Uninitialized token.
111 TK_BlockSequenceStart,
112 TK_BlockMappingStart,
114 TK_FlowSequenceStart,
126 /// A string of length 0 or more whose begin() points to the logical location
127 /// of the token in the input.
130 Token() : Kind(TK_Error) {}
137 struct ilist_sentinel_traits<Token> {
138 Token *createSentinel() const {
141 static void destroySentinel(Token*) {}
143 Token *provideInitialHead() const { return createSentinel(); }
144 Token *ensureHead(Token*) const { return createSentinel(); }
145 static void noteHead(Token*, Token*) {}
148 mutable Token Sentinel;
152 struct ilist_node_traits<Token> {
153 Token *createNode(const Token &V) {
154 return new (Alloc.Allocate<Token>()) Token(V);
156 static void deleteNode(Token *V) {}
158 void addNodeToList(Token *) {}
159 void removeNodeFromList(Token *) {}
160 void transferNodesFromList(ilist_node_traits & /*SrcTraits*/,
161 ilist_iterator<Token> /*first*/,
162 ilist_iterator<Token> /*last*/) {}
164 BumpPtrAllocator Alloc;
168 typedef ilist<Token> TokenQueueT;
171 /// @brief This struct is used to track simple keys.
173 /// Simple keys are handled by creating an entry in SimpleKeys for each Token
174 /// which could legally be the start of a simple key. When peekNext is called,
175 /// if the Token To be returned is referenced by a SimpleKey, we continue
176 /// tokenizing until that potential simple key has either been found to not be
177 /// a simple key (we moved on to the next line or went further than 1024 chars).
178 /// Or when we run into a Value, and then insert a Key token (and possibly
179 /// others) before the SimpleKey's Tok.
181 TokenQueueT::iterator Tok;
187 bool operator ==(const SimpleKey &Other) {
188 return Tok == Other.Tok;
193 /// @brief The Unicode scalar value of a UTF-8 minimal well-formed code unit
194 /// subsequence and the subsequence's length in code units (uint8_t).
195 /// A length of 0 represents an error.
196 typedef std::pair<uint32_t, unsigned> UTF8Decoded;
198 static UTF8Decoded decodeUTF8(StringRef Range) {
199 StringRef::iterator Position= Range.begin();
200 StringRef::iterator End = Range.end();
201 // 1 byte: [0x00, 0x7f]
202 // Bit pattern: 0xxxxxxx
203 if ((*Position & 0x80) == 0) {
204 return std::make_pair(*Position, 1);
206 // 2 bytes: [0x80, 0x7ff]
207 // Bit pattern: 110xxxxx 10xxxxxx
208 if (Position + 1 != End &&
209 ((*Position & 0xE0) == 0xC0) &&
210 ((*(Position + 1) & 0xC0) == 0x80)) {
211 uint32_t codepoint = ((*Position & 0x1F) << 6) |
212 (*(Position + 1) & 0x3F);
213 if (codepoint >= 0x80)
214 return std::make_pair(codepoint, 2);
216 // 3 bytes: [0x8000, 0xffff]
217 // Bit pattern: 1110xxxx 10xxxxxx 10xxxxxx
218 if (Position + 2 != End &&
219 ((*Position & 0xF0) == 0xE0) &&
220 ((*(Position + 1) & 0xC0) == 0x80) &&
221 ((*(Position + 2) & 0xC0) == 0x80)) {
222 uint32_t codepoint = ((*Position & 0x0F) << 12) |
223 ((*(Position + 1) & 0x3F) << 6) |
224 (*(Position + 2) & 0x3F);
225 // Codepoints between 0xD800 and 0xDFFF are invalid, as
226 // they are high / low surrogate halves used by UTF-16.
227 if (codepoint >= 0x800 &&
228 (codepoint < 0xD800 || codepoint > 0xDFFF))
229 return std::make_pair(codepoint, 3);
231 // 4 bytes: [0x10000, 0x10FFFF]
232 // Bit pattern: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
233 if (Position + 3 != End &&
234 ((*Position & 0xF8) == 0xF0) &&
235 ((*(Position + 1) & 0xC0) == 0x80) &&
236 ((*(Position + 2) & 0xC0) == 0x80) &&
237 ((*(Position + 3) & 0xC0) == 0x80)) {
238 uint32_t codepoint = ((*Position & 0x07) << 18) |
239 ((*(Position + 1) & 0x3F) << 12) |
240 ((*(Position + 2) & 0x3F) << 6) |
241 (*(Position + 3) & 0x3F);
242 if (codepoint >= 0x10000 && codepoint <= 0x10FFFF)
243 return std::make_pair(codepoint, 4);
245 return std::make_pair(0, 0);
250 /// @brief Scans YAML tokens from a MemoryBuffer.
253 Scanner(const StringRef Input, SourceMgr &SM);
254 Scanner(MemoryBuffer *Buffer, SourceMgr &SM_);
256 /// @brief Parse the next token and return it without popping it.
259 /// @brief Parse the next token and pop it from the queue.
262 void printError(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Message,
263 ArrayRef<SMRange> Ranges = None) {
264 SM.PrintMessage(Loc, Kind, Message, Ranges);
267 void setError(const Twine &Message, StringRef::iterator Position) {
271 // Don't print out more errors after the first one we encounter. The rest
272 // are just the result of the first, and have no meaning.
274 printError(SMLoc::getFromPointer(Current), SourceMgr::DK_Error, Message);
278 void setError(const Twine &Message) {
279 setError(Message, Current);
282 /// @brief Returns true if an error occurred while parsing.
288 StringRef currentInput() {
289 return StringRef(Current, End - Current);
292 /// @brief Decode a UTF-8 minimal well-formed code unit subsequence starting
295 /// If the UTF-8 code units starting at Position do not form a well-formed
296 /// code unit subsequence, then the Unicode scalar value is 0, and the length
298 UTF8Decoded decodeUTF8(StringRef::iterator Position) {
299 return ::decodeUTF8(StringRef(Position, End - Position));
302 // The following functions are based on the gramar rules in the YAML spec. The
303 // style of the function names it meant to closely match how they are written
304 // in the spec. The number within the [] is the number of the grammar rule in
307 // See 4.2 [Production Naming Conventions] for the meaning of the prefixes.
310 // A production starting and ending with a special character.
312 // A production matching a single line break.
314 // A production starting and ending with a non-break character.
316 // A production starting and ending with a white space character.
318 // A production starting and ending with a non-space character.
320 // A production matching complete line(s).
322 /// @brief Skip a single nb-char[27] starting at Position.
324 /// A nb-char is 0x9 | [0x20-0x7E] | 0x85 | [0xA0-0xD7FF] | [0xE000-0xFEFE]
325 /// | [0xFF00-0xFFFD] | [0x10000-0x10FFFF]
327 /// @returns The code unit after the nb-char, or Position if it's not an
329 StringRef::iterator skip_nb_char(StringRef::iterator Position);
331 /// @brief Skip a single b-break[28] starting at Position.
333 /// A b-break is 0xD 0xA | 0xD | 0xA
335 /// @returns The code unit after the b-break, or Position if it's not a
337 StringRef::iterator skip_b_break(StringRef::iterator Position);
339 /// @brief Skip a single s-white[33] starting at Position.
341 /// A s-white is 0x20 | 0x9
343 /// @returns The code unit after the s-white, or Position if it's not a
345 StringRef::iterator skip_s_white(StringRef::iterator Position);
347 /// @brief Skip a single ns-char[34] starting at Position.
349 /// A ns-char is nb-char - s-white
351 /// @returns The code unit after the ns-char, or Position if it's not a
353 StringRef::iterator skip_ns_char(StringRef::iterator Position);
355 typedef StringRef::iterator (Scanner::*SkipWhileFunc)(StringRef::iterator);
356 /// @brief Skip minimal well-formed code unit subsequences until Func
357 /// returns its input.
359 /// @returns The code unit after the last minimal well-formed code unit
360 /// subsequence that Func accepted.
361 StringRef::iterator skip_while( SkipWhileFunc Func
362 , StringRef::iterator Position);
364 /// @brief Scan ns-uri-char[39]s starting at Cur.
366 /// This updates Cur and Column while scanning.
368 /// @returns A StringRef starting at Cur which covers the longest contiguous
369 /// sequence of ns-uri-char.
370 StringRef scan_ns_uri_char();
372 /// @brief Scan ns-plain-one-line[133] starting at \a Cur.
373 StringRef scan_ns_plain_one_line();
375 /// @brief Consume a minimal well-formed code unit subsequence starting at
376 /// \a Cur. Return false if it is not the same Unicode scalar value as
377 /// \a Expected. This updates \a Column.
378 bool consume(uint32_t Expected);
380 /// @brief Skip \a Distance UTF-8 code units. Updates \a Cur and \a Column.
381 void skip(uint32_t Distance);
383 /// @brief Return true if the minimal well-formed code unit subsequence at
384 /// Pos is whitespace or a new line
385 bool isBlankOrBreak(StringRef::iterator Position);
387 /// @brief If IsSimpleKeyAllowed, create and push_back a new SimpleKey.
388 void saveSimpleKeyCandidate( TokenQueueT::iterator Tok
392 /// @brief Remove simple keys that can no longer be valid simple keys.
394 /// Invalid simple keys are not on the current line or are further than 1024
396 void removeStaleSimpleKeyCandidates();
398 /// @brief Remove all simple keys on FlowLevel \a Level.
399 void removeSimpleKeyCandidatesOnFlowLevel(unsigned Level);
401 /// @brief Unroll indentation in \a Indents back to \a Col. Creates BlockEnd
402 /// tokens if needed.
403 bool unrollIndent(int ToColumn);
405 /// @brief Increase indent to \a Col. Creates \a Kind token at \a InsertPoint
407 bool rollIndent( int ToColumn
408 , Token::TokenKind Kind
409 , TokenQueueT::iterator InsertPoint);
411 /// @brief Skip whitespace and comments until the start of the next token.
412 void scanToNextToken();
414 /// @brief Must be the first token generated.
415 bool scanStreamStart();
417 /// @brief Generate tokens needed to close out the stream.
418 bool scanStreamEnd();
420 /// @brief Scan a %BLAH directive.
421 bool scanDirective();
423 /// @brief Scan a ... or ---.
424 bool scanDocumentIndicator(bool IsStart);
426 /// @brief Scan a [ or { and generate the proper flow collection start token.
427 bool scanFlowCollectionStart(bool IsSequence);
429 /// @brief Scan a ] or } and generate the proper flow collection end token.
430 bool scanFlowCollectionEnd(bool IsSequence);
432 /// @brief Scan the , that separates entries in a flow collection.
433 bool scanFlowEntry();
435 /// @brief Scan the - that starts block sequence entries.
436 bool scanBlockEntry();
438 /// @brief Scan an explicit ? indicating a key.
441 /// @brief Scan an explicit : indicating a value.
444 /// @brief Scan a quoted scalar.
445 bool scanFlowScalar(bool IsDoubleQuoted);
447 /// @brief Scan an unquoted scalar.
448 bool scanPlainScalar();
450 /// @brief Scan an Alias or Anchor starting with * or &.
451 bool scanAliasOrAnchor(bool IsAlias);
453 /// @brief Scan a block scalar starting with | or >.
454 bool scanBlockScalar(bool IsLiteral);
456 /// @brief Scan a tag of the form !stuff.
459 /// @brief Dispatch to the next scanning function based on \a *Cur.
460 bool fetchMoreTokens();
462 /// @brief The SourceMgr used for diagnostics and buffer management.
465 /// @brief The original input.
466 MemoryBuffer *InputBuffer;
468 /// @brief The current position of the scanner.
469 StringRef::iterator Current;
471 /// @brief The end of the input (one past the last character).
472 StringRef::iterator End;
474 /// @brief Current YAML indentation level in spaces.
477 /// @brief Current column number in Unicode code points.
480 /// @brief Current line number.
483 /// @brief How deep we are in flow style containers. 0 Means at block level.
486 /// @brief Are we at the start of the stream?
487 bool IsStartOfStream;
489 /// @brief Can the next token be the start of a simple key?
490 bool IsSimpleKeyAllowed;
492 /// @brief True if an error has occurred.
495 /// @brief Queue of tokens. This is required to queue up tokens while looking
496 /// for the end of a simple key. And for cases where a single character
497 /// can produce multiple tokens (e.g. BlockEnd).
498 TokenQueueT TokenQueue;
500 /// @brief Indentation levels.
501 SmallVector<int, 4> Indents;
503 /// @brief Potential simple keys.
504 SmallVector<SimpleKey, 4> SimpleKeys;
507 } // end namespace yaml
508 } // end namespace llvm
510 /// encodeUTF8 - Encode \a UnicodeScalarValue in UTF-8 and append it to result.
511 static void encodeUTF8( uint32_t UnicodeScalarValue
512 , SmallVectorImpl<char> &Result) {
513 if (UnicodeScalarValue <= 0x7F) {
514 Result.push_back(UnicodeScalarValue & 0x7F);
515 } else if (UnicodeScalarValue <= 0x7FF) {
516 uint8_t FirstByte = 0xC0 | ((UnicodeScalarValue & 0x7C0) >> 6);
517 uint8_t SecondByte = 0x80 | (UnicodeScalarValue & 0x3F);
518 Result.push_back(FirstByte);
519 Result.push_back(SecondByte);
520 } else if (UnicodeScalarValue <= 0xFFFF) {
521 uint8_t FirstByte = 0xE0 | ((UnicodeScalarValue & 0xF000) >> 12);
522 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
523 uint8_t ThirdByte = 0x80 | (UnicodeScalarValue & 0x3F);
524 Result.push_back(FirstByte);
525 Result.push_back(SecondByte);
526 Result.push_back(ThirdByte);
527 } else if (UnicodeScalarValue <= 0x10FFFF) {
528 uint8_t FirstByte = 0xF0 | ((UnicodeScalarValue & 0x1F0000) >> 18);
529 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0x3F000) >> 12);
530 uint8_t ThirdByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
531 uint8_t FourthByte = 0x80 | (UnicodeScalarValue & 0x3F);
532 Result.push_back(FirstByte);
533 Result.push_back(SecondByte);
534 Result.push_back(ThirdByte);
535 Result.push_back(FourthByte);
539 bool yaml::dumpTokens(StringRef Input, raw_ostream &OS) {
541 Scanner scanner(Input, SM);
543 Token T = scanner.getNext();
545 case Token::TK_StreamStart:
546 OS << "Stream-Start: ";
548 case Token::TK_StreamEnd:
549 OS << "Stream-End: ";
551 case Token::TK_VersionDirective:
552 OS << "Version-Directive: ";
554 case Token::TK_TagDirective:
555 OS << "Tag-Directive: ";
557 case Token::TK_DocumentStart:
558 OS << "Document-Start: ";
560 case Token::TK_DocumentEnd:
561 OS << "Document-End: ";
563 case Token::TK_BlockEntry:
564 OS << "Block-Entry: ";
566 case Token::TK_BlockEnd:
569 case Token::TK_BlockSequenceStart:
570 OS << "Block-Sequence-Start: ";
572 case Token::TK_BlockMappingStart:
573 OS << "Block-Mapping-Start: ";
575 case Token::TK_FlowEntry:
576 OS << "Flow-Entry: ";
578 case Token::TK_FlowSequenceStart:
579 OS << "Flow-Sequence-Start: ";
581 case Token::TK_FlowSequenceEnd:
582 OS << "Flow-Sequence-End: ";
584 case Token::TK_FlowMappingStart:
585 OS << "Flow-Mapping-Start: ";
587 case Token::TK_FlowMappingEnd:
588 OS << "Flow-Mapping-End: ";
593 case Token::TK_Value:
596 case Token::TK_Scalar:
599 case Token::TK_Alias:
602 case Token::TK_Anchor:
608 case Token::TK_Error:
611 OS << T.Range << "\n";
612 if (T.Kind == Token::TK_StreamEnd)
614 else if (T.Kind == Token::TK_Error)
620 bool yaml::scanTokens(StringRef Input) {
622 llvm::yaml::Scanner scanner(Input, SM);
624 llvm::yaml::Token T = scanner.getNext();
625 if (T.Kind == Token::TK_StreamEnd)
627 else if (T.Kind == Token::TK_Error)
633 std::string yaml::escape(StringRef Input) {
634 std::string EscapedInput;
635 for (StringRef::iterator i = Input.begin(), e = Input.end(); i != e; ++i) {
637 EscapedInput += "\\\\";
639 EscapedInput += "\\\"";
641 EscapedInput += "\\0";
643 EscapedInput += "\\a";
645 EscapedInput += "\\b";
647 EscapedInput += "\\t";
649 EscapedInput += "\\n";
651 EscapedInput += "\\v";
653 EscapedInput += "\\f";
655 EscapedInput += "\\r";
657 EscapedInput += "\\e";
658 else if ((unsigned char)*i < 0x20) { // Control characters not handled above.
659 std::string HexStr = utohexstr(*i);
660 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
661 } else if (*i & 0x80) { // UTF-8 multiple code unit subsequence.
662 UTF8Decoded UnicodeScalarValue
663 = decodeUTF8(StringRef(i, Input.end() - i));
664 if (UnicodeScalarValue.second == 0) {
665 // Found invalid char.
667 encodeUTF8(0xFFFD, Val);
668 EscapedInput.insert(EscapedInput.end(), Val.begin(), Val.end());
669 // FIXME: Error reporting.
672 if (UnicodeScalarValue.first == 0x85)
673 EscapedInput += "\\N";
674 else if (UnicodeScalarValue.first == 0xA0)
675 EscapedInput += "\\_";
676 else if (UnicodeScalarValue.first == 0x2028)
677 EscapedInput += "\\L";
678 else if (UnicodeScalarValue.first == 0x2029)
679 EscapedInput += "\\P";
681 std::string HexStr = utohexstr(UnicodeScalarValue.first);
682 if (HexStr.size() <= 2)
683 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
684 else if (HexStr.size() <= 4)
685 EscapedInput += "\\u" + std::string(4 - HexStr.size(), '0') + HexStr;
686 else if (HexStr.size() <= 8)
687 EscapedInput += "\\U" + std::string(8 - HexStr.size(), '0') + HexStr;
689 i += UnicodeScalarValue.second - 1;
691 EscapedInput.push_back(*i);
696 Scanner::Scanner(StringRef Input, SourceMgr &sm)
702 , IsStartOfStream(true)
703 , IsSimpleKeyAllowed(true)
705 InputBuffer = MemoryBuffer::getMemBuffer(Input, "YAML");
706 SM.AddNewSourceBuffer(InputBuffer, SMLoc());
707 Current = InputBuffer->getBufferStart();
708 End = InputBuffer->getBufferEnd();
711 Scanner::Scanner(MemoryBuffer *Buffer, SourceMgr &SM_)
713 , InputBuffer(Buffer)
714 , Current(InputBuffer->getBufferStart())
715 , End(InputBuffer->getBufferEnd())
720 , IsStartOfStream(true)
721 , IsSimpleKeyAllowed(true)
723 SM.AddNewSourceBuffer(InputBuffer, SMLoc());
726 Token &Scanner::peekNext() {
727 // If the current token is a possible simple key, keep parsing until we
729 bool NeedMore = false;
731 if (TokenQueue.empty() || NeedMore) {
732 if (!fetchMoreTokens()) {
734 TokenQueue.push_back(Token());
735 return TokenQueue.front();
738 assert(!TokenQueue.empty() &&
739 "fetchMoreTokens lied about getting tokens!");
741 removeStaleSimpleKeyCandidates();
743 SK.Tok = TokenQueue.front();
744 if (std::find(SimpleKeys.begin(), SimpleKeys.end(), SK)
750 return TokenQueue.front();
753 Token Scanner::getNext() {
754 Token Ret = peekNext();
755 // TokenQueue can be empty if there was an error getting the next token.
756 if (!TokenQueue.empty())
757 TokenQueue.pop_front();
759 // There cannot be any referenced Token's if the TokenQueue is empty. So do a
760 // quick deallocation of them all.
761 if (TokenQueue.empty()) {
762 TokenQueue.Alloc.Reset();
768 StringRef::iterator Scanner::skip_nb_char(StringRef::iterator Position) {
771 // Check 7 bit c-printable - b-char.
772 if ( *Position == 0x09
773 || (*Position >= 0x20 && *Position <= 0x7E))
776 // Check for valid UTF-8.
777 if (uint8_t(*Position) & 0x80) {
778 UTF8Decoded u8d = decodeUTF8(Position);
780 && u8d.first != 0xFEFF
781 && ( u8d.first == 0x85
782 || ( u8d.first >= 0xA0
783 && u8d.first <= 0xD7FF)
784 || ( u8d.first >= 0xE000
785 && u8d.first <= 0xFFFD)
786 || ( u8d.first >= 0x10000
787 && u8d.first <= 0x10FFFF)))
788 return Position + u8d.second;
793 StringRef::iterator Scanner::skip_b_break(StringRef::iterator Position) {
796 if (*Position == 0x0D) {
797 if (Position + 1 != End && *(Position + 1) == 0x0A)
802 if (*Position == 0x0A)
808 StringRef::iterator Scanner::skip_s_white(StringRef::iterator Position) {
811 if (*Position == ' ' || *Position == '\t')
816 StringRef::iterator Scanner::skip_ns_char(StringRef::iterator Position) {
819 if (*Position == ' ' || *Position == '\t')
821 return skip_nb_char(Position);
824 StringRef::iterator Scanner::skip_while( SkipWhileFunc Func
825 , StringRef::iterator Position) {
827 StringRef::iterator i = (this->*Func)(Position);
835 static bool is_ns_hex_digit(const char C) {
836 return (C >= '0' && C <= '9')
837 || (C >= 'a' && C <= 'z')
838 || (C >= 'A' && C <= 'Z');
841 static bool is_ns_word_char(const char C) {
843 || (C >= 'a' && C <= 'z')
844 || (C >= 'A' && C <= 'Z');
847 StringRef Scanner::scan_ns_uri_char() {
848 StringRef::iterator Start = Current;
852 if (( *Current == '%'
854 && is_ns_hex_digit(*(Current + 1))
855 && is_ns_hex_digit(*(Current + 2)))
856 || is_ns_word_char(*Current)
857 || StringRef(Current, 1).find_first_of("#;/?:@&=+$,_.!~*'()[]")
858 != StringRef::npos) {
864 return StringRef(Start, Current - Start);
867 StringRef Scanner::scan_ns_plain_one_line() {
868 StringRef::iterator start = Current;
869 // The first character must already be verified.
872 if (Current == End) {
874 } else if (*Current == ':') {
875 // Check if the next character is a ns-char.
876 if (Current + 1 == End)
878 StringRef::iterator i = skip_ns_char(Current + 1);
879 if (Current + 1 != i) {
881 Column += 2; // Consume both the ':' and ns-char.
884 } else if (*Current == '#') {
885 // Check if the previous character was a ns-char.
886 // The & 0x80 check is to check for the trailing byte of a utf-8
887 if (*(Current - 1) & 0x80 || skip_ns_char(Current - 1) == Current) {
893 StringRef::iterator i = skip_nb_char(Current);
900 return StringRef(start, Current - start);
903 bool Scanner::consume(uint32_t Expected) {
904 if (Expected >= 0x80)
905 report_fatal_error("Not dealing with this yet");
908 if (uint8_t(*Current) >= 0x80)
909 report_fatal_error("Not dealing with this yet");
910 if (uint8_t(*Current) == Expected) {
918 void Scanner::skip(uint32_t Distance) {
921 assert(Current <= End && "Skipped past the end");
924 bool Scanner::isBlankOrBreak(StringRef::iterator Position) {
927 if ( *Position == ' ' || *Position == '\t'
928 || *Position == '\r' || *Position == '\n')
933 void Scanner::saveSimpleKeyCandidate( TokenQueueT::iterator Tok
936 if (IsSimpleKeyAllowed) {
940 SK.Column = AtColumn;
941 SK.IsRequired = IsRequired;
942 SK.FlowLevel = FlowLevel;
943 SimpleKeys.push_back(SK);
947 void Scanner::removeStaleSimpleKeyCandidates() {
948 for (SmallVectorImpl<SimpleKey>::iterator i = SimpleKeys.begin();
949 i != SimpleKeys.end();) {
950 if (i->Line != Line || i->Column + 1024 < Column) {
952 setError( "Could not find expected : for simple key"
953 , i->Tok->Range.begin());
954 i = SimpleKeys.erase(i);
960 void Scanner::removeSimpleKeyCandidatesOnFlowLevel(unsigned Level) {
961 if (!SimpleKeys.empty() && (SimpleKeys.end() - 1)->FlowLevel == Level)
962 SimpleKeys.pop_back();
965 bool Scanner::unrollIndent(int ToColumn) {
967 // Indentation is ignored in flow.
971 while (Indent > ToColumn) {
972 T.Kind = Token::TK_BlockEnd;
973 T.Range = StringRef(Current, 1);
974 TokenQueue.push_back(T);
975 Indent = Indents.pop_back_val();
981 bool Scanner::rollIndent( int ToColumn
982 , Token::TokenKind Kind
983 , TokenQueueT::iterator InsertPoint) {
986 if (Indent < ToColumn) {
987 Indents.push_back(Indent);
992 T.Range = StringRef(Current, 0);
993 TokenQueue.insert(InsertPoint, T);
998 void Scanner::scanToNextToken() {
1000 while (*Current == ' ' || *Current == '\t') {
1005 if (*Current == '#') {
1007 // This may skip more than one byte, thus Column is only incremented
1009 StringRef::iterator i = skip_nb_char(Current);
1018 StringRef::iterator i = skip_b_break(Current);
1024 // New lines may start a simple key.
1026 IsSimpleKeyAllowed = true;
1030 bool Scanner::scanStreamStart() {
1031 IsStartOfStream = false;
1033 EncodingInfo EI = getUnicodeEncoding(currentInput());
1036 T.Kind = Token::TK_StreamStart;
1037 T.Range = StringRef(Current, EI.second);
1038 TokenQueue.push_back(T);
1039 Current += EI.second;
1043 bool Scanner::scanStreamEnd() {
1044 // Force an ending new line if one isn't present.
1052 IsSimpleKeyAllowed = false;
1055 T.Kind = Token::TK_StreamEnd;
1056 T.Range = StringRef(Current, 0);
1057 TokenQueue.push_back(T);
1061 bool Scanner::scanDirective() {
1062 // Reset the indentation level.
1065 IsSimpleKeyAllowed = false;
1067 StringRef::iterator Start = Current;
1069 StringRef::iterator NameStart = Current;
1070 Current = skip_while(&Scanner::skip_ns_char, Current);
1071 StringRef Name(NameStart, Current - NameStart);
1072 Current = skip_while(&Scanner::skip_s_white, Current);
1075 if (Name == "YAML") {
1076 Current = skip_while(&Scanner::skip_ns_char, Current);
1077 T.Kind = Token::TK_VersionDirective;
1078 T.Range = StringRef(Start, Current - Start);
1079 TokenQueue.push_back(T);
1081 } else if(Name == "TAG") {
1082 Current = skip_while(&Scanner::skip_ns_char, Current);
1083 Current = skip_while(&Scanner::skip_s_white, Current);
1084 Current = skip_while(&Scanner::skip_ns_char, Current);
1085 T.Kind = Token::TK_TagDirective;
1086 T.Range = StringRef(Start, Current - Start);
1087 TokenQueue.push_back(T);
1093 bool Scanner::scanDocumentIndicator(bool IsStart) {
1096 IsSimpleKeyAllowed = false;
1099 T.Kind = IsStart ? Token::TK_DocumentStart : Token::TK_DocumentEnd;
1100 T.Range = StringRef(Current, 3);
1102 TokenQueue.push_back(T);
1106 bool Scanner::scanFlowCollectionStart(bool IsSequence) {
1108 T.Kind = IsSequence ? Token::TK_FlowSequenceStart
1109 : Token::TK_FlowMappingStart;
1110 T.Range = StringRef(Current, 1);
1112 TokenQueue.push_back(T);
1114 // [ and { may begin a simple key.
1115 saveSimpleKeyCandidate(TokenQueue.back(), Column - 1, false);
1117 // And may also be followed by a simple key.
1118 IsSimpleKeyAllowed = true;
1123 bool Scanner::scanFlowCollectionEnd(bool IsSequence) {
1124 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1125 IsSimpleKeyAllowed = false;
1127 T.Kind = IsSequence ? Token::TK_FlowSequenceEnd
1128 : Token::TK_FlowMappingEnd;
1129 T.Range = StringRef(Current, 1);
1131 TokenQueue.push_back(T);
1137 bool Scanner::scanFlowEntry() {
1138 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1139 IsSimpleKeyAllowed = true;
1141 T.Kind = Token::TK_FlowEntry;
1142 T.Range = StringRef(Current, 1);
1144 TokenQueue.push_back(T);
1148 bool Scanner::scanBlockEntry() {
1149 rollIndent(Column, Token::TK_BlockSequenceStart, TokenQueue.end());
1150 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1151 IsSimpleKeyAllowed = true;
1153 T.Kind = Token::TK_BlockEntry;
1154 T.Range = StringRef(Current, 1);
1156 TokenQueue.push_back(T);
1160 bool Scanner::scanKey() {
1162 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1164 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1165 IsSimpleKeyAllowed = !FlowLevel;
1168 T.Kind = Token::TK_Key;
1169 T.Range = StringRef(Current, 1);
1171 TokenQueue.push_back(T);
1175 bool Scanner::scanValue() {
1176 // If the previous token could have been a simple key, insert the key token
1177 // into the token queue.
1178 if (!SimpleKeys.empty()) {
1179 SimpleKey SK = SimpleKeys.pop_back_val();
1181 T.Kind = Token::TK_Key;
1182 T.Range = SK.Tok->Range;
1183 TokenQueueT::iterator i, e;
1184 for (i = TokenQueue.begin(), e = TokenQueue.end(); i != e; ++i) {
1188 assert(i != e && "SimpleKey not in token queue!");
1189 i = TokenQueue.insert(i, T);
1191 // We may also need to add a Block-Mapping-Start token.
1192 rollIndent(SK.Column, Token::TK_BlockMappingStart, i);
1194 IsSimpleKeyAllowed = false;
1197 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1198 IsSimpleKeyAllowed = !FlowLevel;
1202 T.Kind = Token::TK_Value;
1203 T.Range = StringRef(Current, 1);
1205 TokenQueue.push_back(T);
1209 // Forbidding inlining improves performance by roughly 20%.
1210 // FIXME: Remove once llvm optimizes this to the faster version without hints.
1211 LLVM_ATTRIBUTE_NOINLINE static bool
1212 wasEscaped(StringRef::iterator First, StringRef::iterator Position);
1214 // Returns whether a character at 'Position' was escaped with a leading '\'.
1215 // 'First' specifies the position of the first character in the string.
1216 static bool wasEscaped(StringRef::iterator First,
1217 StringRef::iterator Position) {
1218 assert(Position - 1 >= First);
1219 StringRef::iterator I = Position - 1;
1220 // We calculate the number of consecutive '\'s before the current position
1221 // by iterating backwards through our string.
1222 while (I >= First && *I == '\\') --I;
1223 // (Position - 1 - I) now contains the number of '\'s before the current
1224 // position. If it is odd, the character at 'Position' was escaped.
1225 return (Position - 1 - I) % 2 == 1;
1228 bool Scanner::scanFlowScalar(bool IsDoubleQuoted) {
1229 StringRef::iterator Start = Current;
1230 unsigned ColStart = Column;
1231 if (IsDoubleQuoted) {
1234 while (Current != End && *Current != '"')
1236 // Repeat until the previous character was not a '\' or was an escaped
1238 } while ( Current != End
1239 && *(Current - 1) == '\\'
1240 && wasEscaped(Start + 1, Current));
1244 // Skip a ' followed by another '.
1245 if (Current + 1 < End && *Current == '\'' && *(Current + 1) == '\'') {
1248 } else if (*Current == '\'')
1250 StringRef::iterator i = skip_nb_char(Current);
1252 i = skip_b_break(Current);
1267 if (Current == End) {
1268 setError("Expected quote at end of scalar", Current);
1272 skip(1); // Skip ending quote.
1274 T.Kind = Token::TK_Scalar;
1275 T.Range = StringRef(Start, Current - Start);
1276 TokenQueue.push_back(T);
1278 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1280 IsSimpleKeyAllowed = false;
1285 bool Scanner::scanPlainScalar() {
1286 StringRef::iterator Start = Current;
1287 unsigned ColStart = Column;
1288 unsigned LeadingBlanks = 0;
1289 assert(Indent >= -1 && "Indent must be >= -1 !");
1290 unsigned indent = static_cast<unsigned>(Indent + 1);
1292 if (*Current == '#')
1295 while (!isBlankOrBreak(Current)) {
1296 if ( FlowLevel && *Current == ':'
1297 && !(isBlankOrBreak(Current + 1) || *(Current + 1) == ',')) {
1298 setError("Found unexpected ':' while scanning a plain scalar", Current);
1302 // Check for the end of the plain scalar.
1303 if ( (*Current == ':' && isBlankOrBreak(Current + 1))
1305 && (StringRef(Current, 1).find_first_of(",:?[]{}")
1306 != StringRef::npos)))
1309 StringRef::iterator i = skip_nb_char(Current);
1316 // Are we at the end?
1317 if (!isBlankOrBreak(Current))
1321 StringRef::iterator Tmp = Current;
1322 while (isBlankOrBreak(Tmp)) {
1323 StringRef::iterator i = skip_s_white(Tmp);
1325 if (LeadingBlanks && (Column < indent) && *Tmp == '\t') {
1326 setError("Found invalid tab character in indentation", Tmp);
1332 i = skip_b_break(Tmp);
1341 if (!FlowLevel && Column < indent)
1346 if (Start == Current) {
1347 setError("Got empty plain scalar", Start);
1351 T.Kind = Token::TK_Scalar;
1352 T.Range = StringRef(Start, Current - Start);
1353 TokenQueue.push_back(T);
1355 // Plain scalars can be simple keys.
1356 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1358 IsSimpleKeyAllowed = false;
1363 bool Scanner::scanAliasOrAnchor(bool IsAlias) {
1364 StringRef::iterator Start = Current;
1365 unsigned ColStart = Column;
1368 if ( *Current == '[' || *Current == ']'
1369 || *Current == '{' || *Current == '}'
1373 StringRef::iterator i = skip_ns_char(Current);
1380 if (Start == Current) {
1381 setError("Got empty alias or anchor", Start);
1386 T.Kind = IsAlias ? Token::TK_Alias : Token::TK_Anchor;
1387 T.Range = StringRef(Start, Current - Start);
1388 TokenQueue.push_back(T);
1390 // Alias and anchors can be simple keys.
1391 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1393 IsSimpleKeyAllowed = false;
1398 bool Scanner::scanBlockScalar(bool IsLiteral) {
1399 StringRef::iterator Start = Current;
1400 skip(1); // Eat | or >
1402 StringRef::iterator i = skip_nb_char(Current);
1406 i = skip_b_break(Current);
1408 // We got a line break.
1414 // There was an error, which should already have been printed out.
1422 if (Start == Current) {
1423 setError("Got empty block scalar", Start);
1428 T.Kind = Token::TK_Scalar;
1429 T.Range = StringRef(Start, Current - Start);
1430 TokenQueue.push_back(T);
1434 bool Scanner::scanTag() {
1435 StringRef::iterator Start = Current;
1436 unsigned ColStart = Column;
1438 if (Current == End || isBlankOrBreak(Current)); // An empty tag.
1439 else if (*Current == '<') {
1445 // FIXME: Actually parse the c-ns-shorthand-tag rule.
1446 Current = skip_while(&Scanner::skip_ns_char, Current);
1450 T.Kind = Token::TK_Tag;
1451 T.Range = StringRef(Start, Current - Start);
1452 TokenQueue.push_back(T);
1454 // Tags can be simple keys.
1455 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1457 IsSimpleKeyAllowed = false;
1462 bool Scanner::fetchMoreTokens() {
1463 if (IsStartOfStream)
1464 return scanStreamStart();
1469 return scanStreamEnd();
1471 removeStaleSimpleKeyCandidates();
1473 unrollIndent(Column);
1475 if (Column == 0 && *Current == '%')
1476 return scanDirective();
1478 if (Column == 0 && Current + 4 <= End
1480 && *(Current + 1) == '-'
1481 && *(Current + 2) == '-'
1482 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1483 return scanDocumentIndicator(true);
1485 if (Column == 0 && Current + 4 <= End
1487 && *(Current + 1) == '.'
1488 && *(Current + 2) == '.'
1489 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1490 return scanDocumentIndicator(false);
1492 if (*Current == '[')
1493 return scanFlowCollectionStart(true);
1495 if (*Current == '{')
1496 return scanFlowCollectionStart(false);
1498 if (*Current == ']')
1499 return scanFlowCollectionEnd(true);
1501 if (*Current == '}')
1502 return scanFlowCollectionEnd(false);
1504 if (*Current == ',')
1505 return scanFlowEntry();
1507 if (*Current == '-' && isBlankOrBreak(Current + 1))
1508 return scanBlockEntry();
1510 if (*Current == '?' && (FlowLevel || isBlankOrBreak(Current + 1)))
1513 if (*Current == ':' && (FlowLevel || isBlankOrBreak(Current + 1)))
1516 if (*Current == '*')
1517 return scanAliasOrAnchor(true);
1519 if (*Current == '&')
1520 return scanAliasOrAnchor(false);
1522 if (*Current == '!')
1525 if (*Current == '|' && !FlowLevel)
1526 return scanBlockScalar(true);
1528 if (*Current == '>' && !FlowLevel)
1529 return scanBlockScalar(false);
1531 if (*Current == '\'')
1532 return scanFlowScalar(false);
1534 if (*Current == '"')
1535 return scanFlowScalar(true);
1537 // Get a plain scalar.
1538 StringRef FirstChar(Current, 1);
1539 if (!(isBlankOrBreak(Current)
1540 || FirstChar.find_first_of("-?:,[]{}#&*!|>'\"%@`") != StringRef::npos)
1541 || (*Current == '-' && !isBlankOrBreak(Current + 1))
1542 || (!FlowLevel && (*Current == '?' || *Current == ':')
1543 && isBlankOrBreak(Current + 1))
1544 || (!FlowLevel && *Current == ':'
1545 && Current + 2 < End
1546 && *(Current + 1) == ':'
1547 && !isBlankOrBreak(Current + 2)))
1548 return scanPlainScalar();
1550 setError("Unrecognized character while tokenizing.");
1554 Stream::Stream(StringRef Input, SourceMgr &SM)
1555 : scanner(new Scanner(Input, SM))
1558 Stream::Stream(MemoryBuffer *InputBuffer, SourceMgr &SM)
1559 : scanner(new Scanner(InputBuffer, SM))
1562 Stream::~Stream() {}
1564 bool Stream::failed() { return scanner->failed(); }
1566 void Stream::printError(Node *N, const Twine &Msg) {
1567 SmallVector<SMRange, 1> Ranges;
1568 Ranges.push_back(N->getSourceRange());
1569 scanner->printError( N->getSourceRange().Start
1570 , SourceMgr::DK_Error
1575 document_iterator Stream::begin() {
1577 report_fatal_error("Can only iterate over the stream once");
1579 // Skip Stream-Start.
1582 CurrentDoc.reset(new Document(*this));
1583 return document_iterator(CurrentDoc);
1586 document_iterator Stream::end() {
1587 return document_iterator();
1590 void Stream::skip() {
1591 for (document_iterator i = begin(), e = end(); i != e; ++i)
1595 Node::Node(unsigned int Type, OwningPtr<Document> &D, StringRef A, StringRef T)
1600 SMLoc Start = SMLoc::getFromPointer(peekNext().Range.begin());
1601 SourceRange = SMRange(Start, Start);
1604 std::string Node::getVerbatimTag() const {
1605 StringRef Raw = getRawTag();
1606 if (!Raw.empty() && Raw != "!") {
1608 if (Raw.find_last_of('!') == 0) {
1609 Ret = Doc->getTagMap().find("!")->second;
1610 Ret += Raw.substr(1);
1611 return std::move(Ret);
1612 } else if (Raw.startswith("!!")) {
1613 Ret = Doc->getTagMap().find("!!")->second;
1614 Ret += Raw.substr(2);
1615 return std::move(Ret);
1617 StringRef TagHandle = Raw.substr(0, Raw.find_last_of('!') + 1);
1618 std::map<StringRef, StringRef>::const_iterator It =
1619 Doc->getTagMap().find(TagHandle);
1620 if (It != Doc->getTagMap().end())
1624 T.Kind = Token::TK_Tag;
1625 T.Range = TagHandle;
1626 setError(Twine("Unknown tag handle ") + TagHandle, T);
1628 Ret += Raw.substr(Raw.find_last_of('!') + 1);
1629 return std::move(Ret);
1633 switch (getType()) {
1635 return "tag:yaml.org,2002:null";
1637 // TODO: Tag resolution.
1638 return "tag:yaml.org,2002:str";
1640 return "tag:yaml.org,2002:map";
1642 return "tag:yaml.org,2002:seq";
1648 Token &Node::peekNext() {
1649 return Doc->peekNext();
1652 Token Node::getNext() {
1653 return Doc->getNext();
1656 Node *Node::parseBlockNode() {
1657 return Doc->parseBlockNode();
1660 BumpPtrAllocator &Node::getAllocator() {
1661 return Doc->NodeAllocator;
1664 void Node::setError(const Twine &Msg, Token &Tok) const {
1665 Doc->setError(Msg, Tok);
1668 bool Node::failed() const {
1669 return Doc->failed();
1674 StringRef ScalarNode::getValue(SmallVectorImpl<char> &Storage) const {
1675 // TODO: Handle newlines properly. We need to remove leading whitespace.
1676 if (Value[0] == '"') { // Double quoted.
1677 // Pull off the leading and trailing "s.
1678 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1679 // Search for characters that would require unescaping the value.
1680 StringRef::size_type i = UnquotedValue.find_first_of("\\\r\n");
1681 if (i != StringRef::npos)
1682 return unescapeDoubleQuoted(UnquotedValue, i, Storage);
1683 return UnquotedValue;
1684 } else if (Value[0] == '\'') { // Single quoted.
1685 // Pull off the leading and trailing 's.
1686 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1687 StringRef::size_type i = UnquotedValue.find('\'');
1688 if (i != StringRef::npos) {
1689 // We're going to need Storage.
1691 Storage.reserve(UnquotedValue.size());
1692 for (; i != StringRef::npos; i = UnquotedValue.find('\'')) {
1693 StringRef Valid(UnquotedValue.begin(), i);
1694 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1695 Storage.push_back('\'');
1696 UnquotedValue = UnquotedValue.substr(i + 2);
1698 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1699 return StringRef(Storage.begin(), Storage.size());
1701 return UnquotedValue;
1704 return Value.rtrim(" ");
1707 StringRef ScalarNode::unescapeDoubleQuoted( StringRef UnquotedValue
1708 , StringRef::size_type i
1709 , SmallVectorImpl<char> &Storage)
1711 // Use Storage to build proper value.
1713 Storage.reserve(UnquotedValue.size());
1714 for (; i != StringRef::npos; i = UnquotedValue.find_first_of("\\\r\n")) {
1715 // Insert all previous chars into Storage.
1716 StringRef Valid(UnquotedValue.begin(), i);
1717 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1718 // Chop off inserted chars.
1719 UnquotedValue = UnquotedValue.substr(i);
1721 assert(!UnquotedValue.empty() && "Can't be empty!");
1723 // Parse escape or line break.
1724 switch (UnquotedValue[0]) {
1727 Storage.push_back('\n');
1728 if ( UnquotedValue.size() > 1
1729 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1730 UnquotedValue = UnquotedValue.substr(1);
1731 UnquotedValue = UnquotedValue.substr(1);
1734 if (UnquotedValue.size() == 1)
1735 // TODO: Report error.
1737 UnquotedValue = UnquotedValue.substr(1);
1738 switch (UnquotedValue[0]) {
1741 T.Range = StringRef(UnquotedValue.begin(), 1);
1742 setError("Unrecognized escape code!", T);
1747 // Remove the new line.
1748 if ( UnquotedValue.size() > 1
1749 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1750 UnquotedValue = UnquotedValue.substr(1);
1751 // If this was just a single byte newline, it will get skipped
1755 Storage.push_back(0x00);
1758 Storage.push_back(0x07);
1761 Storage.push_back(0x08);
1765 Storage.push_back(0x09);
1768 Storage.push_back(0x0A);
1771 Storage.push_back(0x0B);
1774 Storage.push_back(0x0C);
1777 Storage.push_back(0x0D);
1780 Storage.push_back(0x1B);
1783 Storage.push_back(0x20);
1786 Storage.push_back(0x22);
1789 Storage.push_back(0x2F);
1792 Storage.push_back(0x5C);
1795 encodeUTF8(0x85, Storage);
1798 encodeUTF8(0xA0, Storage);
1801 encodeUTF8(0x2028, Storage);
1804 encodeUTF8(0x2029, Storage);
1807 if (UnquotedValue.size() < 3)
1808 // TODO: Report error.
1810 unsigned int UnicodeScalarValue;
1811 if (UnquotedValue.substr(1, 2).getAsInteger(16, UnicodeScalarValue))
1812 // TODO: Report error.
1813 UnicodeScalarValue = 0xFFFD;
1814 encodeUTF8(UnicodeScalarValue, Storage);
1815 UnquotedValue = UnquotedValue.substr(2);
1819 if (UnquotedValue.size() < 5)
1820 // TODO: Report error.
1822 unsigned int UnicodeScalarValue;
1823 if (UnquotedValue.substr(1, 4).getAsInteger(16, UnicodeScalarValue))
1824 // TODO: Report error.
1825 UnicodeScalarValue = 0xFFFD;
1826 encodeUTF8(UnicodeScalarValue, Storage);
1827 UnquotedValue = UnquotedValue.substr(4);
1831 if (UnquotedValue.size() < 9)
1832 // TODO: Report error.
1834 unsigned int UnicodeScalarValue;
1835 if (UnquotedValue.substr(1, 8).getAsInteger(16, UnicodeScalarValue))
1836 // TODO: Report error.
1837 UnicodeScalarValue = 0xFFFD;
1838 encodeUTF8(UnicodeScalarValue, Storage);
1839 UnquotedValue = UnquotedValue.substr(8);
1843 UnquotedValue = UnquotedValue.substr(1);
1846 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1847 return StringRef(Storage.begin(), Storage.size());
1850 Node *KeyValueNode::getKey() {
1853 // Handle implicit null keys.
1855 Token &t = peekNext();
1856 if ( t.Kind == Token::TK_BlockEnd
1857 || t.Kind == Token::TK_Value
1858 || t.Kind == Token::TK_Error) {
1859 return Key = new (getAllocator()) NullNode(Doc);
1861 if (t.Kind == Token::TK_Key)
1862 getNext(); // skip TK_Key.
1865 // Handle explicit null keys.
1866 Token &t = peekNext();
1867 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Value) {
1868 return Key = new (getAllocator()) NullNode(Doc);
1871 // We've got a normal key.
1872 return Key = parseBlockNode();
1875 Node *KeyValueNode::getValue() {
1880 return Value = new (getAllocator()) NullNode(Doc);
1882 // Handle implicit null values.
1884 Token &t = peekNext();
1885 if ( t.Kind == Token::TK_BlockEnd
1886 || t.Kind == Token::TK_FlowMappingEnd
1887 || t.Kind == Token::TK_Key
1888 || t.Kind == Token::TK_FlowEntry
1889 || t.Kind == Token::TK_Error) {
1890 return Value = new (getAllocator()) NullNode(Doc);
1893 if (t.Kind != Token::TK_Value) {
1894 setError("Unexpected token in Key Value.", t);
1895 return Value = new (getAllocator()) NullNode(Doc);
1897 getNext(); // skip TK_Value.
1900 // Handle explicit null values.
1901 Token &t = peekNext();
1902 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Key) {
1903 return Value = new (getAllocator()) NullNode(Doc);
1906 // We got a normal value.
1907 return Value = parseBlockNode();
1910 void MappingNode::increment() {
1917 CurrentEntry->skip();
1918 if (Type == MT_Inline) {
1924 Token T = peekNext();
1925 if (T.Kind == Token::TK_Key || T.Kind == Token::TK_Scalar) {
1926 // KeyValueNode eats the TK_Key. That way it can detect null keys.
1927 CurrentEntry = new (getAllocator()) KeyValueNode(Doc);
1928 } else if (Type == MT_Block) {
1930 case Token::TK_BlockEnd:
1936 setError("Unexpected token. Expected Key or Block End", T);
1937 case Token::TK_Error:
1943 case Token::TK_FlowEntry:
1944 // Eat the flow entry and recurse.
1947 case Token::TK_FlowMappingEnd:
1949 case Token::TK_Error:
1950 // Set this to end iterator.
1955 setError( "Unexpected token. Expected Key, Flow Entry, or Flow "
1964 void SequenceNode::increment() {
1971 CurrentEntry->skip();
1972 Token T = peekNext();
1973 if (SeqType == ST_Block) {
1975 case Token::TK_BlockEntry:
1977 CurrentEntry = parseBlockNode();
1978 if (CurrentEntry == 0) { // An error occurred.
1983 case Token::TK_BlockEnd:
1989 setError( "Unexpected token. Expected Block Entry or Block End."
1991 case Token::TK_Error:
1995 } else if (SeqType == ST_Indentless) {
1997 case Token::TK_BlockEntry:
1999 CurrentEntry = parseBlockNode();
2000 if (CurrentEntry == 0) { // An error occurred.
2006 case Token::TK_Error:
2010 } else if (SeqType == ST_Flow) {
2012 case Token::TK_FlowEntry:
2013 // Eat the flow entry and recurse.
2015 WasPreviousTokenFlowEntry = true;
2017 case Token::TK_FlowSequenceEnd:
2019 case Token::TK_Error:
2020 // Set this to end iterator.
2024 case Token::TK_StreamEnd:
2025 case Token::TK_DocumentEnd:
2026 case Token::TK_DocumentStart:
2027 setError("Could not find closing ]!", T);
2028 // Set this to end iterator.
2033 if (!WasPreviousTokenFlowEntry) {
2034 setError("Expected , between entries!", T);
2039 // Otherwise it must be a flow entry.
2040 CurrentEntry = parseBlockNode();
2041 if (!CurrentEntry) {
2044 WasPreviousTokenFlowEntry = false;
2050 Document::Document(Stream &S) : stream(S), Root(0) {
2051 // Tag maps starts with two default mappings.
2053 TagMap["!!"] = "tag:yaml.org,2002:";
2055 if (parseDirectives())
2056 expectToken(Token::TK_DocumentStart);
2057 Token &T = peekNext();
2058 if (T.Kind == Token::TK_DocumentStart)
2062 bool Document::skip() {
2063 if (stream.scanner->failed())
2068 Token &T = peekNext();
2069 if (T.Kind == Token::TK_StreamEnd)
2071 if (T.Kind == Token::TK_DocumentEnd) {
2078 Token &Document::peekNext() {
2079 return stream.scanner->peekNext();
2082 Token Document::getNext() {
2083 return stream.scanner->getNext();
2086 void Document::setError(const Twine &Message, Token &Location) const {
2087 stream.scanner->setError(Message, Location.Range.begin());
2090 bool Document::failed() const {
2091 return stream.scanner->failed();
2094 Node *Document::parseBlockNode() {
2095 Token T = peekNext();
2096 // Handle properties.
2101 case Token::TK_Alias:
2103 return new (NodeAllocator) AliasNode(stream.CurrentDoc, T.Range.substr(1));
2104 case Token::TK_Anchor:
2105 if (AnchorInfo.Kind == Token::TK_Anchor) {
2106 setError("Already encountered an anchor for this node!", T);
2109 AnchorInfo = getNext(); // Consume TK_Anchor.
2111 goto parse_property;
2113 if (TagInfo.Kind == Token::TK_Tag) {
2114 setError("Already encountered a tag for this node!", T);
2117 TagInfo = getNext(); // Consume TK_Tag.
2119 goto parse_property;
2125 case Token::TK_BlockEntry:
2126 // We got an unindented BlockEntry sequence. This is not terminated with
2128 // Don't eat the TK_BlockEntry, SequenceNode needs it.
2129 return new (NodeAllocator) SequenceNode( stream.CurrentDoc
2130 , AnchorInfo.Range.substr(1)
2132 , SequenceNode::ST_Indentless);
2133 case Token::TK_BlockSequenceStart:
2135 return new (NodeAllocator)
2136 SequenceNode( stream.CurrentDoc
2137 , AnchorInfo.Range.substr(1)
2139 , SequenceNode::ST_Block);
2140 case Token::TK_BlockMappingStart:
2142 return new (NodeAllocator)
2143 MappingNode( stream.CurrentDoc
2144 , AnchorInfo.Range.substr(1)
2146 , MappingNode::MT_Block);
2147 case Token::TK_FlowSequenceStart:
2149 return new (NodeAllocator)
2150 SequenceNode( stream.CurrentDoc
2151 , AnchorInfo.Range.substr(1)
2153 , SequenceNode::ST_Flow);
2154 case Token::TK_FlowMappingStart:
2156 return new (NodeAllocator)
2157 MappingNode( stream.CurrentDoc
2158 , AnchorInfo.Range.substr(1)
2160 , MappingNode::MT_Flow);
2161 case Token::TK_Scalar:
2163 return new (NodeAllocator)
2164 ScalarNode( stream.CurrentDoc
2165 , AnchorInfo.Range.substr(1)
2169 // Don't eat the TK_Key, KeyValueNode expects it.
2170 return new (NodeAllocator)
2171 MappingNode( stream.CurrentDoc
2172 , AnchorInfo.Range.substr(1)
2174 , MappingNode::MT_Inline);
2175 case Token::TK_DocumentStart:
2176 case Token::TK_DocumentEnd:
2177 case Token::TK_StreamEnd:
2179 // TODO: Properly handle tags. "[!!str ]" should resolve to !!str "", not
2181 return new (NodeAllocator) NullNode(stream.CurrentDoc);
2182 case Token::TK_Error:
2185 llvm_unreachable("Control flow shouldn't reach here.");
2189 bool Document::parseDirectives() {
2190 bool isDirective = false;
2192 Token T = peekNext();
2193 if (T.Kind == Token::TK_TagDirective) {
2194 parseTAGDirective();
2196 } else if (T.Kind == Token::TK_VersionDirective) {
2197 parseYAMLDirective();
2205 void Document::parseYAMLDirective() {
2206 getNext(); // Eat %YAML <version>
2209 void Document::parseTAGDirective() {
2210 Token Tag = getNext(); // %TAG <handle> <prefix>
2211 StringRef T = Tag.Range;
2213 T = T.substr(T.find_first_of(" \t")).ltrim(" \t");
2214 std::size_t HandleEnd = T.find_first_of(" \t");
2215 StringRef TagHandle = T.substr(0, HandleEnd);
2216 StringRef TagPrefix = T.substr(HandleEnd).ltrim(" \t");
2217 TagMap[TagHandle] = TagPrefix;
2220 bool Document::expectToken(int TK) {
2221 Token T = getNext();
2223 setError("Unexpected token", T);