1 //===--- YAMLParser.cpp - Simple YAML parser ------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements a YAML parser.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Support/YAMLParser.h"
15 #include "llvm/ADT/SmallString.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/ADT/Twine.h"
19 #include "llvm/ADT/ilist.h"
20 #include "llvm/ADT/ilist_node.h"
21 #include "llvm/Support/ErrorHandling.h"
22 #include "llvm/Support/MemoryBuffer.h"
23 #include "llvm/Support/SourceMgr.h"
24 #include "llvm/Support/raw_ostream.h"
29 enum UnicodeEncodingForm {
30 UEF_UTF32_LE, ///< UTF-32 Little Endian
31 UEF_UTF32_BE, ///< UTF-32 Big Endian
32 UEF_UTF16_LE, ///< UTF-16 Little Endian
33 UEF_UTF16_BE, ///< UTF-16 Big Endian
34 UEF_UTF8, ///< UTF-8 or ascii.
35 UEF_Unknown ///< Not a valid Unicode encoding.
38 /// EncodingInfo - Holds the encoding type and length of the byte order mark if
39 /// it exists. Length is in {0, 2, 3, 4}.
40 typedef std::pair<UnicodeEncodingForm, unsigned> EncodingInfo;
42 /// getUnicodeEncoding - Reads up to the first 4 bytes to determine the Unicode
43 /// encoding form of \a Input.
45 /// @param Input A string of length 0 or more.
46 /// @returns An EncodingInfo indicating the Unicode encoding form of the input
47 /// and how long the byte order mark is if one exists.
48 static EncodingInfo getUnicodeEncoding(StringRef Input) {
49 if (Input.size() == 0)
50 return std::make_pair(UEF_Unknown, 0);
52 switch (uint8_t(Input[0])) {
54 if (Input.size() >= 4) {
56 && uint8_t(Input[2]) == 0xFE
57 && uint8_t(Input[3]) == 0xFF)
58 return std::make_pair(UEF_UTF32_BE, 4);
59 if (Input[1] == 0 && Input[2] == 0 && Input[3] != 0)
60 return std::make_pair(UEF_UTF32_BE, 0);
63 if (Input.size() >= 2 && Input[1] != 0)
64 return std::make_pair(UEF_UTF16_BE, 0);
65 return std::make_pair(UEF_Unknown, 0);
67 if ( Input.size() >= 4
68 && uint8_t(Input[1]) == 0xFE
71 return std::make_pair(UEF_UTF32_LE, 4);
73 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFE)
74 return std::make_pair(UEF_UTF16_LE, 2);
75 return std::make_pair(UEF_Unknown, 0);
77 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFF)
78 return std::make_pair(UEF_UTF16_BE, 2);
79 return std::make_pair(UEF_Unknown, 0);
81 if ( Input.size() >= 3
82 && uint8_t(Input[1]) == 0xBB
83 && uint8_t(Input[2]) == 0xBF)
84 return std::make_pair(UEF_UTF8, 3);
85 return std::make_pair(UEF_Unknown, 0);
88 // It could still be utf-32 or utf-16.
89 if (Input.size() >= 4 && Input[1] == 0 && Input[2] == 0 && Input[3] == 0)
90 return std::make_pair(UEF_UTF32_LE, 0);
92 if (Input.size() >= 2 && Input[1] == 0)
93 return std::make_pair(UEF_UTF16_LE, 0);
95 return std::make_pair(UEF_UTF8, 0);
100 /// Pin the vtables to this file.
101 void Node::anchor() {}
102 void NullNode::anchor() {}
103 void ScalarNode::anchor() {}
104 void KeyValueNode::anchor() {}
105 void MappingNode::anchor() {}
106 void SequenceNode::anchor() {}
107 void AliasNode::anchor() {}
109 /// Token - A single YAML token.
110 struct Token : ilist_node<Token> {
112 TK_Error, // Uninitialized token.
121 TK_BlockSequenceStart,
122 TK_BlockMappingStart,
124 TK_FlowSequenceStart,
136 /// A string of length 0 or more whose begin() points to the logical location
137 /// of the token in the input.
140 Token() : Kind(TK_Error) {}
147 struct ilist_sentinel_traits<Token> {
148 Token *createSentinel() const {
151 static void destroySentinel(Token*) {}
153 Token *provideInitialHead() const { return createSentinel(); }
154 Token *ensureHead(Token*) const { return createSentinel(); }
155 static void noteHead(Token*, Token*) {}
158 mutable Token Sentinel;
162 struct ilist_node_traits<Token> {
163 Token *createNode(const Token &V) {
164 return new (Alloc.Allocate<Token>()) Token(V);
166 static void deleteNode(Token *V) {}
168 void addNodeToList(Token *) {}
169 void removeNodeFromList(Token *) {}
170 void transferNodesFromList(ilist_node_traits & /*SrcTraits*/,
171 ilist_iterator<Token> /*first*/,
172 ilist_iterator<Token> /*last*/) {}
174 BumpPtrAllocator Alloc;
178 typedef ilist<Token> TokenQueueT;
181 /// @brief This struct is used to track simple keys.
183 /// Simple keys are handled by creating an entry in SimpleKeys for each Token
184 /// which could legally be the start of a simple key. When peekNext is called,
185 /// if the Token To be returned is referenced by a SimpleKey, we continue
186 /// tokenizing until that potential simple key has either been found to not be
187 /// a simple key (we moved on to the next line or went further than 1024 chars).
188 /// Or when we run into a Value, and then insert a Key token (and possibly
189 /// others) before the SimpleKey's Tok.
191 TokenQueueT::iterator Tok;
197 bool operator ==(const SimpleKey &Other) {
198 return Tok == Other.Tok;
203 /// @brief The Unicode scalar value of a UTF-8 minimal well-formed code unit
204 /// subsequence and the subsequence's length in code units (uint8_t).
205 /// A length of 0 represents an error.
206 typedef std::pair<uint32_t, unsigned> UTF8Decoded;
208 static UTF8Decoded decodeUTF8(StringRef Range) {
209 StringRef::iterator Position= Range.begin();
210 StringRef::iterator End = Range.end();
211 // 1 byte: [0x00, 0x7f]
212 // Bit pattern: 0xxxxxxx
213 if ((*Position & 0x80) == 0) {
214 return std::make_pair(*Position, 1);
216 // 2 bytes: [0x80, 0x7ff]
217 // Bit pattern: 110xxxxx 10xxxxxx
218 if (Position + 1 != End &&
219 ((*Position & 0xE0) == 0xC0) &&
220 ((*(Position + 1) & 0xC0) == 0x80)) {
221 uint32_t codepoint = ((*Position & 0x1F) << 6) |
222 (*(Position + 1) & 0x3F);
223 if (codepoint >= 0x80)
224 return std::make_pair(codepoint, 2);
226 // 3 bytes: [0x8000, 0xffff]
227 // Bit pattern: 1110xxxx 10xxxxxx 10xxxxxx
228 if (Position + 2 != End &&
229 ((*Position & 0xF0) == 0xE0) &&
230 ((*(Position + 1) & 0xC0) == 0x80) &&
231 ((*(Position + 2) & 0xC0) == 0x80)) {
232 uint32_t codepoint = ((*Position & 0x0F) << 12) |
233 ((*(Position + 1) & 0x3F) << 6) |
234 (*(Position + 2) & 0x3F);
235 // Codepoints between 0xD800 and 0xDFFF are invalid, as
236 // they are high / low surrogate halves used by UTF-16.
237 if (codepoint >= 0x800 &&
238 (codepoint < 0xD800 || codepoint > 0xDFFF))
239 return std::make_pair(codepoint, 3);
241 // 4 bytes: [0x10000, 0x10FFFF]
242 // Bit pattern: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
243 if (Position + 3 != End &&
244 ((*Position & 0xF8) == 0xF0) &&
245 ((*(Position + 1) & 0xC0) == 0x80) &&
246 ((*(Position + 2) & 0xC0) == 0x80) &&
247 ((*(Position + 3) & 0xC0) == 0x80)) {
248 uint32_t codepoint = ((*Position & 0x07) << 18) |
249 ((*(Position + 1) & 0x3F) << 12) |
250 ((*(Position + 2) & 0x3F) << 6) |
251 (*(Position + 3) & 0x3F);
252 if (codepoint >= 0x10000 && codepoint <= 0x10FFFF)
253 return std::make_pair(codepoint, 4);
255 return std::make_pair(0, 0);
260 /// @brief Scans YAML tokens from a MemoryBuffer.
263 Scanner(StringRef Input, SourceMgr &SM, bool ShowColors = true);
264 Scanner(MemoryBufferRef Buffer, SourceMgr &SM_, bool ShowColors = true);
266 /// @brief Parse the next token and return it without popping it.
269 /// @brief Parse the next token and pop it from the queue.
272 void printError(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Message,
273 ArrayRef<SMRange> Ranges = None) {
274 SM.PrintMessage(Loc, Kind, Message, Ranges, /* FixIts= */ None, ShowColors);
277 void setError(const Twine &Message, StringRef::iterator Position) {
281 // Don't print out more errors after the first one we encounter. The rest
282 // are just the result of the first, and have no meaning.
284 printError(SMLoc::getFromPointer(Current), SourceMgr::DK_Error, Message);
288 void setError(const Twine &Message) {
289 setError(Message, Current);
292 /// @brief Returns true if an error occurred while parsing.
298 void init(MemoryBufferRef Buffer);
300 StringRef currentInput() {
301 return StringRef(Current, End - Current);
304 /// @brief Decode a UTF-8 minimal well-formed code unit subsequence starting
307 /// If the UTF-8 code units starting at Position do not form a well-formed
308 /// code unit subsequence, then the Unicode scalar value is 0, and the length
310 UTF8Decoded decodeUTF8(StringRef::iterator Position) {
311 return ::decodeUTF8(StringRef(Position, End - Position));
314 // The following functions are based on the gramar rules in the YAML spec. The
315 // style of the function names it meant to closely match how they are written
316 // in the spec. The number within the [] is the number of the grammar rule in
319 // See 4.2 [Production Naming Conventions] for the meaning of the prefixes.
322 // A production starting and ending with a special character.
324 // A production matching a single line break.
326 // A production starting and ending with a non-break character.
328 // A production starting and ending with a white space character.
330 // A production starting and ending with a non-space character.
332 // A production matching complete line(s).
334 /// @brief Skip a single nb-char[27] starting at Position.
336 /// A nb-char is 0x9 | [0x20-0x7E] | 0x85 | [0xA0-0xD7FF] | [0xE000-0xFEFE]
337 /// | [0xFF00-0xFFFD] | [0x10000-0x10FFFF]
339 /// @returns The code unit after the nb-char, or Position if it's not an
341 StringRef::iterator skip_nb_char(StringRef::iterator Position);
343 /// @brief Skip a single b-break[28] starting at Position.
345 /// A b-break is 0xD 0xA | 0xD | 0xA
347 /// @returns The code unit after the b-break, or Position if it's not a
349 StringRef::iterator skip_b_break(StringRef::iterator Position);
351 /// @brief Skip a single s-white[33] starting at Position.
353 /// A s-white is 0x20 | 0x9
355 /// @returns The code unit after the s-white, or Position if it's not a
357 StringRef::iterator skip_s_white(StringRef::iterator Position);
359 /// @brief Skip a single ns-char[34] starting at Position.
361 /// A ns-char is nb-char - s-white
363 /// @returns The code unit after the ns-char, or Position if it's not a
365 StringRef::iterator skip_ns_char(StringRef::iterator Position);
367 typedef StringRef::iterator (Scanner::*SkipWhileFunc)(StringRef::iterator);
368 /// @brief Skip minimal well-formed code unit subsequences until Func
369 /// returns its input.
371 /// @returns The code unit after the last minimal well-formed code unit
372 /// subsequence that Func accepted.
373 StringRef::iterator skip_while( SkipWhileFunc Func
374 , StringRef::iterator Position);
376 /// @brief Scan ns-uri-char[39]s starting at Cur.
378 /// This updates Cur and Column while scanning.
380 /// @returns A StringRef starting at Cur which covers the longest contiguous
381 /// sequence of ns-uri-char.
382 StringRef scan_ns_uri_char();
384 /// @brief Consume a minimal well-formed code unit subsequence starting at
385 /// \a Cur. Return false if it is not the same Unicode scalar value as
386 /// \a Expected. This updates \a Column.
387 bool consume(uint32_t Expected);
389 /// @brief Skip \a Distance UTF-8 code units. Updates \a Cur and \a Column.
390 void skip(uint32_t Distance);
392 /// @brief Return true if the minimal well-formed code unit subsequence at
393 /// Pos is whitespace or a new line
394 bool isBlankOrBreak(StringRef::iterator Position);
396 /// @brief If IsSimpleKeyAllowed, create and push_back a new SimpleKey.
397 void saveSimpleKeyCandidate( TokenQueueT::iterator Tok
401 /// @brief Remove simple keys that can no longer be valid simple keys.
403 /// Invalid simple keys are not on the current line or are further than 1024
405 void removeStaleSimpleKeyCandidates();
407 /// @brief Remove all simple keys on FlowLevel \a Level.
408 void removeSimpleKeyCandidatesOnFlowLevel(unsigned Level);
410 /// @brief Unroll indentation in \a Indents back to \a Col. Creates BlockEnd
411 /// tokens if needed.
412 bool unrollIndent(int ToColumn);
414 /// @brief Increase indent to \a Col. Creates \a Kind token at \a InsertPoint
416 bool rollIndent( int ToColumn
417 , Token::TokenKind Kind
418 , TokenQueueT::iterator InsertPoint);
420 /// @brief Skip a single-line comment when the comment starts at the current
421 /// position of the scanner.
424 /// @brief Skip whitespace and comments until the start of the next token.
425 void scanToNextToken();
427 /// @brief Must be the first token generated.
428 bool scanStreamStart();
430 /// @brief Generate tokens needed to close out the stream.
431 bool scanStreamEnd();
433 /// @brief Scan a %BLAH directive.
434 bool scanDirective();
436 /// @brief Scan a ... or ---.
437 bool scanDocumentIndicator(bool IsStart);
439 /// @brief Scan a [ or { and generate the proper flow collection start token.
440 bool scanFlowCollectionStart(bool IsSequence);
442 /// @brief Scan a ] or } and generate the proper flow collection end token.
443 bool scanFlowCollectionEnd(bool IsSequence);
445 /// @brief Scan the , that separates entries in a flow collection.
446 bool scanFlowEntry();
448 /// @brief Scan the - that starts block sequence entries.
449 bool scanBlockEntry();
451 /// @brief Scan an explicit ? indicating a key.
454 /// @brief Scan an explicit : indicating a value.
457 /// @brief Scan a quoted scalar.
458 bool scanFlowScalar(bool IsDoubleQuoted);
460 /// @brief Scan an unquoted scalar.
461 bool scanPlainScalar();
463 /// @brief Scan an Alias or Anchor starting with * or &.
464 bool scanAliasOrAnchor(bool IsAlias);
466 /// @brief Scan a block scalar starting with | or >.
467 bool scanBlockScalar(bool IsLiteral);
469 /// @brief Scan a tag of the form !stuff.
472 /// @brief Dispatch to the next scanning function based on \a *Cur.
473 bool fetchMoreTokens();
475 /// @brief The SourceMgr used for diagnostics and buffer management.
478 /// @brief The original input.
479 MemoryBufferRef InputBuffer;
481 /// @brief The current position of the scanner.
482 StringRef::iterator Current;
484 /// @brief The end of the input (one past the last character).
485 StringRef::iterator End;
487 /// @brief Current YAML indentation level in spaces.
490 /// @brief Current column number in Unicode code points.
493 /// @brief Current line number.
496 /// @brief How deep we are in flow style containers. 0 Means at block level.
499 /// @brief Are we at the start of the stream?
500 bool IsStartOfStream;
502 /// @brief Can the next token be the start of a simple key?
503 bool IsSimpleKeyAllowed;
505 /// @brief True if an error has occurred.
508 /// @brief Should colors be used when printing out the diagnostic messages?
511 /// @brief Queue of tokens. This is required to queue up tokens while looking
512 /// for the end of a simple key. And for cases where a single character
513 /// can produce multiple tokens (e.g. BlockEnd).
514 TokenQueueT TokenQueue;
516 /// @brief Indentation levels.
517 SmallVector<int, 4> Indents;
519 /// @brief Potential simple keys.
520 SmallVector<SimpleKey, 4> SimpleKeys;
523 } // end namespace yaml
524 } // end namespace llvm
526 /// encodeUTF8 - Encode \a UnicodeScalarValue in UTF-8 and append it to result.
527 static void encodeUTF8( uint32_t UnicodeScalarValue
528 , SmallVectorImpl<char> &Result) {
529 if (UnicodeScalarValue <= 0x7F) {
530 Result.push_back(UnicodeScalarValue & 0x7F);
531 } else if (UnicodeScalarValue <= 0x7FF) {
532 uint8_t FirstByte = 0xC0 | ((UnicodeScalarValue & 0x7C0) >> 6);
533 uint8_t SecondByte = 0x80 | (UnicodeScalarValue & 0x3F);
534 Result.push_back(FirstByte);
535 Result.push_back(SecondByte);
536 } else if (UnicodeScalarValue <= 0xFFFF) {
537 uint8_t FirstByte = 0xE0 | ((UnicodeScalarValue & 0xF000) >> 12);
538 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
539 uint8_t ThirdByte = 0x80 | (UnicodeScalarValue & 0x3F);
540 Result.push_back(FirstByte);
541 Result.push_back(SecondByte);
542 Result.push_back(ThirdByte);
543 } else if (UnicodeScalarValue <= 0x10FFFF) {
544 uint8_t FirstByte = 0xF0 | ((UnicodeScalarValue & 0x1F0000) >> 18);
545 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0x3F000) >> 12);
546 uint8_t ThirdByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
547 uint8_t FourthByte = 0x80 | (UnicodeScalarValue & 0x3F);
548 Result.push_back(FirstByte);
549 Result.push_back(SecondByte);
550 Result.push_back(ThirdByte);
551 Result.push_back(FourthByte);
555 bool yaml::dumpTokens(StringRef Input, raw_ostream &OS) {
557 Scanner scanner(Input, SM);
559 Token T = scanner.getNext();
561 case Token::TK_StreamStart:
562 OS << "Stream-Start: ";
564 case Token::TK_StreamEnd:
565 OS << "Stream-End: ";
567 case Token::TK_VersionDirective:
568 OS << "Version-Directive: ";
570 case Token::TK_TagDirective:
571 OS << "Tag-Directive: ";
573 case Token::TK_DocumentStart:
574 OS << "Document-Start: ";
576 case Token::TK_DocumentEnd:
577 OS << "Document-End: ";
579 case Token::TK_BlockEntry:
580 OS << "Block-Entry: ";
582 case Token::TK_BlockEnd:
585 case Token::TK_BlockSequenceStart:
586 OS << "Block-Sequence-Start: ";
588 case Token::TK_BlockMappingStart:
589 OS << "Block-Mapping-Start: ";
591 case Token::TK_FlowEntry:
592 OS << "Flow-Entry: ";
594 case Token::TK_FlowSequenceStart:
595 OS << "Flow-Sequence-Start: ";
597 case Token::TK_FlowSequenceEnd:
598 OS << "Flow-Sequence-End: ";
600 case Token::TK_FlowMappingStart:
601 OS << "Flow-Mapping-Start: ";
603 case Token::TK_FlowMappingEnd:
604 OS << "Flow-Mapping-End: ";
609 case Token::TK_Value:
612 case Token::TK_Scalar:
615 case Token::TK_Alias:
618 case Token::TK_Anchor:
624 case Token::TK_Error:
627 OS << T.Range << "\n";
628 if (T.Kind == Token::TK_StreamEnd)
630 else if (T.Kind == Token::TK_Error)
636 bool yaml::scanTokens(StringRef Input) {
638 llvm::yaml::Scanner scanner(Input, SM);
640 llvm::yaml::Token T = scanner.getNext();
641 if (T.Kind == Token::TK_StreamEnd)
643 else if (T.Kind == Token::TK_Error)
649 std::string yaml::escape(StringRef Input) {
650 std::string EscapedInput;
651 for (StringRef::iterator i = Input.begin(), e = Input.end(); i != e; ++i) {
653 EscapedInput += "\\\\";
655 EscapedInput += "\\\"";
657 EscapedInput += "\\0";
659 EscapedInput += "\\a";
661 EscapedInput += "\\b";
663 EscapedInput += "\\t";
665 EscapedInput += "\\n";
667 EscapedInput += "\\v";
669 EscapedInput += "\\f";
671 EscapedInput += "\\r";
673 EscapedInput += "\\e";
674 else if ((unsigned char)*i < 0x20) { // Control characters not handled above.
675 std::string HexStr = utohexstr(*i);
676 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
677 } else if (*i & 0x80) { // UTF-8 multiple code unit subsequence.
678 UTF8Decoded UnicodeScalarValue
679 = decodeUTF8(StringRef(i, Input.end() - i));
680 if (UnicodeScalarValue.second == 0) {
681 // Found invalid char.
683 encodeUTF8(0xFFFD, Val);
684 EscapedInput.insert(EscapedInput.end(), Val.begin(), Val.end());
685 // FIXME: Error reporting.
688 if (UnicodeScalarValue.first == 0x85)
689 EscapedInput += "\\N";
690 else if (UnicodeScalarValue.first == 0xA0)
691 EscapedInput += "\\_";
692 else if (UnicodeScalarValue.first == 0x2028)
693 EscapedInput += "\\L";
694 else if (UnicodeScalarValue.first == 0x2029)
695 EscapedInput += "\\P";
697 std::string HexStr = utohexstr(UnicodeScalarValue.first);
698 if (HexStr.size() <= 2)
699 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
700 else if (HexStr.size() <= 4)
701 EscapedInput += "\\u" + std::string(4 - HexStr.size(), '0') + HexStr;
702 else if (HexStr.size() <= 8)
703 EscapedInput += "\\U" + std::string(8 - HexStr.size(), '0') + HexStr;
705 i += UnicodeScalarValue.second - 1;
707 EscapedInput.push_back(*i);
712 Scanner::Scanner(StringRef Input, SourceMgr &sm, bool ShowColors)
713 : SM(sm), ShowColors(ShowColors) {
714 init(MemoryBufferRef(Input, "YAML"));
717 Scanner::Scanner(MemoryBufferRef Buffer, SourceMgr &SM_, bool ShowColors)
718 : SM(SM_), ShowColors(ShowColors) {
722 void Scanner::init(MemoryBufferRef Buffer) {
723 InputBuffer = Buffer;
724 Current = InputBuffer.getBufferStart();
725 End = InputBuffer.getBufferEnd();
730 IsStartOfStream = true;
731 IsSimpleKeyAllowed = true;
733 std::unique_ptr<MemoryBuffer> InputBufferOwner =
734 MemoryBuffer::getMemBuffer(Buffer);
735 SM.AddNewSourceBuffer(std::move(InputBufferOwner), SMLoc());
738 Token &Scanner::peekNext() {
739 // If the current token is a possible simple key, keep parsing until we
741 bool NeedMore = false;
743 if (TokenQueue.empty() || NeedMore) {
744 if (!fetchMoreTokens()) {
746 TokenQueue.push_back(Token());
747 return TokenQueue.front();
750 assert(!TokenQueue.empty() &&
751 "fetchMoreTokens lied about getting tokens!");
753 removeStaleSimpleKeyCandidates();
755 SK.Tok = TokenQueue.front();
756 if (std::find(SimpleKeys.begin(), SimpleKeys.end(), SK)
762 return TokenQueue.front();
765 Token Scanner::getNext() {
766 Token Ret = peekNext();
767 // TokenQueue can be empty if there was an error getting the next token.
768 if (!TokenQueue.empty())
769 TokenQueue.pop_front();
771 // There cannot be any referenced Token's if the TokenQueue is empty. So do a
772 // quick deallocation of them all.
773 if (TokenQueue.empty()) {
774 TokenQueue.Alloc.Reset();
780 StringRef::iterator Scanner::skip_nb_char(StringRef::iterator Position) {
783 // Check 7 bit c-printable - b-char.
784 if ( *Position == 0x09
785 || (*Position >= 0x20 && *Position <= 0x7E))
788 // Check for valid UTF-8.
789 if (uint8_t(*Position) & 0x80) {
790 UTF8Decoded u8d = decodeUTF8(Position);
792 && u8d.first != 0xFEFF
793 && ( u8d.first == 0x85
794 || ( u8d.first >= 0xA0
795 && u8d.first <= 0xD7FF)
796 || ( u8d.first >= 0xE000
797 && u8d.first <= 0xFFFD)
798 || ( u8d.first >= 0x10000
799 && u8d.first <= 0x10FFFF)))
800 return Position + u8d.second;
805 StringRef::iterator Scanner::skip_b_break(StringRef::iterator Position) {
808 if (*Position == 0x0D) {
809 if (Position + 1 != End && *(Position + 1) == 0x0A)
814 if (*Position == 0x0A)
820 StringRef::iterator Scanner::skip_s_white(StringRef::iterator Position) {
823 if (*Position == ' ' || *Position == '\t')
828 StringRef::iterator Scanner::skip_ns_char(StringRef::iterator Position) {
831 if (*Position == ' ' || *Position == '\t')
833 return skip_nb_char(Position);
836 StringRef::iterator Scanner::skip_while( SkipWhileFunc Func
837 , StringRef::iterator Position) {
839 StringRef::iterator i = (this->*Func)(Position);
847 static bool is_ns_hex_digit(const char C) {
848 return (C >= '0' && C <= '9')
849 || (C >= 'a' && C <= 'z')
850 || (C >= 'A' && C <= 'Z');
853 static bool is_ns_word_char(const char C) {
855 || (C >= 'a' && C <= 'z')
856 || (C >= 'A' && C <= 'Z');
859 StringRef Scanner::scan_ns_uri_char() {
860 StringRef::iterator Start = Current;
864 if (( *Current == '%'
866 && is_ns_hex_digit(*(Current + 1))
867 && is_ns_hex_digit(*(Current + 2)))
868 || is_ns_word_char(*Current)
869 || StringRef(Current, 1).find_first_of("#;/?:@&=+$,_.!~*'()[]")
870 != StringRef::npos) {
876 return StringRef(Start, Current - Start);
879 bool Scanner::consume(uint32_t Expected) {
880 if (Expected >= 0x80)
881 report_fatal_error("Not dealing with this yet");
884 if (uint8_t(*Current) >= 0x80)
885 report_fatal_error("Not dealing with this yet");
886 if (uint8_t(*Current) == Expected) {
894 void Scanner::skip(uint32_t Distance) {
897 assert(Current <= End && "Skipped past the end");
900 bool Scanner::isBlankOrBreak(StringRef::iterator Position) {
903 if ( *Position == ' ' || *Position == '\t'
904 || *Position == '\r' || *Position == '\n')
909 void Scanner::saveSimpleKeyCandidate( TokenQueueT::iterator Tok
912 if (IsSimpleKeyAllowed) {
916 SK.Column = AtColumn;
917 SK.IsRequired = IsRequired;
918 SK.FlowLevel = FlowLevel;
919 SimpleKeys.push_back(SK);
923 void Scanner::removeStaleSimpleKeyCandidates() {
924 for (SmallVectorImpl<SimpleKey>::iterator i = SimpleKeys.begin();
925 i != SimpleKeys.end();) {
926 if (i->Line != Line || i->Column + 1024 < Column) {
928 setError( "Could not find expected : for simple key"
929 , i->Tok->Range.begin());
930 i = SimpleKeys.erase(i);
936 void Scanner::removeSimpleKeyCandidatesOnFlowLevel(unsigned Level) {
937 if (!SimpleKeys.empty() && (SimpleKeys.end() - 1)->FlowLevel == Level)
938 SimpleKeys.pop_back();
941 bool Scanner::unrollIndent(int ToColumn) {
943 // Indentation is ignored in flow.
947 while (Indent > ToColumn) {
948 T.Kind = Token::TK_BlockEnd;
949 T.Range = StringRef(Current, 1);
950 TokenQueue.push_back(T);
951 Indent = Indents.pop_back_val();
957 bool Scanner::rollIndent( int ToColumn
958 , Token::TokenKind Kind
959 , TokenQueueT::iterator InsertPoint) {
962 if (Indent < ToColumn) {
963 Indents.push_back(Indent);
968 T.Range = StringRef(Current, 0);
969 TokenQueue.insert(InsertPoint, T);
974 void Scanner::skipComment() {
978 // This may skip more than one byte, thus Column is only incremented
980 StringRef::iterator I = skip_nb_char(Current);
988 void Scanner::scanToNextToken() {
990 while (*Current == ' ' || *Current == '\t') {
997 StringRef::iterator i = skip_b_break(Current);
1003 // New lines may start a simple key.
1005 IsSimpleKeyAllowed = true;
1009 bool Scanner::scanStreamStart() {
1010 IsStartOfStream = false;
1012 EncodingInfo EI = getUnicodeEncoding(currentInput());
1015 T.Kind = Token::TK_StreamStart;
1016 T.Range = StringRef(Current, EI.second);
1017 TokenQueue.push_back(T);
1018 Current += EI.second;
1022 bool Scanner::scanStreamEnd() {
1023 // Force an ending new line if one isn't present.
1031 IsSimpleKeyAllowed = false;
1034 T.Kind = Token::TK_StreamEnd;
1035 T.Range = StringRef(Current, 0);
1036 TokenQueue.push_back(T);
1040 bool Scanner::scanDirective() {
1041 // Reset the indentation level.
1044 IsSimpleKeyAllowed = false;
1046 StringRef::iterator Start = Current;
1048 StringRef::iterator NameStart = Current;
1049 Current = skip_while(&Scanner::skip_ns_char, Current);
1050 StringRef Name(NameStart, Current - NameStart);
1051 Current = skip_while(&Scanner::skip_s_white, Current);
1054 if (Name == "YAML") {
1055 Current = skip_while(&Scanner::skip_ns_char, Current);
1056 T.Kind = Token::TK_VersionDirective;
1057 T.Range = StringRef(Start, Current - Start);
1058 TokenQueue.push_back(T);
1060 } else if(Name == "TAG") {
1061 Current = skip_while(&Scanner::skip_ns_char, Current);
1062 Current = skip_while(&Scanner::skip_s_white, Current);
1063 Current = skip_while(&Scanner::skip_ns_char, Current);
1064 T.Kind = Token::TK_TagDirective;
1065 T.Range = StringRef(Start, Current - Start);
1066 TokenQueue.push_back(T);
1072 bool Scanner::scanDocumentIndicator(bool IsStart) {
1075 IsSimpleKeyAllowed = false;
1078 T.Kind = IsStart ? Token::TK_DocumentStart : Token::TK_DocumentEnd;
1079 T.Range = StringRef(Current, 3);
1081 TokenQueue.push_back(T);
1085 bool Scanner::scanFlowCollectionStart(bool IsSequence) {
1087 T.Kind = IsSequence ? Token::TK_FlowSequenceStart
1088 : Token::TK_FlowMappingStart;
1089 T.Range = StringRef(Current, 1);
1091 TokenQueue.push_back(T);
1093 // [ and { may begin a simple key.
1094 saveSimpleKeyCandidate(TokenQueue.back(), Column - 1, false);
1096 // And may also be followed by a simple key.
1097 IsSimpleKeyAllowed = true;
1102 bool Scanner::scanFlowCollectionEnd(bool IsSequence) {
1103 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1104 IsSimpleKeyAllowed = false;
1106 T.Kind = IsSequence ? Token::TK_FlowSequenceEnd
1107 : Token::TK_FlowMappingEnd;
1108 T.Range = StringRef(Current, 1);
1110 TokenQueue.push_back(T);
1116 bool Scanner::scanFlowEntry() {
1117 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1118 IsSimpleKeyAllowed = true;
1120 T.Kind = Token::TK_FlowEntry;
1121 T.Range = StringRef(Current, 1);
1123 TokenQueue.push_back(T);
1127 bool Scanner::scanBlockEntry() {
1128 rollIndent(Column, Token::TK_BlockSequenceStart, TokenQueue.end());
1129 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1130 IsSimpleKeyAllowed = true;
1132 T.Kind = Token::TK_BlockEntry;
1133 T.Range = StringRef(Current, 1);
1135 TokenQueue.push_back(T);
1139 bool Scanner::scanKey() {
1141 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1143 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1144 IsSimpleKeyAllowed = !FlowLevel;
1147 T.Kind = Token::TK_Key;
1148 T.Range = StringRef(Current, 1);
1150 TokenQueue.push_back(T);
1154 bool Scanner::scanValue() {
1155 // If the previous token could have been a simple key, insert the key token
1156 // into the token queue.
1157 if (!SimpleKeys.empty()) {
1158 SimpleKey SK = SimpleKeys.pop_back_val();
1160 T.Kind = Token::TK_Key;
1161 T.Range = SK.Tok->Range;
1162 TokenQueueT::iterator i, e;
1163 for (i = TokenQueue.begin(), e = TokenQueue.end(); i != e; ++i) {
1167 assert(i != e && "SimpleKey not in token queue!");
1168 i = TokenQueue.insert(i, T);
1170 // We may also need to add a Block-Mapping-Start token.
1171 rollIndent(SK.Column, Token::TK_BlockMappingStart, i);
1173 IsSimpleKeyAllowed = false;
1176 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1177 IsSimpleKeyAllowed = !FlowLevel;
1181 T.Kind = Token::TK_Value;
1182 T.Range = StringRef(Current, 1);
1184 TokenQueue.push_back(T);
1188 // Forbidding inlining improves performance by roughly 20%.
1189 // FIXME: Remove once llvm optimizes this to the faster version without hints.
1190 LLVM_ATTRIBUTE_NOINLINE static bool
1191 wasEscaped(StringRef::iterator First, StringRef::iterator Position);
1193 // Returns whether a character at 'Position' was escaped with a leading '\'.
1194 // 'First' specifies the position of the first character in the string.
1195 static bool wasEscaped(StringRef::iterator First,
1196 StringRef::iterator Position) {
1197 assert(Position - 1 >= First);
1198 StringRef::iterator I = Position - 1;
1199 // We calculate the number of consecutive '\'s before the current position
1200 // by iterating backwards through our string.
1201 while (I >= First && *I == '\\') --I;
1202 // (Position - 1 - I) now contains the number of '\'s before the current
1203 // position. If it is odd, the character at 'Position' was escaped.
1204 return (Position - 1 - I) % 2 == 1;
1207 bool Scanner::scanFlowScalar(bool IsDoubleQuoted) {
1208 StringRef::iterator Start = Current;
1209 unsigned ColStart = Column;
1210 if (IsDoubleQuoted) {
1213 while (Current != End && *Current != '"')
1215 // Repeat until the previous character was not a '\' or was an escaped
1217 } while ( Current != End
1218 && *(Current - 1) == '\\'
1219 && wasEscaped(Start + 1, Current));
1223 // Skip a ' followed by another '.
1224 if (Current + 1 < End && *Current == '\'' && *(Current + 1) == '\'') {
1227 } else if (*Current == '\'')
1229 StringRef::iterator i = skip_nb_char(Current);
1231 i = skip_b_break(Current);
1246 if (Current == End) {
1247 setError("Expected quote at end of scalar", Current);
1251 skip(1); // Skip ending quote.
1253 T.Kind = Token::TK_Scalar;
1254 T.Range = StringRef(Start, Current - Start);
1255 TokenQueue.push_back(T);
1257 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1259 IsSimpleKeyAllowed = false;
1264 bool Scanner::scanPlainScalar() {
1265 StringRef::iterator Start = Current;
1266 unsigned ColStart = Column;
1267 unsigned LeadingBlanks = 0;
1268 assert(Indent >= -1 && "Indent must be >= -1 !");
1269 unsigned indent = static_cast<unsigned>(Indent + 1);
1271 if (*Current == '#')
1274 while (!isBlankOrBreak(Current)) {
1275 if ( FlowLevel && *Current == ':'
1276 && !(isBlankOrBreak(Current + 1) || *(Current + 1) == ',')) {
1277 setError("Found unexpected ':' while scanning a plain scalar", Current);
1281 // Check for the end of the plain scalar.
1282 if ( (*Current == ':' && isBlankOrBreak(Current + 1))
1284 && (StringRef(Current, 1).find_first_of(",:?[]{}")
1285 != StringRef::npos)))
1288 StringRef::iterator i = skip_nb_char(Current);
1295 // Are we at the end?
1296 if (!isBlankOrBreak(Current))
1300 StringRef::iterator Tmp = Current;
1301 while (isBlankOrBreak(Tmp)) {
1302 StringRef::iterator i = skip_s_white(Tmp);
1304 if (LeadingBlanks && (Column < indent) && *Tmp == '\t') {
1305 setError("Found invalid tab character in indentation", Tmp);
1311 i = skip_b_break(Tmp);
1320 if (!FlowLevel && Column < indent)
1325 if (Start == Current) {
1326 setError("Got empty plain scalar", Start);
1330 T.Kind = Token::TK_Scalar;
1331 T.Range = StringRef(Start, Current - Start);
1332 TokenQueue.push_back(T);
1334 // Plain scalars can be simple keys.
1335 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1337 IsSimpleKeyAllowed = false;
1342 bool Scanner::scanAliasOrAnchor(bool IsAlias) {
1343 StringRef::iterator Start = Current;
1344 unsigned ColStart = Column;
1347 if ( *Current == '[' || *Current == ']'
1348 || *Current == '{' || *Current == '}'
1352 StringRef::iterator i = skip_ns_char(Current);
1359 if (Start == Current) {
1360 setError("Got empty alias or anchor", Start);
1365 T.Kind = IsAlias ? Token::TK_Alias : Token::TK_Anchor;
1366 T.Range = StringRef(Start, Current - Start);
1367 TokenQueue.push_back(T);
1369 // Alias and anchors can be simple keys.
1370 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1372 IsSimpleKeyAllowed = false;
1377 bool Scanner::scanBlockScalar(bool IsLiteral) {
1378 StringRef::iterator Start = Current;
1379 skip(1); // Eat | or >
1381 StringRef::iterator i = skip_nb_char(Current);
1385 i = skip_b_break(Current);
1387 // We got a line break.
1393 // There was an error, which should already have been printed out.
1401 if (Start == Current) {
1402 setError("Got empty block scalar", Start);
1407 T.Kind = Token::TK_Scalar;
1408 T.Range = StringRef(Start, Current - Start);
1409 TokenQueue.push_back(T);
1413 bool Scanner::scanTag() {
1414 StringRef::iterator Start = Current;
1415 unsigned ColStart = Column;
1417 if (Current == End || isBlankOrBreak(Current)); // An empty tag.
1418 else if (*Current == '<') {
1424 // FIXME: Actually parse the c-ns-shorthand-tag rule.
1425 Current = skip_while(&Scanner::skip_ns_char, Current);
1429 T.Kind = Token::TK_Tag;
1430 T.Range = StringRef(Start, Current - Start);
1431 TokenQueue.push_back(T);
1433 // Tags can be simple keys.
1434 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1436 IsSimpleKeyAllowed = false;
1441 bool Scanner::fetchMoreTokens() {
1442 if (IsStartOfStream)
1443 return scanStreamStart();
1448 return scanStreamEnd();
1450 removeStaleSimpleKeyCandidates();
1452 unrollIndent(Column);
1454 if (Column == 0 && *Current == '%')
1455 return scanDirective();
1457 if (Column == 0 && Current + 4 <= End
1459 && *(Current + 1) == '-'
1460 && *(Current + 2) == '-'
1461 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1462 return scanDocumentIndicator(true);
1464 if (Column == 0 && Current + 4 <= End
1466 && *(Current + 1) == '.'
1467 && *(Current + 2) == '.'
1468 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1469 return scanDocumentIndicator(false);
1471 if (*Current == '[')
1472 return scanFlowCollectionStart(true);
1474 if (*Current == '{')
1475 return scanFlowCollectionStart(false);
1477 if (*Current == ']')
1478 return scanFlowCollectionEnd(true);
1480 if (*Current == '}')
1481 return scanFlowCollectionEnd(false);
1483 if (*Current == ',')
1484 return scanFlowEntry();
1486 if (*Current == '-' && isBlankOrBreak(Current + 1))
1487 return scanBlockEntry();
1489 if (*Current == '?' && (FlowLevel || isBlankOrBreak(Current + 1)))
1492 if (*Current == ':' && (FlowLevel || isBlankOrBreak(Current + 1)))
1495 if (*Current == '*')
1496 return scanAliasOrAnchor(true);
1498 if (*Current == '&')
1499 return scanAliasOrAnchor(false);
1501 if (*Current == '!')
1504 if (*Current == '|' && !FlowLevel)
1505 return scanBlockScalar(true);
1507 if (*Current == '>' && !FlowLevel)
1508 return scanBlockScalar(false);
1510 if (*Current == '\'')
1511 return scanFlowScalar(false);
1513 if (*Current == '"')
1514 return scanFlowScalar(true);
1516 // Get a plain scalar.
1517 StringRef FirstChar(Current, 1);
1518 if (!(isBlankOrBreak(Current)
1519 || FirstChar.find_first_of("-?:,[]{}#&*!|>'\"%@`") != StringRef::npos)
1520 || (*Current == '-' && !isBlankOrBreak(Current + 1))
1521 || (!FlowLevel && (*Current == '?' || *Current == ':')
1522 && isBlankOrBreak(Current + 1))
1523 || (!FlowLevel && *Current == ':'
1524 && Current + 2 < End
1525 && *(Current + 1) == ':'
1526 && !isBlankOrBreak(Current + 2)))
1527 return scanPlainScalar();
1529 setError("Unrecognized character while tokenizing.");
1533 Stream::Stream(StringRef Input, SourceMgr &SM, bool ShowColors)
1534 : scanner(new Scanner(Input, SM, ShowColors)), CurrentDoc() {}
1536 Stream::Stream(MemoryBufferRef InputBuffer, SourceMgr &SM, bool ShowColors)
1537 : scanner(new Scanner(InputBuffer, SM, ShowColors)), CurrentDoc() {}
1539 Stream::~Stream() {}
1541 bool Stream::failed() { return scanner->failed(); }
1543 void Stream::printError(Node *N, const Twine &Msg) {
1544 scanner->printError( N->getSourceRange().Start
1545 , SourceMgr::DK_Error
1547 , N->getSourceRange());
1550 document_iterator Stream::begin() {
1552 report_fatal_error("Can only iterate over the stream once");
1554 // Skip Stream-Start.
1557 CurrentDoc.reset(new Document(*this));
1558 return document_iterator(CurrentDoc);
1561 document_iterator Stream::end() {
1562 return document_iterator();
1565 void Stream::skip() {
1566 for (document_iterator i = begin(), e = end(); i != e; ++i)
1570 Node::Node(unsigned int Type, std::unique_ptr<Document> &D, StringRef A,
1572 : Doc(D), TypeID(Type), Anchor(A), Tag(T) {
1573 SMLoc Start = SMLoc::getFromPointer(peekNext().Range.begin());
1574 SourceRange = SMRange(Start, Start);
1577 std::string Node::getVerbatimTag() const {
1578 StringRef Raw = getRawTag();
1579 if (!Raw.empty() && Raw != "!") {
1581 if (Raw.find_last_of('!') == 0) {
1582 Ret = Doc->getTagMap().find("!")->second;
1583 Ret += Raw.substr(1);
1585 } else if (Raw.startswith("!!")) {
1586 Ret = Doc->getTagMap().find("!!")->second;
1587 Ret += Raw.substr(2);
1590 StringRef TagHandle = Raw.substr(0, Raw.find_last_of('!') + 1);
1591 std::map<StringRef, StringRef>::const_iterator It =
1592 Doc->getTagMap().find(TagHandle);
1593 if (It != Doc->getTagMap().end())
1597 T.Kind = Token::TK_Tag;
1598 T.Range = TagHandle;
1599 setError(Twine("Unknown tag handle ") + TagHandle, T);
1601 Ret += Raw.substr(Raw.find_last_of('!') + 1);
1606 switch (getType()) {
1608 return "tag:yaml.org,2002:null";
1610 // TODO: Tag resolution.
1611 return "tag:yaml.org,2002:str";
1613 return "tag:yaml.org,2002:map";
1615 return "tag:yaml.org,2002:seq";
1621 Token &Node::peekNext() {
1622 return Doc->peekNext();
1625 Token Node::getNext() {
1626 return Doc->getNext();
1629 Node *Node::parseBlockNode() {
1630 return Doc->parseBlockNode();
1633 BumpPtrAllocator &Node::getAllocator() {
1634 return Doc->NodeAllocator;
1637 void Node::setError(const Twine &Msg, Token &Tok) const {
1638 Doc->setError(Msg, Tok);
1641 bool Node::failed() const {
1642 return Doc->failed();
1647 StringRef ScalarNode::getValue(SmallVectorImpl<char> &Storage) const {
1648 // TODO: Handle newlines properly. We need to remove leading whitespace.
1649 if (Value[0] == '"') { // Double quoted.
1650 // Pull off the leading and trailing "s.
1651 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1652 // Search for characters that would require unescaping the value.
1653 StringRef::size_type i = UnquotedValue.find_first_of("\\\r\n");
1654 if (i != StringRef::npos)
1655 return unescapeDoubleQuoted(UnquotedValue, i, Storage);
1656 return UnquotedValue;
1657 } else if (Value[0] == '\'') { // Single quoted.
1658 // Pull off the leading and trailing 's.
1659 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1660 StringRef::size_type i = UnquotedValue.find('\'');
1661 if (i != StringRef::npos) {
1662 // We're going to need Storage.
1664 Storage.reserve(UnquotedValue.size());
1665 for (; i != StringRef::npos; i = UnquotedValue.find('\'')) {
1666 StringRef Valid(UnquotedValue.begin(), i);
1667 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1668 Storage.push_back('\'');
1669 UnquotedValue = UnquotedValue.substr(i + 2);
1671 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1672 return StringRef(Storage.begin(), Storage.size());
1674 return UnquotedValue;
1677 return Value.rtrim(" ");
1680 StringRef ScalarNode::unescapeDoubleQuoted( StringRef UnquotedValue
1681 , StringRef::size_type i
1682 , SmallVectorImpl<char> &Storage)
1684 // Use Storage to build proper value.
1686 Storage.reserve(UnquotedValue.size());
1687 for (; i != StringRef::npos; i = UnquotedValue.find_first_of("\\\r\n")) {
1688 // Insert all previous chars into Storage.
1689 StringRef Valid(UnquotedValue.begin(), i);
1690 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1691 // Chop off inserted chars.
1692 UnquotedValue = UnquotedValue.substr(i);
1694 assert(!UnquotedValue.empty() && "Can't be empty!");
1696 // Parse escape or line break.
1697 switch (UnquotedValue[0]) {
1700 Storage.push_back('\n');
1701 if ( UnquotedValue.size() > 1
1702 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1703 UnquotedValue = UnquotedValue.substr(1);
1704 UnquotedValue = UnquotedValue.substr(1);
1707 if (UnquotedValue.size() == 1)
1708 // TODO: Report error.
1710 UnquotedValue = UnquotedValue.substr(1);
1711 switch (UnquotedValue[0]) {
1714 T.Range = StringRef(UnquotedValue.begin(), 1);
1715 setError("Unrecognized escape code!", T);
1720 // Remove the new line.
1721 if ( UnquotedValue.size() > 1
1722 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1723 UnquotedValue = UnquotedValue.substr(1);
1724 // If this was just a single byte newline, it will get skipped
1728 Storage.push_back(0x00);
1731 Storage.push_back(0x07);
1734 Storage.push_back(0x08);
1738 Storage.push_back(0x09);
1741 Storage.push_back(0x0A);
1744 Storage.push_back(0x0B);
1747 Storage.push_back(0x0C);
1750 Storage.push_back(0x0D);
1753 Storage.push_back(0x1B);
1756 Storage.push_back(0x20);
1759 Storage.push_back(0x22);
1762 Storage.push_back(0x2F);
1765 Storage.push_back(0x5C);
1768 encodeUTF8(0x85, Storage);
1771 encodeUTF8(0xA0, Storage);
1774 encodeUTF8(0x2028, Storage);
1777 encodeUTF8(0x2029, Storage);
1780 if (UnquotedValue.size() < 3)
1781 // TODO: Report error.
1783 unsigned int UnicodeScalarValue;
1784 if (UnquotedValue.substr(1, 2).getAsInteger(16, UnicodeScalarValue))
1785 // TODO: Report error.
1786 UnicodeScalarValue = 0xFFFD;
1787 encodeUTF8(UnicodeScalarValue, Storage);
1788 UnquotedValue = UnquotedValue.substr(2);
1792 if (UnquotedValue.size() < 5)
1793 // TODO: Report error.
1795 unsigned int UnicodeScalarValue;
1796 if (UnquotedValue.substr(1, 4).getAsInteger(16, UnicodeScalarValue))
1797 // TODO: Report error.
1798 UnicodeScalarValue = 0xFFFD;
1799 encodeUTF8(UnicodeScalarValue, Storage);
1800 UnquotedValue = UnquotedValue.substr(4);
1804 if (UnquotedValue.size() < 9)
1805 // TODO: Report error.
1807 unsigned int UnicodeScalarValue;
1808 if (UnquotedValue.substr(1, 8).getAsInteger(16, UnicodeScalarValue))
1809 // TODO: Report error.
1810 UnicodeScalarValue = 0xFFFD;
1811 encodeUTF8(UnicodeScalarValue, Storage);
1812 UnquotedValue = UnquotedValue.substr(8);
1816 UnquotedValue = UnquotedValue.substr(1);
1819 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1820 return StringRef(Storage.begin(), Storage.size());
1823 Node *KeyValueNode::getKey() {
1826 // Handle implicit null keys.
1828 Token &t = peekNext();
1829 if ( t.Kind == Token::TK_BlockEnd
1830 || t.Kind == Token::TK_Value
1831 || t.Kind == Token::TK_Error) {
1832 return Key = new (getAllocator()) NullNode(Doc);
1834 if (t.Kind == Token::TK_Key)
1835 getNext(); // skip TK_Key.
1838 // Handle explicit null keys.
1839 Token &t = peekNext();
1840 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Value) {
1841 return Key = new (getAllocator()) NullNode(Doc);
1844 // We've got a normal key.
1845 return Key = parseBlockNode();
1848 Node *KeyValueNode::getValue() {
1853 return Value = new (getAllocator()) NullNode(Doc);
1855 // Handle implicit null values.
1857 Token &t = peekNext();
1858 if ( t.Kind == Token::TK_BlockEnd
1859 || t.Kind == Token::TK_FlowMappingEnd
1860 || t.Kind == Token::TK_Key
1861 || t.Kind == Token::TK_FlowEntry
1862 || t.Kind == Token::TK_Error) {
1863 return Value = new (getAllocator()) NullNode(Doc);
1866 if (t.Kind != Token::TK_Value) {
1867 setError("Unexpected token in Key Value.", t);
1868 return Value = new (getAllocator()) NullNode(Doc);
1870 getNext(); // skip TK_Value.
1873 // Handle explicit null values.
1874 Token &t = peekNext();
1875 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Key) {
1876 return Value = new (getAllocator()) NullNode(Doc);
1879 // We got a normal value.
1880 return Value = parseBlockNode();
1883 void MappingNode::increment() {
1886 CurrentEntry = nullptr;
1890 CurrentEntry->skip();
1891 if (Type == MT_Inline) {
1893 CurrentEntry = nullptr;
1897 Token T = peekNext();
1898 if (T.Kind == Token::TK_Key || T.Kind == Token::TK_Scalar) {
1899 // KeyValueNode eats the TK_Key. That way it can detect null keys.
1900 CurrentEntry = new (getAllocator()) KeyValueNode(Doc);
1901 } else if (Type == MT_Block) {
1903 case Token::TK_BlockEnd:
1906 CurrentEntry = nullptr;
1909 setError("Unexpected token. Expected Key or Block End", T);
1910 case Token::TK_Error:
1912 CurrentEntry = nullptr;
1916 case Token::TK_FlowEntry:
1917 // Eat the flow entry and recurse.
1920 case Token::TK_FlowMappingEnd:
1922 case Token::TK_Error:
1923 // Set this to end iterator.
1925 CurrentEntry = nullptr;
1928 setError( "Unexpected token. Expected Key, Flow Entry, or Flow "
1932 CurrentEntry = nullptr;
1937 void SequenceNode::increment() {
1940 CurrentEntry = nullptr;
1944 CurrentEntry->skip();
1945 Token T = peekNext();
1946 if (SeqType == ST_Block) {
1948 case Token::TK_BlockEntry:
1950 CurrentEntry = parseBlockNode();
1951 if (!CurrentEntry) { // An error occurred.
1953 CurrentEntry = nullptr;
1956 case Token::TK_BlockEnd:
1959 CurrentEntry = nullptr;
1962 setError( "Unexpected token. Expected Block Entry or Block End."
1964 case Token::TK_Error:
1966 CurrentEntry = nullptr;
1968 } else if (SeqType == ST_Indentless) {
1970 case Token::TK_BlockEntry:
1972 CurrentEntry = parseBlockNode();
1973 if (!CurrentEntry) { // An error occurred.
1975 CurrentEntry = nullptr;
1979 case Token::TK_Error:
1981 CurrentEntry = nullptr;
1983 } else if (SeqType == ST_Flow) {
1985 case Token::TK_FlowEntry:
1986 // Eat the flow entry and recurse.
1988 WasPreviousTokenFlowEntry = true;
1990 case Token::TK_FlowSequenceEnd:
1992 case Token::TK_Error:
1993 // Set this to end iterator.
1995 CurrentEntry = nullptr;
1997 case Token::TK_StreamEnd:
1998 case Token::TK_DocumentEnd:
1999 case Token::TK_DocumentStart:
2000 setError("Could not find closing ]!", T);
2001 // Set this to end iterator.
2003 CurrentEntry = nullptr;
2006 if (!WasPreviousTokenFlowEntry) {
2007 setError("Expected , between entries!", T);
2009 CurrentEntry = nullptr;
2012 // Otherwise it must be a flow entry.
2013 CurrentEntry = parseBlockNode();
2014 if (!CurrentEntry) {
2017 WasPreviousTokenFlowEntry = false;
2023 Document::Document(Stream &S) : stream(S), Root(nullptr) {
2024 // Tag maps starts with two default mappings.
2026 TagMap["!!"] = "tag:yaml.org,2002:";
2028 if (parseDirectives())
2029 expectToken(Token::TK_DocumentStart);
2030 Token &T = peekNext();
2031 if (T.Kind == Token::TK_DocumentStart)
2035 bool Document::skip() {
2036 if (stream.scanner->failed())
2041 Token &T = peekNext();
2042 if (T.Kind == Token::TK_StreamEnd)
2044 if (T.Kind == Token::TK_DocumentEnd) {
2051 Token &Document::peekNext() {
2052 return stream.scanner->peekNext();
2055 Token Document::getNext() {
2056 return stream.scanner->getNext();
2059 void Document::setError(const Twine &Message, Token &Location) const {
2060 stream.scanner->setError(Message, Location.Range.begin());
2063 bool Document::failed() const {
2064 return stream.scanner->failed();
2067 Node *Document::parseBlockNode() {
2068 Token T = peekNext();
2069 // Handle properties.
2074 case Token::TK_Alias:
2076 return new (NodeAllocator) AliasNode(stream.CurrentDoc, T.Range.substr(1));
2077 case Token::TK_Anchor:
2078 if (AnchorInfo.Kind == Token::TK_Anchor) {
2079 setError("Already encountered an anchor for this node!", T);
2082 AnchorInfo = getNext(); // Consume TK_Anchor.
2084 goto parse_property;
2086 if (TagInfo.Kind == Token::TK_Tag) {
2087 setError("Already encountered a tag for this node!", T);
2090 TagInfo = getNext(); // Consume TK_Tag.
2092 goto parse_property;
2098 case Token::TK_BlockEntry:
2099 // We got an unindented BlockEntry sequence. This is not terminated with
2101 // Don't eat the TK_BlockEntry, SequenceNode needs it.
2102 return new (NodeAllocator) SequenceNode( stream.CurrentDoc
2103 , AnchorInfo.Range.substr(1)
2105 , SequenceNode::ST_Indentless);
2106 case Token::TK_BlockSequenceStart:
2108 return new (NodeAllocator)
2109 SequenceNode( stream.CurrentDoc
2110 , AnchorInfo.Range.substr(1)
2112 , SequenceNode::ST_Block);
2113 case Token::TK_BlockMappingStart:
2115 return new (NodeAllocator)
2116 MappingNode( stream.CurrentDoc
2117 , AnchorInfo.Range.substr(1)
2119 , MappingNode::MT_Block);
2120 case Token::TK_FlowSequenceStart:
2122 return new (NodeAllocator)
2123 SequenceNode( stream.CurrentDoc
2124 , AnchorInfo.Range.substr(1)
2126 , SequenceNode::ST_Flow);
2127 case Token::TK_FlowMappingStart:
2129 return new (NodeAllocator)
2130 MappingNode( stream.CurrentDoc
2131 , AnchorInfo.Range.substr(1)
2133 , MappingNode::MT_Flow);
2134 case Token::TK_Scalar:
2136 return new (NodeAllocator)
2137 ScalarNode( stream.CurrentDoc
2138 , AnchorInfo.Range.substr(1)
2142 // Don't eat the TK_Key, KeyValueNode expects it.
2143 return new (NodeAllocator)
2144 MappingNode( stream.CurrentDoc
2145 , AnchorInfo.Range.substr(1)
2147 , MappingNode::MT_Inline);
2148 case Token::TK_DocumentStart:
2149 case Token::TK_DocumentEnd:
2150 case Token::TK_StreamEnd:
2152 // TODO: Properly handle tags. "[!!str ]" should resolve to !!str "", not
2154 return new (NodeAllocator) NullNode(stream.CurrentDoc);
2155 case Token::TK_Error:
2158 llvm_unreachable("Control flow shouldn't reach here.");
2162 bool Document::parseDirectives() {
2163 bool isDirective = false;
2165 Token T = peekNext();
2166 if (T.Kind == Token::TK_TagDirective) {
2167 parseTAGDirective();
2169 } else if (T.Kind == Token::TK_VersionDirective) {
2170 parseYAMLDirective();
2178 void Document::parseYAMLDirective() {
2179 getNext(); // Eat %YAML <version>
2182 void Document::parseTAGDirective() {
2183 Token Tag = getNext(); // %TAG <handle> <prefix>
2184 StringRef T = Tag.Range;
2186 T = T.substr(T.find_first_of(" \t")).ltrim(" \t");
2187 std::size_t HandleEnd = T.find_first_of(" \t");
2188 StringRef TagHandle = T.substr(0, HandleEnd);
2189 StringRef TagPrefix = T.substr(HandleEnd).ltrim(" \t");
2190 TagMap[TagHandle] = TagPrefix;
2193 bool Document::expectToken(int TK) {
2194 Token T = getNext();
2196 setError("Unexpected token", T);