1 //===--- YAMLParser.cpp - Simple YAML parser ------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements a YAML parser.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Support/YAMLParser.h"
16 #include "llvm/ADT/ilist.h"
17 #include "llvm/ADT/ilist_node.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringExtras.h"
20 #include "llvm/ADT/Twine.h"
21 #include "llvm/Support/ErrorHandling.h"
22 #include "llvm/Support/MemoryBuffer.h"
23 #include "llvm/Support/raw_ostream.h"
24 #include "llvm/Support/SourceMgr.h"
29 enum UnicodeEncodingForm {
30 UEF_UTF32_LE, //< UTF-32 Little Endian
31 UEF_UTF32_BE, //< UTF-32 Big Endian
32 UEF_UTF16_LE, //< UTF-16 Little Endian
33 UEF_UTF16_BE, //< UTF-16 Big Endian
34 UEF_UTF8, //< UTF-8 or ascii.
35 UEF_Unknown //< Not a valid Unicode encoding.
38 /// EncodingInfo - Holds the encoding type and length of the byte order mark if
39 /// it exists. Length is in {0, 2, 3, 4}.
40 typedef std::pair<UnicodeEncodingForm, unsigned> EncodingInfo;
42 /// getUnicodeEncoding - Reads up to the first 4 bytes to determine the Unicode
43 /// encoding form of \a Input.
45 /// @param Input A string of length 0 or more.
46 /// @returns An EncodingInfo indicating the Unicode encoding form of the input
47 /// and how long the byte order mark is if one exists.
48 static EncodingInfo getUnicodeEncoding(StringRef Input) {
49 if (Input.size() == 0)
50 return std::make_pair(UEF_Unknown, 0);
52 switch (uint8_t(Input[0])) {
54 if (Input.size() >= 4) {
56 && uint8_t(Input[2]) == 0xFE
57 && uint8_t(Input[3]) == 0xFF)
58 return std::make_pair(UEF_UTF32_BE, 4);
59 if (Input[1] == 0 && Input[2] == 0 && Input[3] != 0)
60 return std::make_pair(UEF_UTF32_BE, 0);
63 if (Input.size() >= 2 && Input[1] != 0)
64 return std::make_pair(UEF_UTF16_BE, 0);
65 return std::make_pair(UEF_Unknown, 0);
67 if ( Input.size() >= 4
68 && uint8_t(Input[1]) == 0xFE
71 return std::make_pair(UEF_UTF32_LE, 4);
73 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFE)
74 return std::make_pair(UEF_UTF16_LE, 2);
75 return std::make_pair(UEF_Unknown, 0);
77 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFF)
78 return std::make_pair(UEF_UTF16_BE, 2);
79 return std::make_pair(UEF_Unknown, 0);
81 if ( Input.size() >= 3
82 && uint8_t(Input[1]) == 0xBB
83 && uint8_t(Input[2]) == 0xBF)
84 return std::make_pair(UEF_UTF8, 3);
85 return std::make_pair(UEF_Unknown, 0);
88 // It could still be utf-32 or utf-16.
89 if (Input.size() >= 4 && Input[1] == 0 && Input[2] == 0 && Input[3] == 0)
90 return std::make_pair(UEF_UTF32_LE, 0);
92 if (Input.size() >= 2 && Input[1] == 0)
93 return std::make_pair(UEF_UTF16_LE, 0);
95 return std::make_pair(UEF_UTF8, 0);
100 /// Token - A single YAML token.
101 struct Token : ilist_node<Token> {
103 TK_Error, // Uninitialized token.
112 TK_BlockSequenceStart,
113 TK_BlockMappingStart,
115 TK_FlowSequenceStart,
127 /// A string of length 0 or more whose begin() points to the logical location
128 /// of the token in the input.
131 Token() : Kind(TK_Error) {}
137 struct ilist_sentinel_traits<Token> {
138 Token *createSentinel() const {
141 static void destroySentinel(Token*) {}
143 Token *provideInitialHead() const { return createSentinel(); }
144 Token *ensureHead(Token*) const { return createSentinel(); }
145 static void noteHead(Token*, Token*) {}
148 mutable Token Sentinel;
152 struct ilist_node_traits<Token> {
153 Token *createNode(const Token &V) {
154 return new (Alloc.Allocate<Token>()) Token(V);
156 static void deleteNode(Token *V) {}
158 void addNodeToList(Token *) {}
159 void removeNodeFromList(Token *) {}
160 void transferNodesFromList(ilist_node_traits & /*SrcTraits*/,
161 ilist_iterator<Token> /*first*/,
162 ilist_iterator<Token> /*last*/) {}
164 BumpPtrAllocator Alloc;
167 typedef ilist<Token> TokenQueueT;
170 /// @brief This struct is used to track simple keys.
172 /// Simple keys are handled by creating an entry in SimpleKeys for each Token
173 /// which could legally be the start of a simple key. When peekNext is called,
174 /// if the Token To be returned is referenced by a SimpleKey, we continue
175 /// tokenizing until that potential simple key has either been found to not be
176 /// a simple key (we moved on to the next line or went further than 1024 chars).
177 /// Or when we run into a Value, and then insert a Key token (and possibly
178 /// others) before the SimpleKey's Tok.
180 TokenQueueT::iterator Tok;
186 bool operator ==(const SimpleKey &Other) {
187 return Tok == Other.Tok;
192 /// @brief The Unicode scalar value of a UTF-8 minimal well-formed code unit
193 /// subsequence and the subsequence's length in code units (uint8_t).
194 /// A length of 0 represents an error.
195 typedef std::pair<uint32_t, unsigned> UTF8Decoded;
197 static UTF8Decoded decodeUTF8(StringRef Range) {
198 StringRef::iterator Position= Range.begin();
199 StringRef::iterator End = Range.end();
200 // 1 byte: [0x00, 0x7f]
201 // Bit pattern: 0xxxxxxx
202 if ((*Position & 0x80) == 0) {
203 return std::make_pair(*Position, 1);
205 // 2 bytes: [0x80, 0x7ff]
206 // Bit pattern: 110xxxxx 10xxxxxx
207 if (Position + 1 != End &&
208 ((*Position & 0xE0) == 0xC0) &&
209 ((*(Position + 1) & 0xC0) == 0x80)) {
210 uint32_t codepoint = ((*Position & 0x1F) << 6) |
211 (*(Position + 1) & 0x3F);
212 if (codepoint >= 0x80)
213 return std::make_pair(codepoint, 2);
215 // 3 bytes: [0x8000, 0xffff]
216 // Bit pattern: 1110xxxx 10xxxxxx 10xxxxxx
217 if (Position + 2 != End &&
218 ((*Position & 0xF0) == 0xE0) &&
219 ((*(Position + 1) & 0xC0) == 0x80) &&
220 ((*(Position + 2) & 0xC0) == 0x80)) {
221 uint32_t codepoint = ((*Position & 0x0F) << 12) |
222 ((*(Position + 1) & 0x3F) << 6) |
223 (*(Position + 2) & 0x3F);
224 // Codepoints between 0xD800 and 0xDFFF are invalid, as
225 // they are high / low surrogate halves used by UTF-16.
226 if (codepoint >= 0x800 &&
227 (codepoint < 0xD800 || codepoint > 0xDFFF))
228 return std::make_pair(codepoint, 3);
230 // 4 bytes: [0x10000, 0x10FFFF]
231 // Bit pattern: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
232 if (Position + 3 != End &&
233 ((*Position & 0xF8) == 0xF0) &&
234 ((*(Position + 1) & 0xC0) == 0x80) &&
235 ((*(Position + 2) & 0xC0) == 0x80) &&
236 ((*(Position + 3) & 0xC0) == 0x80)) {
237 uint32_t codepoint = ((*Position & 0x07) << 18) |
238 ((*(Position + 1) & 0x3F) << 12) |
239 ((*(Position + 2) & 0x3F) << 6) |
240 (*(Position + 3) & 0x3F);
241 if (codepoint >= 0x10000 && codepoint <= 0x10FFFF)
242 return std::make_pair(codepoint, 4);
244 return std::make_pair(0, 0);
249 /// @brief Scans YAML tokens from a MemoryBuffer.
252 Scanner(const StringRef Input, SourceMgr &SM);
254 /// @brief Parse the next token and return it without popping it.
257 /// @brief Parse the next token and pop it from the queue.
260 void printError(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Message,
261 ArrayRef<SMRange> Ranges = ArrayRef<SMRange>()) {
262 SM.PrintMessage(Loc, Kind, Message, Ranges);
265 void setError(const Twine &Message, StringRef::iterator Position) {
269 // Don't print out more errors after the first one we encounter. The rest
270 // are just the result of the first, and have no meaning.
272 printError(SMLoc::getFromPointer(Current), SourceMgr::DK_Error, Message);
276 void setError(const Twine &Message) {
277 setError(Message, Current);
280 /// @brief Returns true if an error occurred while parsing.
286 StringRef currentInput() {
287 return StringRef(Current, End - Current);
290 /// @brief Decode a UTF-8 minimal well-formed code unit subsequence starting
293 /// If the UTF-8 code units starting at Position do not form a well-formed
294 /// code unit subsequence, then the Unicode scalar value is 0, and the length
296 UTF8Decoded decodeUTF8(StringRef::iterator Position) {
297 return ::decodeUTF8(StringRef(Position, End - Position));
300 // The following functions are based on the gramar rules in the YAML spec. The
301 // style of the function names it meant to closely match how they are written
302 // in the spec. The number within the [] is the number of the grammar rule in
305 // See 4.2 [Production Naming Conventions] for the meaning of the prefixes.
308 // A production starting and ending with a special character.
310 // A production matching a single line break.
312 // A production starting and ending with a non-break character.
314 // A production starting and ending with a white space character.
316 // A production starting and ending with a non-space character.
318 // A production matching complete line(s).
320 /// @brief Skip a single nb-char[27] starting at Position.
322 /// A nb-char is 0x9 | [0x20-0x7E] | 0x85 | [0xA0-0xD7FF] | [0xE000-0xFEFE]
323 /// | [0xFF00-0xFFFD] | [0x10000-0x10FFFF]
325 /// @returns The code unit after the nb-char, or Position if it's not an
327 StringRef::iterator skip_nb_char(StringRef::iterator Position);
329 /// @brief Skip a single b-break[28] starting at Position.
331 /// A b-break is 0xD 0xA | 0xD | 0xA
333 /// @returns The code unit after the b-break, or Position if it's not a
335 StringRef::iterator skip_b_break(StringRef::iterator Position);
337 /// @brief Skip a single s-white[33] starting at Position.
339 /// A s-white is 0x20 | 0x9
341 /// @returns The code unit after the s-white, or Position if it's not a
343 StringRef::iterator skip_s_white(StringRef::iterator Position);
345 /// @brief Skip a single ns-char[34] starting at Position.
347 /// A ns-char is nb-char - s-white
349 /// @returns The code unit after the ns-char, or Position if it's not a
351 StringRef::iterator skip_ns_char(StringRef::iterator Position);
353 typedef StringRef::iterator (Scanner::*SkipWhileFunc)(StringRef::iterator);
354 /// @brief Skip minimal well-formed code unit subsequences until Func
355 /// returns its input.
357 /// @returns The code unit after the last minimal well-formed code unit
358 /// subsequence that Func accepted.
359 StringRef::iterator skip_while( SkipWhileFunc Func
360 , StringRef::iterator Position);
362 /// @brief Scan ns-uri-char[39]s starting at Cur.
364 /// This updates Cur and Column while scanning.
366 /// @returns A StringRef starting at Cur which covers the longest contiguous
367 /// sequence of ns-uri-char.
368 StringRef scan_ns_uri_char();
370 /// @brief Scan ns-plain-one-line[133] starting at \a Cur.
371 StringRef scan_ns_plain_one_line();
373 /// @brief Consume a minimal well-formed code unit subsequence starting at
374 /// \a Cur. Return false if it is not the same Unicode scalar value as
375 /// \a Expected. This updates \a Column.
376 bool consume(uint32_t Expected);
378 /// @brief Skip \a Distance UTF-8 code units. Updates \a Cur and \a Column.
379 void skip(uint32_t Distance);
381 /// @brief Return true if the minimal well-formed code unit subsequence at
382 /// Pos is whitespace or a new line
383 bool isBlankOrBreak(StringRef::iterator Position);
385 /// @brief If IsSimpleKeyAllowed, create and push_back a new SimpleKey.
386 void saveSimpleKeyCandidate( TokenQueueT::iterator Tok
390 /// @brief Remove simple keys that can no longer be valid simple keys.
392 /// Invalid simple keys are not on the current line or are further than 1024
394 void removeStaleSimpleKeyCandidates();
396 /// @brief Remove all simple keys on FlowLevel \a Level.
397 void removeSimpleKeyCandidatesOnFlowLevel(unsigned Level);
399 /// @brief Unroll indentation in \a Indents back to \a Col. Creates BlockEnd
400 /// tokens if needed.
401 bool unrollIndent(int ToColumn);
403 /// @brief Increase indent to \a Col. Creates \a Kind token at \a InsertPoint
405 bool rollIndent( int ToColumn
406 , Token::TokenKind Kind
407 , TokenQueueT::iterator InsertPoint);
409 /// @brief Skip whitespace and comments until the start of the next token.
410 void scanToNextToken();
412 /// @brief Must be the first token generated.
413 bool scanStreamStart();
415 /// @brief Generate tokens needed to close out the stream.
416 bool scanStreamEnd();
418 /// @brief Scan a %BLAH directive.
419 bool scanDirective();
421 /// @brief Scan a ... or ---.
422 bool scanDocumentIndicator(bool IsStart);
424 /// @brief Scan a [ or { and generate the proper flow collection start token.
425 bool scanFlowCollectionStart(bool IsSequence);
427 /// @brief Scan a ] or } and generate the proper flow collection end token.
428 bool scanFlowCollectionEnd(bool IsSequence);
430 /// @brief Scan the , that separates entries in a flow collection.
431 bool scanFlowEntry();
433 /// @brief Scan the - that starts block sequence entries.
434 bool scanBlockEntry();
436 /// @brief Scan an explicit ? indicating a key.
439 /// @brief Scan an explicit : indicating a value.
442 /// @brief Scan a quoted scalar.
443 bool scanFlowScalar(bool IsDoubleQuoted);
445 /// @brief Scan an unquoted scalar.
446 bool scanPlainScalar();
448 /// @brief Scan an Alias or Anchor starting with * or &.
449 bool scanAliasOrAnchor(bool IsAlias);
451 /// @brief Scan a block scalar starting with | or >.
452 bool scanBlockScalar(bool IsLiteral);
454 /// @brief Scan a tag of the form !stuff.
457 /// @brief Dispatch to the next scanning function based on \a *Cur.
458 bool fetchMoreTokens();
460 /// @brief The SourceMgr used for diagnostics and buffer management.
463 /// @brief The original input.
464 MemoryBuffer *InputBuffer;
466 /// @brief The current position of the scanner.
467 StringRef::iterator Current;
469 /// @brief The end of the input (one past the last character).
470 StringRef::iterator End;
472 /// @brief Current YAML indentation level in spaces.
475 /// @brief Current column number in Unicode code points.
478 /// @brief Current line number.
481 /// @brief How deep we are in flow style containers. 0 Means at block level.
484 /// @brief Are we at the start of the stream?
485 bool IsStartOfStream;
487 /// @brief Can the next token be the start of a simple key?
488 bool IsSimpleKeyAllowed;
490 /// @brief Is the next token required to start a simple key?
491 bool IsSimpleKeyRequired;
493 /// @brief True if an error has occurred.
496 /// @brief Queue of tokens. This is required to queue up tokens while looking
497 /// for the end of a simple key. And for cases where a single character
498 /// can produce multiple tokens (e.g. BlockEnd).
499 TokenQueueT TokenQueue;
501 /// @brief Indentation levels.
502 SmallVector<int, 4> Indents;
504 /// @brief Potential simple keys.
505 SmallVector<SimpleKey, 4> SimpleKeys;
508 } // end namespace yaml
509 } // end namespace llvm
511 /// encodeUTF8 - Encode \a UnicodeScalarValue in UTF-8 and append it to result.
512 static void encodeUTF8( uint32_t UnicodeScalarValue
513 , SmallVectorImpl<char> &Result) {
514 if (UnicodeScalarValue <= 0x7F) {
515 Result.push_back(UnicodeScalarValue & 0x7F);
516 } else if (UnicodeScalarValue <= 0x7FF) {
517 uint8_t FirstByte = 0xC0 | ((UnicodeScalarValue & 0x7C0) >> 6);
518 uint8_t SecondByte = 0x80 | (UnicodeScalarValue & 0x3F);
519 Result.push_back(FirstByte);
520 Result.push_back(SecondByte);
521 } else if (UnicodeScalarValue <= 0xFFFF) {
522 uint8_t FirstByte = 0xE0 | ((UnicodeScalarValue & 0xF000) >> 12);
523 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
524 uint8_t ThirdByte = 0x80 | (UnicodeScalarValue & 0x3F);
525 Result.push_back(FirstByte);
526 Result.push_back(SecondByte);
527 Result.push_back(ThirdByte);
528 } else if (UnicodeScalarValue <= 0x10FFFF) {
529 uint8_t FirstByte = 0xF0 | ((UnicodeScalarValue & 0x1F0000) >> 18);
530 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0x3F000) >> 12);
531 uint8_t ThirdByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
532 uint8_t FourthByte = 0x80 | (UnicodeScalarValue & 0x3F);
533 Result.push_back(FirstByte);
534 Result.push_back(SecondByte);
535 Result.push_back(ThirdByte);
536 Result.push_back(FourthByte);
540 bool yaml::dumpTokens(StringRef Input, raw_ostream &OS) {
542 Scanner scanner(Input, SM);
544 Token T = scanner.getNext();
546 case Token::TK_StreamStart:
547 OS << "Stream-Start: ";
549 case Token::TK_StreamEnd:
550 OS << "Stream-End: ";
552 case Token::TK_VersionDirective:
553 OS << "Version-Directive: ";
555 case Token::TK_TagDirective:
556 OS << "Tag-Directive: ";
558 case Token::TK_DocumentStart:
559 OS << "Document-Start: ";
561 case Token::TK_DocumentEnd:
562 OS << "Document-End: ";
564 case Token::TK_BlockEntry:
565 OS << "Block-Entry: ";
567 case Token::TK_BlockEnd:
570 case Token::TK_BlockSequenceStart:
571 OS << "Block-Sequence-Start: ";
573 case Token::TK_BlockMappingStart:
574 OS << "Block-Mapping-Start: ";
576 case Token::TK_FlowEntry:
577 OS << "Flow-Entry: ";
579 case Token::TK_FlowSequenceStart:
580 OS << "Flow-Sequence-Start: ";
582 case Token::TK_FlowSequenceEnd:
583 OS << "Flow-Sequence-End: ";
585 case Token::TK_FlowMappingStart:
586 OS << "Flow-Mapping-Start: ";
588 case Token::TK_FlowMappingEnd:
589 OS << "Flow-Mapping-End: ";
594 case Token::TK_Value:
597 case Token::TK_Scalar:
600 case Token::TK_Alias:
603 case Token::TK_Anchor:
609 case Token::TK_Error:
612 OS << T.Range << "\n";
613 if (T.Kind == Token::TK_StreamEnd)
615 else if (T.Kind == Token::TK_Error)
621 bool yaml::scanTokens(StringRef Input) {
623 llvm::yaml::Scanner scanner(Input, SM);
625 llvm::yaml::Token T = scanner.getNext();
626 if (T.Kind == Token::TK_StreamEnd)
628 else if (T.Kind == Token::TK_Error)
634 std::string yaml::escape(StringRef Input) {
635 std::string EscapedInput;
636 for (StringRef::iterator i = Input.begin(), e = Input.end(); i != e; ++i) {
638 EscapedInput += "\\\\";
640 EscapedInput += "\\\"";
642 EscapedInput += "\\0";
644 EscapedInput += "\\a";
646 EscapedInput += "\\b";
648 EscapedInput += "\\t";
650 EscapedInput += "\\n";
652 EscapedInput += "\\v";
654 EscapedInput += "\\f";
656 EscapedInput += "\\r";
658 EscapedInput += "\\e";
659 else if (*i >= 0 && *i < 0x20) { // Control characters not handled above.
660 std::string HexStr = utohexstr(*i);
661 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
662 } else if (*i & 0x80) { // UTF-8 multiple code unit subsequence.
663 UTF8Decoded UnicodeScalarValue
664 = decodeUTF8(StringRef(i, Input.end() - i));
665 if (UnicodeScalarValue.second == 0) {
666 // Found invalid char.
668 encodeUTF8(0xFFFD, Val);
669 EscapedInput.insert(EscapedInput.end(), Val.begin(), Val.end());
670 // FIXME: Error reporting.
673 if (UnicodeScalarValue.first == 0x85)
674 EscapedInput += "\\N";
675 else if (UnicodeScalarValue.first == 0xA0)
676 EscapedInput += "\\_";
677 else if (UnicodeScalarValue.first == 0x2028)
678 EscapedInput += "\\L";
679 else if (UnicodeScalarValue.first == 0x2029)
680 EscapedInput += "\\P";
682 std::string HexStr = utohexstr(UnicodeScalarValue.first);
683 if (HexStr.size() <= 2)
684 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
685 else if (HexStr.size() <= 4)
686 EscapedInput += "\\u" + std::string(4 - HexStr.size(), '0') + HexStr;
687 else if (HexStr.size() <= 8)
688 EscapedInput += "\\U" + std::string(8 - HexStr.size(), '0') + HexStr;
690 i += UnicodeScalarValue.second - 1;
692 EscapedInput.push_back(*i);
697 Scanner::Scanner(StringRef Input, SourceMgr &sm)
703 , IsStartOfStream(true)
704 , IsSimpleKeyAllowed(true)
705 , IsSimpleKeyRequired(false)
707 InputBuffer = MemoryBuffer::getMemBuffer(Input, "YAML");
708 SM.AddNewSourceBuffer(InputBuffer, SMLoc());
709 Current = InputBuffer->getBufferStart();
710 End = InputBuffer->getBufferEnd();
713 Token &Scanner::peekNext() {
714 // If the current token is a possible simple key, keep parsing until we
716 bool NeedMore = false;
718 if (TokenQueue.empty() || NeedMore) {
719 if (!fetchMoreTokens()) {
721 TokenQueue.push_back(Token());
722 return TokenQueue.front();
725 assert(!TokenQueue.empty() &&
726 "fetchMoreTokens lied about getting tokens!");
728 removeStaleSimpleKeyCandidates();
730 SK.Tok = TokenQueue.front();
731 if (std::find(SimpleKeys.begin(), SimpleKeys.end(), SK)
737 return TokenQueue.front();
740 Token Scanner::getNext() {
741 Token Ret = peekNext();
742 // TokenQueue can be empty if there was an error getting the next token.
743 if (!TokenQueue.empty())
744 TokenQueue.pop_front();
746 // There cannot be any referenced Token's if the TokenQueue is empty. So do a
747 // quick deallocation of them all.
748 if (TokenQueue.empty()) {
749 TokenQueue.Alloc.Reset();
755 StringRef::iterator Scanner::skip_nb_char(StringRef::iterator Position) {
756 // Check 7 bit c-printable - b-char.
757 if ( *Position == 0x09
758 || (*Position >= 0x20 && *Position <= 0x7E))
761 // Check for valid UTF-8.
762 if (uint8_t(*Position) & 0x80) {
763 UTF8Decoded u8d = decodeUTF8(Position);
765 && u8d.first != 0xFEFF
766 && ( u8d.first == 0x85
767 || ( u8d.first >= 0xA0
768 && u8d.first <= 0xD7FF)
769 || ( u8d.first >= 0xE000
770 && u8d.first <= 0xFFFD)
771 || ( u8d.first >= 0x10000
772 && u8d.first <= 0x10FFFF)))
773 return Position + u8d.second;
778 StringRef::iterator Scanner::skip_b_break(StringRef::iterator Position) {
779 if (*Position == 0x0D) {
780 if (Position + 1 != End && *(Position + 1) == 0x0A)
785 if (*Position == 0x0A)
791 StringRef::iterator Scanner::skip_s_white(StringRef::iterator Position) {
794 if (*Position == ' ' || *Position == '\t')
799 StringRef::iterator Scanner::skip_ns_char(StringRef::iterator Position) {
802 if (*Position == ' ' || *Position == '\t')
804 return skip_nb_char(Position);
807 StringRef::iterator Scanner::skip_while( SkipWhileFunc Func
808 , StringRef::iterator Position) {
810 StringRef::iterator i = (this->*Func)(Position);
818 static bool is_ns_hex_digit(const char C) {
819 return (C >= '0' && C <= '9')
820 || (C >= 'a' && C <= 'z')
821 || (C >= 'A' && C <= 'Z');
824 static bool is_ns_word_char(const char C) {
826 || (C >= 'a' && C <= 'z')
827 || (C >= 'A' && C <= 'Z');
830 StringRef Scanner::scan_ns_uri_char() {
831 StringRef::iterator Start = Current;
835 if (( *Current == '%'
837 && is_ns_hex_digit(*(Current + 1))
838 && is_ns_hex_digit(*(Current + 2)))
839 || is_ns_word_char(*Current)
840 || StringRef(Current, 1).find_first_of("#;/?:@&=+$,_.!~*'()[]")
841 != StringRef::npos) {
847 return StringRef(Start, Current - Start);
850 StringRef Scanner::scan_ns_plain_one_line() {
851 StringRef::iterator start = Current;
852 // The first character must already be verified.
855 if (Current == End) {
857 } else if (*Current == ':') {
858 // Check if the next character is a ns-char.
859 if (Current + 1 == End)
861 StringRef::iterator i = skip_ns_char(Current + 1);
862 if (Current + 1 != i) {
864 Column += 2; // Consume both the ':' and ns-char.
867 } else if (*Current == '#') {
868 // Check if the previous character was a ns-char.
869 // The & 0x80 check is to check for the trailing byte of a utf-8
870 if (*(Current - 1) & 0x80 || skip_ns_char(Current - 1) == Current) {
876 StringRef::iterator i = skip_nb_char(Current);
883 return StringRef(start, Current - start);
886 bool Scanner::consume(uint32_t Expected) {
887 if (Expected >= 0x80)
888 report_fatal_error("Not dealing with this yet");
891 if (uint8_t(*Current) >= 0x80)
892 report_fatal_error("Not dealing with this yet");
893 if (uint8_t(*Current) == Expected) {
901 void Scanner::skip(uint32_t Distance) {
906 bool Scanner::isBlankOrBreak(StringRef::iterator Position) {
909 if ( *Position == ' ' || *Position == '\t'
910 || *Position == '\r' || *Position == '\n')
915 void Scanner::saveSimpleKeyCandidate( TokenQueueT::iterator Tok
918 if (IsSimpleKeyAllowed) {
922 SK.Column = AtColumn;
923 SK.IsRequired = IsRequired;
924 SK.FlowLevel = FlowLevel;
925 SimpleKeys.push_back(SK);
929 void Scanner::removeStaleSimpleKeyCandidates() {
930 for (SmallVectorImpl<SimpleKey>::iterator i = SimpleKeys.begin();
931 i != SimpleKeys.end();) {
932 if (i->Line != Line || i->Column + 1024 < Column) {
934 setError( "Could not find expected : for simple key"
935 , i->Tok->Range.begin());
936 i = SimpleKeys.erase(i);
942 void Scanner::removeSimpleKeyCandidatesOnFlowLevel(unsigned Level) {
943 if (!SimpleKeys.empty() && (SimpleKeys.end() - 1)->FlowLevel == Level)
944 SimpleKeys.pop_back();
947 bool Scanner::unrollIndent(int ToColumn) {
949 // Indentation is ignored in flow.
953 while (Indent > ToColumn) {
954 T.Kind = Token::TK_BlockEnd;
955 T.Range = StringRef(Current, 1);
956 TokenQueue.push_back(T);
957 Indent = Indents.pop_back_val();
963 bool Scanner::rollIndent( int ToColumn
964 , Token::TokenKind Kind
965 , TokenQueueT::iterator InsertPoint) {
968 if (Indent < ToColumn) {
969 Indents.push_back(Indent);
974 T.Range = StringRef(Current, 0);
975 TokenQueue.insert(InsertPoint, T);
980 void Scanner::scanToNextToken() {
982 while (*Current == ' ' || *Current == '\t') {
987 if (*Current == '#') {
989 // This may skip more than one byte, thus Column is only incremented
991 StringRef::iterator i = skip_nb_char(Current);
1000 StringRef::iterator i = skip_b_break(Current);
1006 // New lines may start a simple key.
1008 IsSimpleKeyAllowed = true;
1012 bool Scanner::scanStreamStart() {
1013 IsStartOfStream = false;
1015 EncodingInfo EI = getUnicodeEncoding(currentInput());
1018 T.Kind = Token::TK_StreamStart;
1019 T.Range = StringRef(Current, EI.second);
1020 TokenQueue.push_back(T);
1021 Current += EI.second;
1025 bool Scanner::scanStreamEnd() {
1026 // Force an ending new line if one isn't present.
1034 IsSimpleKeyAllowed = false;
1037 T.Kind = Token::TK_StreamEnd;
1038 T.Range = StringRef(Current, 0);
1039 TokenQueue.push_back(T);
1043 bool Scanner::scanDirective() {
1044 // Reset the indentation level.
1047 IsSimpleKeyAllowed = false;
1049 StringRef::iterator Start = Current;
1051 StringRef::iterator NameStart = Current;
1052 Current = skip_while(&Scanner::skip_ns_char, Current);
1053 StringRef Name(NameStart, Current - NameStart);
1054 Current = skip_while(&Scanner::skip_s_white, Current);
1056 if (Name == "YAML") {
1057 Current = skip_while(&Scanner::skip_ns_char, Current);
1059 T.Kind = Token::TK_VersionDirective;
1060 T.Range = StringRef(Start, Current - Start);
1061 TokenQueue.push_back(T);
1067 bool Scanner::scanDocumentIndicator(bool IsStart) {
1070 IsSimpleKeyAllowed = false;
1073 T.Kind = IsStart ? Token::TK_DocumentStart : Token::TK_DocumentEnd;
1074 T.Range = StringRef(Current, 3);
1076 TokenQueue.push_back(T);
1080 bool Scanner::scanFlowCollectionStart(bool IsSequence) {
1082 T.Kind = IsSequence ? Token::TK_FlowSequenceStart
1083 : Token::TK_FlowMappingStart;
1084 T.Range = StringRef(Current, 1);
1086 TokenQueue.push_back(T);
1088 // [ and { may begin a simple key.
1089 saveSimpleKeyCandidate(TokenQueue.back(), Column - 1, false);
1091 // And may also be followed by a simple key.
1092 IsSimpleKeyAllowed = true;
1097 bool Scanner::scanFlowCollectionEnd(bool IsSequence) {
1098 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1099 IsSimpleKeyAllowed = false;
1101 T.Kind = IsSequence ? Token::TK_FlowSequenceEnd
1102 : Token::TK_FlowMappingEnd;
1103 T.Range = StringRef(Current, 1);
1105 TokenQueue.push_back(T);
1111 bool Scanner::scanFlowEntry() {
1112 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1113 IsSimpleKeyAllowed = true;
1115 T.Kind = Token::TK_FlowEntry;
1116 T.Range = StringRef(Current, 1);
1118 TokenQueue.push_back(T);
1122 bool Scanner::scanBlockEntry() {
1123 rollIndent(Column, Token::TK_BlockSequenceStart, TokenQueue.end());
1124 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1125 IsSimpleKeyAllowed = true;
1127 T.Kind = Token::TK_BlockEntry;
1128 T.Range = StringRef(Current, 1);
1130 TokenQueue.push_back(T);
1134 bool Scanner::scanKey() {
1136 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1138 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1139 IsSimpleKeyAllowed = !FlowLevel;
1142 T.Kind = Token::TK_Key;
1143 T.Range = StringRef(Current, 1);
1145 TokenQueue.push_back(T);
1149 bool Scanner::scanValue() {
1150 // If the previous token could have been a simple key, insert the key token
1151 // into the token queue.
1152 if (!SimpleKeys.empty()) {
1153 SimpleKey SK = SimpleKeys.pop_back_val();
1155 T.Kind = Token::TK_Key;
1156 T.Range = SK.Tok->Range;
1157 TokenQueueT::iterator i, e;
1158 for (i = TokenQueue.begin(), e = TokenQueue.end(); i != e; ++i) {
1162 assert(i != e && "SimpleKey not in token queue!");
1163 i = TokenQueue.insert(i, T);
1165 // We may also need to add a Block-Mapping-Start token.
1166 rollIndent(SK.Column, Token::TK_BlockMappingStart, i);
1168 IsSimpleKeyAllowed = false;
1171 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1172 IsSimpleKeyAllowed = !FlowLevel;
1176 T.Kind = Token::TK_Value;
1177 T.Range = StringRef(Current, 1);
1179 TokenQueue.push_back(T);
1183 // Forbidding inlining improves performance by roughly 20%.
1184 // FIXME: Remove once llvm optimizes this to the faster version without hints.
1185 LLVM_ATTRIBUTE_NOINLINE static bool
1186 wasEscaped(StringRef::iterator First, StringRef::iterator Position);
1188 // Returns whether a character at 'Position' was escaped with a leading '\'.
1189 // 'First' specifies the position of the first character in the string.
1190 static bool wasEscaped(StringRef::iterator First,
1191 StringRef::iterator Position) {
1192 assert(Position - 1 >= First);
1193 StringRef::iterator I = Position - 1;
1194 // We calculate the number of consecutive '\'s before the current position
1195 // by iterating backwards through our string.
1196 while (I >= First && *I == '\\') --I;
1197 // (Position - 1 - I) now contains the number of '\'s before the current
1198 // position. If it is odd, the character at 'Position' was escaped.
1199 return (Position - 1 - I) % 2 == 1;
1202 bool Scanner::scanFlowScalar(bool IsDoubleQuoted) {
1203 StringRef::iterator Start = Current;
1204 unsigned ColStart = Column;
1205 if (IsDoubleQuoted) {
1208 while (Current != End && *Current != '"')
1210 // Repeat until the previous character was not a '\' or was an escaped
1212 } while (*(Current - 1) == '\\' && wasEscaped(Start + 1, Current));
1216 // Skip a ' followed by another '.
1217 if (Current + 1 < End && *Current == '\'' && *(Current + 1) == '\'') {
1220 } else if (*Current == '\'')
1222 StringRef::iterator i = skip_nb_char(Current);
1224 i = skip_b_break(Current);
1238 skip(1); // Skip ending quote.
1240 T.Kind = Token::TK_Scalar;
1241 T.Range = StringRef(Start, Current - Start);
1242 TokenQueue.push_back(T);
1244 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1246 IsSimpleKeyAllowed = false;
1251 bool Scanner::scanPlainScalar() {
1252 StringRef::iterator Start = Current;
1253 unsigned ColStart = Column;
1254 unsigned LeadingBlanks = 0;
1255 assert(Indent >= -1 && "Indent must be >= -1 !");
1256 unsigned indent = static_cast<unsigned>(Indent + 1);
1258 if (*Current == '#')
1261 while (!isBlankOrBreak(Current)) {
1262 if ( FlowLevel && *Current == ':'
1263 && !(isBlankOrBreak(Current + 1) || *(Current + 1) == ',')) {
1264 setError("Found unexpected ':' while scanning a plain scalar", Current);
1268 // Check for the end of the plain scalar.
1269 if ( (*Current == ':' && isBlankOrBreak(Current + 1))
1271 && (StringRef(Current, 1).find_first_of(",:?[]{}")
1272 != StringRef::npos)))
1275 StringRef::iterator i = skip_nb_char(Current);
1282 // Are we at the end?
1283 if (!isBlankOrBreak(Current))
1287 StringRef::iterator Tmp = Current;
1288 while (isBlankOrBreak(Tmp)) {
1289 StringRef::iterator i = skip_s_white(Tmp);
1291 if (LeadingBlanks && (Column < indent) && *Tmp == '\t') {
1292 setError("Found invalid tab character in indentation", Tmp);
1298 i = skip_b_break(Tmp);
1307 if (!FlowLevel && Column < indent)
1312 if (Start == Current) {
1313 setError("Got empty plain scalar", Start);
1317 T.Kind = Token::TK_Scalar;
1318 T.Range = StringRef(Start, Current - Start);
1319 TokenQueue.push_back(T);
1321 // Plain scalars can be simple keys.
1322 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1324 IsSimpleKeyAllowed = false;
1329 bool Scanner::scanAliasOrAnchor(bool IsAlias) {
1330 StringRef::iterator Start = Current;
1331 unsigned ColStart = Column;
1334 if ( *Current == '[' || *Current == ']'
1335 || *Current == '{' || *Current == '}'
1339 StringRef::iterator i = skip_ns_char(Current);
1346 if (Start == Current) {
1347 setError("Got empty alias or anchor", Start);
1352 T.Kind = IsAlias ? Token::TK_Alias : Token::TK_Anchor;
1353 T.Range = StringRef(Start, Current - Start);
1354 TokenQueue.push_back(T);
1356 // Alias and anchors can be simple keys.
1357 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1359 IsSimpleKeyAllowed = false;
1364 bool Scanner::scanBlockScalar(bool IsLiteral) {
1365 StringRef::iterator Start = Current;
1366 skip(1); // Eat | or >
1368 StringRef::iterator i = skip_nb_char(Current);
1372 i = skip_b_break(Current);
1374 // We got a line break.
1380 // There was an error, which should already have been printed out.
1388 if (Start == Current) {
1389 setError("Got empty block scalar", Start);
1394 T.Kind = Token::TK_Scalar;
1395 T.Range = StringRef(Start, Current - Start);
1396 TokenQueue.push_back(T);
1400 bool Scanner::scanTag() {
1401 StringRef::iterator Start = Current;
1402 unsigned ColStart = Column;
1404 if (Current == End || isBlankOrBreak(Current)); // An empty tag.
1405 else if (*Current == '<') {
1411 // FIXME: Actually parse the c-ns-shorthand-tag rule.
1412 Current = skip_while(&Scanner::skip_ns_char, Current);
1416 T.Kind = Token::TK_Tag;
1417 T.Range = StringRef(Start, Current - Start);
1418 TokenQueue.push_back(T);
1420 // Tags can be simple keys.
1421 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1423 IsSimpleKeyAllowed = false;
1428 bool Scanner::fetchMoreTokens() {
1429 if (IsStartOfStream)
1430 return scanStreamStart();
1435 return scanStreamEnd();
1437 removeStaleSimpleKeyCandidates();
1439 unrollIndent(Column);
1441 if (Column == 0 && *Current == '%')
1442 return scanDirective();
1444 if (Column == 0 && Current + 4 <= End
1446 && *(Current + 1) == '-'
1447 && *(Current + 2) == '-'
1448 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1449 return scanDocumentIndicator(true);
1451 if (Column == 0 && Current + 4 <= End
1453 && *(Current + 1) == '.'
1454 && *(Current + 2) == '.'
1455 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1456 return scanDocumentIndicator(false);
1458 if (*Current == '[')
1459 return scanFlowCollectionStart(true);
1461 if (*Current == '{')
1462 return scanFlowCollectionStart(false);
1464 if (*Current == ']')
1465 return scanFlowCollectionEnd(true);
1467 if (*Current == '}')
1468 return scanFlowCollectionEnd(false);
1470 if (*Current == ',')
1471 return scanFlowEntry();
1473 if (*Current == '-' && isBlankOrBreak(Current + 1))
1474 return scanBlockEntry();
1476 if (*Current == '?' && (FlowLevel || isBlankOrBreak(Current + 1)))
1479 if (*Current == ':' && (FlowLevel || isBlankOrBreak(Current + 1)))
1482 if (*Current == '*')
1483 return scanAliasOrAnchor(true);
1485 if (*Current == '&')
1486 return scanAliasOrAnchor(false);
1488 if (*Current == '!')
1491 if (*Current == '|' && !FlowLevel)
1492 return scanBlockScalar(true);
1494 if (*Current == '>' && !FlowLevel)
1495 return scanBlockScalar(false);
1497 if (*Current == '\'')
1498 return scanFlowScalar(false);
1500 if (*Current == '"')
1501 return scanFlowScalar(true);
1503 // Get a plain scalar.
1504 StringRef FirstChar(Current, 1);
1505 if (!(isBlankOrBreak(Current)
1506 || FirstChar.find_first_of("-?:,[]{}#&*!|>'\"%@`") != StringRef::npos)
1507 || (*Current == '-' && !isBlankOrBreak(Current + 1))
1508 || (!FlowLevel && (*Current == '?' || *Current == ':')
1509 && isBlankOrBreak(Current + 1))
1510 || (!FlowLevel && *Current == ':'
1511 && Current + 2 < End
1512 && *(Current + 1) == ':'
1513 && !isBlankOrBreak(Current + 2)))
1514 return scanPlainScalar();
1516 setError("Unrecognized character while tokenizing.");
1520 Stream::Stream(StringRef Input, SourceMgr &SM)
1521 : scanner(new Scanner(Input, SM))
1524 bool Stream::failed() { return scanner->failed(); }
1526 void Stream::printError(Node *N, const Twine &Msg) {
1527 SmallVector<SMRange, 1> Ranges;
1528 Ranges.push_back(N->getSourceRange());
1529 scanner->printError( N->getSourceRange().Start
1530 , SourceMgr::DK_Error
1535 void Stream::handleYAMLDirective(const Token &t) {
1536 // TODO: Ensure version is 1.x.
1539 document_iterator Stream::begin() {
1541 report_fatal_error("Can only iterate over the stream once");
1543 // Skip Stream-Start.
1546 CurrentDoc.reset(new Document(*this));
1547 return document_iterator(CurrentDoc);
1550 document_iterator Stream::end() {
1551 return document_iterator();
1554 void Stream::skip() {
1555 for (document_iterator i = begin(), e = end(); i != e; ++i)
1559 Node::Node(unsigned int Type, OwningPtr<Document> &D, StringRef A)
1563 SMLoc Start = SMLoc::getFromPointer(peekNext().Range.begin());
1564 SourceRange = SMRange(Start, Start);
1569 Token &Node::peekNext() {
1570 return Doc->peekNext();
1573 Token Node::getNext() {
1574 return Doc->getNext();
1577 Node *Node::parseBlockNode() {
1578 return Doc->parseBlockNode();
1581 BumpPtrAllocator &Node::getAllocator() {
1582 return Doc->NodeAllocator;
1585 void Node::setError(const Twine &Msg, Token &Tok) const {
1586 Doc->setError(Msg, Tok);
1589 bool Node::failed() const {
1590 return Doc->failed();
1595 StringRef ScalarNode::getValue(SmallVectorImpl<char> &Storage) const {
1596 // TODO: Handle newlines properly. We need to remove leading whitespace.
1597 if (Value[0] == '"') { // Double quoted.
1598 // Pull off the leading and trailing "s.
1599 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1600 // Search for characters that would require unescaping the value.
1601 StringRef::size_type i = UnquotedValue.find_first_of("\\\r\n");
1602 if (i != StringRef::npos)
1603 return unescapeDoubleQuoted(UnquotedValue, i, Storage);
1604 return UnquotedValue;
1605 } else if (Value[0] == '\'') { // Single quoted.
1606 // Pull off the leading and trailing 's.
1607 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1608 StringRef::size_type i = UnquotedValue.find('\'');
1609 if (i != StringRef::npos) {
1610 // We're going to need Storage.
1612 Storage.reserve(UnquotedValue.size());
1613 for (; i != StringRef::npos; i = UnquotedValue.find('\'')) {
1614 StringRef Valid(UnquotedValue.begin(), i);
1615 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1616 Storage.push_back('\'');
1617 UnquotedValue = UnquotedValue.substr(i + 2);
1619 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1620 return StringRef(Storage.begin(), Storage.size());
1622 return UnquotedValue;
1625 size_t trimtrail = Value.rfind(' ');
1626 return Value.drop_back(
1627 trimtrail == StringRef::npos ? 0 : Value.size() - trimtrail);
1630 StringRef ScalarNode::unescapeDoubleQuoted( StringRef UnquotedValue
1631 , StringRef::size_type i
1632 , SmallVectorImpl<char> &Storage)
1634 // Use Storage to build proper value.
1636 Storage.reserve(UnquotedValue.size());
1637 for (; i != StringRef::npos; i = UnquotedValue.find_first_of("\\\r\n")) {
1638 // Insert all previous chars into Storage.
1639 StringRef Valid(UnquotedValue.begin(), i);
1640 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1641 // Chop off inserted chars.
1642 UnquotedValue = UnquotedValue.substr(i);
1644 assert(!UnquotedValue.empty() && "Can't be empty!");
1646 // Parse escape or line break.
1647 switch (UnquotedValue[0]) {
1650 Storage.push_back('\n');
1651 if ( UnquotedValue.size() > 1
1652 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1653 UnquotedValue = UnquotedValue.substr(1);
1654 UnquotedValue = UnquotedValue.substr(1);
1657 if (UnquotedValue.size() == 1)
1658 // TODO: Report error.
1660 UnquotedValue = UnquotedValue.substr(1);
1661 switch (UnquotedValue[0]) {
1664 T.Range = StringRef(UnquotedValue.begin(), 1);
1665 setError("Unrecognized escape code!", T);
1670 // Remove the new line.
1671 if ( UnquotedValue.size() > 1
1672 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1673 UnquotedValue = UnquotedValue.substr(1);
1674 // If this was just a single byte newline, it will get skipped
1678 Storage.push_back(0x00);
1681 Storage.push_back(0x07);
1684 Storage.push_back(0x08);
1688 Storage.push_back(0x09);
1691 Storage.push_back(0x0A);
1694 Storage.push_back(0x0B);
1697 Storage.push_back(0x0C);
1700 Storage.push_back(0x0D);
1703 Storage.push_back(0x1B);
1706 Storage.push_back(0x20);
1709 Storage.push_back(0x22);
1712 Storage.push_back(0x2F);
1715 Storage.push_back(0x5C);
1718 encodeUTF8(0x85, Storage);
1721 encodeUTF8(0xA0, Storage);
1724 encodeUTF8(0x2028, Storage);
1727 encodeUTF8(0x2029, Storage);
1730 if (UnquotedValue.size() < 3)
1731 // TODO: Report error.
1733 unsigned int UnicodeScalarValue;
1734 UnquotedValue.substr(1, 2).getAsInteger(16, UnicodeScalarValue);
1735 encodeUTF8(UnicodeScalarValue, Storage);
1736 UnquotedValue = UnquotedValue.substr(2);
1740 if (UnquotedValue.size() < 5)
1741 // TODO: Report error.
1743 unsigned int UnicodeScalarValue;
1744 UnquotedValue.substr(1, 4).getAsInteger(16, UnicodeScalarValue);
1745 encodeUTF8(UnicodeScalarValue, Storage);
1746 UnquotedValue = UnquotedValue.substr(4);
1750 if (UnquotedValue.size() < 9)
1751 // TODO: Report error.
1753 unsigned int UnicodeScalarValue;
1754 UnquotedValue.substr(1, 8).getAsInteger(16, UnicodeScalarValue);
1755 encodeUTF8(UnicodeScalarValue, Storage);
1756 UnquotedValue = UnquotedValue.substr(8);
1760 UnquotedValue = UnquotedValue.substr(1);
1763 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1764 return StringRef(Storage.begin(), Storage.size());
1767 Node *KeyValueNode::getKey() {
1770 // Handle implicit null keys.
1772 Token &t = peekNext();
1773 if ( t.Kind == Token::TK_BlockEnd
1774 || t.Kind == Token::TK_Value
1775 || t.Kind == Token::TK_Error) {
1776 return Key = new (getAllocator()) NullNode(Doc);
1778 if (t.Kind == Token::TK_Key)
1779 getNext(); // skip TK_Key.
1782 // Handle explicit null keys.
1783 Token &t = peekNext();
1784 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Value) {
1785 return Key = new (getAllocator()) NullNode(Doc);
1788 // We've got a normal key.
1789 return Key = parseBlockNode();
1792 Node *KeyValueNode::getValue() {
1797 return Value = new (getAllocator()) NullNode(Doc);
1799 // Handle implicit null values.
1801 Token &t = peekNext();
1802 if ( t.Kind == Token::TK_BlockEnd
1803 || t.Kind == Token::TK_FlowMappingEnd
1804 || t.Kind == Token::TK_Key
1805 || t.Kind == Token::TK_FlowEntry
1806 || t.Kind == Token::TK_Error) {
1807 return Value = new (getAllocator()) NullNode(Doc);
1810 if (t.Kind != Token::TK_Value) {
1811 setError("Unexpected token in Key Value.", t);
1812 return Value = new (getAllocator()) NullNode(Doc);
1814 getNext(); // skip TK_Value.
1817 // Handle explicit null values.
1818 Token &t = peekNext();
1819 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Key) {
1820 return Value = new (getAllocator()) NullNode(Doc);
1823 // We got a normal value.
1824 return Value = parseBlockNode();
1827 void MappingNode::increment() {
1834 CurrentEntry->skip();
1835 if (Type == MT_Inline) {
1841 Token T = peekNext();
1842 if (T.Kind == Token::TK_Key || T.Kind == Token::TK_Scalar) {
1843 // KeyValueNode eats the TK_Key. That way it can detect null keys.
1844 CurrentEntry = new (getAllocator()) KeyValueNode(Doc);
1845 } else if (Type == MT_Block) {
1847 case Token::TK_BlockEnd:
1853 setError("Unexpected token. Expected Key or Block End", T);
1854 case Token::TK_Error:
1860 case Token::TK_FlowEntry:
1861 // Eat the flow entry and recurse.
1864 case Token::TK_FlowMappingEnd:
1866 case Token::TK_Error:
1867 // Set this to end iterator.
1872 setError( "Unexpected token. Expected Key, Flow Entry, or Flow "
1881 void SequenceNode::increment() {
1888 CurrentEntry->skip();
1889 Token T = peekNext();
1890 if (SeqType == ST_Block) {
1892 case Token::TK_BlockEntry:
1894 CurrentEntry = parseBlockNode();
1895 if (CurrentEntry == 0) { // An error occurred.
1900 case Token::TK_BlockEnd:
1906 setError( "Unexpected token. Expected Block Entry or Block End."
1908 case Token::TK_Error:
1912 } else if (SeqType == ST_Indentless) {
1914 case Token::TK_BlockEntry:
1916 CurrentEntry = parseBlockNode();
1917 if (CurrentEntry == 0) { // An error occurred.
1923 case Token::TK_Error:
1927 } else if (SeqType == ST_Flow) {
1929 case Token::TK_FlowEntry:
1930 // Eat the flow entry and recurse.
1932 WasPreviousTokenFlowEntry = true;
1934 case Token::TK_FlowSequenceEnd:
1936 case Token::TK_Error:
1937 // Set this to end iterator.
1941 case Token::TK_StreamEnd:
1942 case Token::TK_DocumentEnd:
1943 case Token::TK_DocumentStart:
1944 setError("Could not find closing ]!", T);
1945 // Set this to end iterator.
1950 if (!WasPreviousTokenFlowEntry) {
1951 setError("Expected , between entries!", T);
1956 // Otherwise it must be a flow entry.
1957 CurrentEntry = parseBlockNode();
1958 if (!CurrentEntry) {
1961 WasPreviousTokenFlowEntry = false;
1967 Document::Document(Stream &S) : stream(S), Root(0) {
1968 if (parseDirectives())
1969 expectToken(Token::TK_DocumentStart);
1970 Token &T = peekNext();
1971 if (T.Kind == Token::TK_DocumentStart)
1975 bool Document::skip() {
1976 if (stream.scanner->failed())
1981 Token &T = peekNext();
1982 if (T.Kind == Token::TK_StreamEnd)
1984 if (T.Kind == Token::TK_DocumentEnd) {
1991 Token &Document::peekNext() {
1992 return stream.scanner->peekNext();
1995 Token Document::getNext() {
1996 return stream.scanner->getNext();
1999 void Document::setError(const Twine &Message, Token &Location) const {
2000 stream.scanner->setError(Message, Location.Range.begin());
2003 bool Document::failed() const {
2004 return stream.scanner->failed();
2007 Node *Document::parseBlockNode() {
2008 Token T = peekNext();
2009 // Handle properties.
2013 case Token::TK_Alias:
2015 return new (NodeAllocator) AliasNode(stream.CurrentDoc, T.Range.substr(1));
2016 case Token::TK_Anchor:
2017 if (AnchorInfo.Kind == Token::TK_Anchor) {
2018 setError("Already encountered an anchor for this node!", T);
2021 AnchorInfo = getNext(); // Consume TK_Anchor.
2023 goto parse_property;
2025 getNext(); // Skip TK_Tag.
2027 goto parse_property;
2033 case Token::TK_BlockEntry:
2034 // We got an unindented BlockEntry sequence. This is not terminated with
2036 // Don't eat the TK_BlockEntry, SequenceNode needs it.
2037 return new (NodeAllocator) SequenceNode( stream.CurrentDoc
2038 , AnchorInfo.Range.substr(1)
2039 , SequenceNode::ST_Indentless);
2040 case Token::TK_BlockSequenceStart:
2042 return new (NodeAllocator)
2043 SequenceNode( stream.CurrentDoc
2044 , AnchorInfo.Range.substr(1)
2045 , SequenceNode::ST_Block);
2046 case Token::TK_BlockMappingStart:
2048 return new (NodeAllocator)
2049 MappingNode( stream.CurrentDoc
2050 , AnchorInfo.Range.substr(1)
2051 , MappingNode::MT_Block);
2052 case Token::TK_FlowSequenceStart:
2054 return new (NodeAllocator)
2055 SequenceNode( stream.CurrentDoc
2056 , AnchorInfo.Range.substr(1)
2057 , SequenceNode::ST_Flow);
2058 case Token::TK_FlowMappingStart:
2060 return new (NodeAllocator)
2061 MappingNode( stream.CurrentDoc
2062 , AnchorInfo.Range.substr(1)
2063 , MappingNode::MT_Flow);
2064 case Token::TK_Scalar:
2066 return new (NodeAllocator)
2067 ScalarNode( stream.CurrentDoc
2068 , AnchorInfo.Range.substr(1)
2071 // Don't eat the TK_Key, KeyValueNode expects it.
2072 return new (NodeAllocator)
2073 MappingNode( stream.CurrentDoc
2074 , AnchorInfo.Range.substr(1)
2075 , MappingNode::MT_Inline);
2076 case Token::TK_DocumentStart:
2077 case Token::TK_DocumentEnd:
2078 case Token::TK_StreamEnd:
2080 // TODO: Properly handle tags. "[!!str ]" should resolve to !!str "", not
2082 return new (NodeAllocator) NullNode(stream.CurrentDoc);
2083 case Token::TK_Error:
2086 llvm_unreachable("Control flow shouldn't reach here.");
2090 bool Document::parseDirectives() {
2091 bool isDirective = false;
2093 Token T = peekNext();
2094 if (T.Kind == Token::TK_TagDirective) {
2095 handleTagDirective(getNext());
2097 } else if (T.Kind == Token::TK_VersionDirective) {
2098 stream.handleYAMLDirective(getNext());
2106 bool Document::expectToken(int TK) {
2107 Token T = getNext();
2109 setError("Unexpected token", T);
2115 OwningPtr<Document> document_iterator::NullDoc;