1 //===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Implement the Lexer for TableGen.
12 //===----------------------------------------------------------------------===//
15 #include "TGSourceMgr.h"
16 #include "llvm/Support/Streams.h"
17 #include "llvm/Support/MemoryBuffer.h"
19 #include "llvm/Config/config.h"
27 TGLexer::TGLexer(TGSourceMgr &SM) : SrcMgr(SM) {
29 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
30 CurPtr = CurBuf->getBufferStart();
35 /// ReturnError - Set the error to the specified string at the specified
36 /// location. This is defined to always return tgtok::Error.
37 tgtok::TokKind TGLexer::ReturnError(const char *Loc, const std::string &Msg) {
43 void TGLexer::PrintError(LocTy Loc, const std::string &Msg) const {
44 SrcMgr.PrintError(Loc, Msg);
47 int TGLexer::getNextChar() {
48 char CurChar = *CurPtr++;
51 return (unsigned char)CurChar;
53 // A nul character in the stream is either the end of the current buffer or
54 // a random nul in the file. Disambiguate that here.
55 if (CurPtr-1 != CurBuf->getBufferEnd())
56 return 0; // Just whitespace.
58 // If this is the end of an included file, pop the parent file off the
60 TGLocTy ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
61 if (ParentIncludeLoc != TGLocTy()) {
62 CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
63 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
64 CurPtr = ParentIncludeLoc;
68 // Otherwise, return end of file.
69 --CurPtr; // Another call to lex will return EOF again.
74 // Handle the newline character by ignoring it and incrementing the line
75 // count. However, be careful about 'dos style' files with \n\r in them.
76 // Only treat a \n\r or \r\n as a single line.
77 if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
79 ++CurPtr; // Eat the two char newline sequence.
84 tgtok::TokKind TGLexer::LexToken() {
86 // This always consumes at least one character.
87 int CurChar = getNextChar();
91 // Handle letters: [a-zA-Z_]
92 if (isalpha(CurChar) || CurChar == '_')
93 return LexIdentifier();
95 // Unknown character, emit an error.
96 return ReturnError(TokStart, "Unexpected character");
97 case EOF: return tgtok::Eof;
98 case ':': return tgtok::colon;
99 case ';': return tgtok::semi;
100 case '.': return tgtok::period;
101 case ',': return tgtok::comma;
102 case '<': return tgtok::less;
103 case '>': return tgtok::greater;
104 case ']': return tgtok::r_square;
105 case '{': return tgtok::l_brace;
106 case '}': return tgtok::r_brace;
107 case '(': return tgtok::l_paren;
108 case ')': return tgtok::r_paren;
109 case '=': return tgtok::equal;
110 case '?': return tgtok::question;
117 // Ignore whitespace.
120 // If this is the start of a // comment, skip until the end of the line or
121 // the end of the buffer.
124 else if (*CurPtr == '*') {
127 } else // Otherwise, this is an error.
128 return ReturnError(TokStart, "Unexpected character");
131 case '0': case '1': case '2': case '3': case '4': case '5': case '6':
132 case '7': case '8': case '9':
134 case '"': return LexString();
135 case '$': return LexVarName();
136 case '[': return LexBracket();
137 case '!': return LexExclaim();
141 /// LexString - Lex "[^"]*"
142 tgtok::TokKind TGLexer::LexString() {
143 const char *StrStart = CurPtr;
145 while (*CurPtr != '"') {
146 // If we hit the end of the buffer, report an error.
147 if (*CurPtr == 0 && CurPtr == CurBuf->getBufferEnd())
148 return ReturnError(StrStart, "End of file in string literal");
150 if (*CurPtr == '\n' || *CurPtr == '\r')
151 return ReturnError(StrStart, "End of line in string literal");
156 CurStrVal.assign(StrStart, CurPtr);
158 return tgtok::StrVal;
161 tgtok::TokKind TGLexer::LexVarName() {
162 if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
163 return ReturnError(TokStart, "Invalid variable name");
165 // Otherwise, we're ok, consume the rest of the characters.
166 const char *VarNameStart = CurPtr++;
168 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
171 CurStrVal.assign(VarNameStart, CurPtr);
172 return tgtok::VarName;
176 tgtok::TokKind TGLexer::LexIdentifier() {
177 // The first letter is [a-zA-Z_].
178 const char *IdentStart = TokStart;
180 // Match the rest of the identifier regex: [0-9a-zA-Z_]*
181 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
184 // Check to see if this identifier is a keyword.
185 unsigned Len = CurPtr-IdentStart;
187 if (Len == 3 && !memcmp(IdentStart, "int", 3)) return tgtok::Int;
188 if (Len == 3 && !memcmp(IdentStart, "bit", 3)) return tgtok::Bit;
189 if (Len == 4 && !memcmp(IdentStart, "bits", 4)) return tgtok::Bits;
190 if (Len == 6 && !memcmp(IdentStart, "string", 6)) return tgtok::String;
191 if (Len == 4 && !memcmp(IdentStart, "list", 4)) return tgtok::List;
192 if (Len == 4 && !memcmp(IdentStart, "code", 4)) return tgtok::Code;
193 if (Len == 3 && !memcmp(IdentStart, "dag", 3)) return tgtok::Dag;
195 if (Len == 5 && !memcmp(IdentStart, "class", 5)) return tgtok::Class;
196 if (Len == 3 && !memcmp(IdentStart, "def", 3)) return tgtok::Def;
197 if (Len == 4 && !memcmp(IdentStart, "defm", 4)) return tgtok::Defm;
198 if (Len == 10 && !memcmp(IdentStart, "multiclass", 10))
199 return tgtok::MultiClass;
200 if (Len == 5 && !memcmp(IdentStart, "field", 5)) return tgtok::Field;
201 if (Len == 3 && !memcmp(IdentStart, "let", 3)) return tgtok::Let;
202 if (Len == 2 && !memcmp(IdentStart, "in", 2)) return tgtok::In;
204 if (Len == 7 && !memcmp(IdentStart, "include", 7)) {
205 if (LexInclude()) return tgtok::Error;
209 CurStrVal.assign(IdentStart, CurPtr);
213 /// LexInclude - We just read the "include" token. Get the string token that
214 /// comes next and enter the include.
215 bool TGLexer::LexInclude() {
216 // The token after the include must be a string.
217 tgtok::TokKind Tok = LexToken();
218 if (Tok == tgtok::Error) return true;
219 if (Tok != tgtok::StrVal) {
220 PrintError(getLoc(), "Expected filename after include");
225 std::string Filename = CurStrVal;
227 // Try to find the file.
228 MemoryBuffer *NewBuf = MemoryBuffer::getFile(Filename.c_str());
230 // If the file didn't exist directly, see if it's in an include path.
231 for (unsigned i = 0, e = IncludeDirectories.size(); i != e && !NewBuf; ++i) {
232 std::string IncFile = IncludeDirectories[i] + "/" + Filename;
233 NewBuf = MemoryBuffer::getFile(IncFile.c_str());
237 PrintError(getLoc(), "Could not find include file '" + Filename + "'");
241 // Save the line number and lex buffer of the includer.
242 CurBuffer = SrcMgr.AddNewSourceBuffer(NewBuf, CurPtr);
245 CurPtr = CurBuf->getBufferStart();
249 void TGLexer::SkipBCPLComment() {
250 ++CurPtr; // skip the second slash.
255 return; // Newline is end of comment.
257 // If this is the end of the buffer, end the comment.
258 if (CurPtr == CurBuf->getBufferEnd())
262 // Otherwise, skip the character.
267 /// SkipCComment - This skips C-style /**/ comments. The only difference from C
268 /// is that we allow nesting.
269 bool TGLexer::SkipCComment() {
270 ++CurPtr; // skip the star.
271 unsigned CommentDepth = 1;
274 int CurChar = getNextChar();
277 PrintError(TokStart, "Unterminated comment!");
280 // End of the comment?
281 if (CurPtr[0] != '/') break;
283 ++CurPtr; // End the */.
284 if (--CommentDepth == 0)
288 // Start of a nested comment?
289 if (CurPtr[0] != '*') break;
301 tgtok::TokKind TGLexer::LexNumber() {
302 if (CurPtr[-1] == '0') {
303 if (CurPtr[0] == 'x') {
305 const char *NumStart = CurPtr;
306 while (isxdigit(CurPtr[0]))
309 // Requires at least one hex digit.
310 if (CurPtr == NumStart)
311 return ReturnError(CurPtr-2, "Invalid hexadecimal number");
314 CurIntVal = strtoll(NumStart, 0, 16);
316 return ReturnError(CurPtr-2, "Invalid hexadecimal number");
317 if (errno == ERANGE) {
319 CurIntVal = (int64_t)strtoull(NumStart, 0, 16);
321 return ReturnError(CurPtr-2, "Invalid hexadecimal number");
323 return ReturnError(CurPtr-2, "Hexadecimal number out of range");
325 return tgtok::IntVal;
326 } else if (CurPtr[0] == 'b') {
328 const char *NumStart = CurPtr;
329 while (CurPtr[0] == '0' || CurPtr[0] == '1')
332 // Requires at least one binary digit.
333 if (CurPtr == NumStart)
334 return ReturnError(CurPtr-2, "Invalid binary number");
335 CurIntVal = strtoll(NumStart, 0, 2);
336 return tgtok::IntVal;
340 // Check for a sign without a digit.
341 if (!isdigit(CurPtr[0])) {
342 if (CurPtr[-1] == '-')
344 else if (CurPtr[-1] == '+')
348 while (isdigit(CurPtr[0]))
350 CurIntVal = strtoll(TokStart, 0, 10);
351 return tgtok::IntVal;
354 /// LexBracket - We just read '['. If this is a code block, return it,
355 /// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
356 tgtok::TokKind TGLexer::LexBracket() {
357 if (CurPtr[0] != '{')
358 return tgtok::l_square;
360 const char *CodeStart = CurPtr;
362 int Char = getNextChar();
363 if (Char == EOF) break;
365 if (Char != '}') continue;
367 Char = getNextChar();
368 if (Char == EOF) break;
370 CurStrVal.assign(CodeStart, CurPtr-2);
371 return tgtok::CodeFragment;
375 return ReturnError(CodeStart-2, "Unterminated Code Block");
378 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
379 tgtok::TokKind TGLexer::LexExclaim() {
380 if (!isalpha(*CurPtr))
381 return ReturnError(CurPtr-1, "Invalid \"!operator\"");
383 const char *Start = CurPtr++;
384 while (isalpha(*CurPtr))
387 // Check to see which operator this is.
388 unsigned Len = CurPtr-Start;
390 if (Len == 3 && !memcmp(Start, "con", 3)) return tgtok::XConcat;
391 if (Len == 3 && !memcmp(Start, "sra", 3)) return tgtok::XSRA;
392 if (Len == 3 && !memcmp(Start, "srl", 3)) return tgtok::XSRL;
393 if (Len == 3 && !memcmp(Start, "shl", 3)) return tgtok::XSHL;
394 if (Len == 9 && !memcmp(Start, "strconcat", 9)) return tgtok::XStrConcat;
396 return ReturnError(Start-1, "Unknown operator");