1 //===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file was developed by Chris Lattner and is distributed under
6 // the University of Illinois Open Source License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Implement the Lexer for TableGen.
12 //===----------------------------------------------------------------------===//
16 #include "llvm/Support/Streams.h"
18 #include "llvm/Support/MemoryBuffer.h"
19 typedef std::pair<llvm::Record*, std::vector<llvm::Init*>*> SubClassRefTy;
20 #include "FileParser.h"
24 // FIXME: REMOVE THIS.
28 TGLexer::TGLexer(MemoryBuffer *StartBuf) : CurLineNo(1), CurBuf(StartBuf) {
29 CurPtr = CurBuf->getBufferStart();
34 while (!IncludeStack.empty()) {
35 delete IncludeStack.back().Buffer;
36 IncludeStack.pop_back();
41 /// ReturnError - Set the error to the specified string at the specified
42 /// location. This is defined to always return YYERROR.
43 int TGLexer::ReturnError(const char *Loc, const std::string &Msg) {
48 std::ostream &TGLexer::err() const {
49 PrintIncludeStack(*cerr.stream());
50 return *cerr.stream();
54 void TGLexer::PrintIncludeStack(std::ostream &OS) const {
55 for (unsigned i = 0, e = IncludeStack.size(); i != e; ++i)
56 OS << "Included from " << IncludeStack[i].Buffer->getBufferIdentifier()
57 << ":" << IncludeStack[i].LineNo << ":\n";
58 OS << "Parsing " << CurBuf->getBufferIdentifier() << ":"
62 /// PrintError - Print the error at the specified location.
63 void TGLexer::PrintError(const char *ErrorLoc, const std::string &Msg) const {
65 assert(ErrorLoc && "Location not specified!");
67 // Scan backward to find the start of the line.
68 const char *LineStart = ErrorLoc;
69 while (LineStart != CurBuf->getBufferStart() &&
70 LineStart[-1] != '\n' && LineStart[-1] != '\r')
72 // Get the end of the line.
73 const char *LineEnd = ErrorLoc;
74 while (LineEnd != CurBuf->getBufferEnd() &&
75 LineEnd[0] != '\n' && LineEnd[0] != '\r')
77 // Print out the line.
78 cerr << std::string(LineStart, LineEnd) << "\n";
79 // Print out spaces before the carat.
80 for (const char *Pos = LineStart; Pos != ErrorLoc; ++Pos)
81 cerr << (*Pos == '\t' ? '\t' : ' ');
85 int TGLexer::getNextChar() {
86 char CurChar = *CurPtr++;
89 return (unsigned char)CurChar;
91 // A nul character in the stream is either the end of the current buffer or
92 // a random nul in the file. Disambiguate that here.
93 if (CurPtr-1 != CurBuf->getBufferEnd())
94 return 0; // Just whitespace.
96 // If this is the end of an included file, pop the parent file off the
98 if (!IncludeStack.empty()) {
100 CurBuf = IncludeStack.back().Buffer;
101 CurLineNo = IncludeStack.back().LineNo;
102 CurPtr = IncludeStack.back().CurPtr;
103 IncludeStack.pop_back();
104 return getNextChar();
107 // Otherwise, return end of file.
108 --CurPtr; // Another call to lex will return EOF again.
112 // Handle the newline character by ignoring it and incrementing the line
113 // count. However, be careful about 'dos style' files with \n\r in them.
114 // Only treat a \n\r or \r\n as a single line.
115 if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
117 ++CurPtr; // Eat the two char newline sequence.
124 int TGLexer::LexToken() {
126 // This always consumes at least one character.
127 int CurChar = getNextChar();
131 // Handle letters: [a-zA-Z_]
132 if (isalpha(CurChar) || CurChar == '_')
133 return LexIdentifier();
135 // Unknown character, return the char itself.
136 return (unsigned char)CurChar;
137 case EOF: return YYEOF;
143 // Ignore whitespace.
146 // If this is the start of a // comment, skip until the end of the line or
147 // the end of the buffer.
150 else if (*CurPtr == '*') {
153 } else // Otherwise, return this / as a token.
157 case '0': case '1': case '2': case '3': case '4': case '5': case '6':
158 case '7': case '8': case '9':
160 case '"': return LexString();
161 case '$': return LexVarName();
162 case '[': return LexBracket();
163 case '!': return LexExclaim();
167 /// LexString - Lex "[^"]*"
168 int TGLexer::LexString() {
169 const char *StrStart = CurPtr;
171 while (*CurPtr != '"') {
172 // If we hit the end of the buffer, report an error.
173 if (*CurPtr == 0 && CurPtr == CurBuf->getBufferEnd())
174 return ReturnError(StrStart, "End of file in string literal");
176 if (*CurPtr == '\n' || *CurPtr == '\r')
177 return ReturnError(StrStart, "End of line in string literal");
182 Filelval.StrVal = new std::string(StrStart, CurPtr);
187 int TGLexer::LexVarName() {
188 if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
189 return '$'; // Invalid varname.
191 // Otherwise, we're ok, consume the rest of the characters.
192 const char *VarNameStart = CurPtr++;
194 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
197 Filelval.StrVal = new std::string(VarNameStart, CurPtr);
202 int TGLexer::LexIdentifier() {
203 // The first letter is [a-zA-Z_].
204 const char *IdentStart = CurPtr-1;
206 // Match the rest of the identifier regex: [0-9a-zA-Z_]*
207 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
210 // Check to see if this identifier is a keyword.
211 unsigned Len = CurPtr-IdentStart;
213 if (Len == 3 && !memcmp(IdentStart, "int", 3)) return INT;
214 if (Len == 3 && !memcmp(IdentStart, "bit", 3)) return BIT;
215 if (Len == 4 && !memcmp(IdentStart, "bits", 4)) return BITS;
216 if (Len == 6 && !memcmp(IdentStart, "string", 6)) return STRING;
217 if (Len == 4 && !memcmp(IdentStart, "list", 4)) return LIST;
218 if (Len == 4 && !memcmp(IdentStart, "code", 4)) return CODE;
219 if (Len == 3 && !memcmp(IdentStart, "dag", 3)) return DAG;
221 if (Len == 5 && !memcmp(IdentStart, "class", 5)) return CLASS;
222 if (Len == 3 && !memcmp(IdentStart, "def", 3)) return DEF;
223 if (Len == 4 && !memcmp(IdentStart, "defm", 4)) return DEFM;
224 if (Len == 10 && !memcmp(IdentStart, "multiclass", 10)) return MULTICLASS;
225 if (Len == 5 && !memcmp(IdentStart, "field", 5)) return FIELD;
226 if (Len == 3 && !memcmp(IdentStart, "let", 3)) return LET;
227 if (Len == 2 && !memcmp(IdentStart, "in", 2)) return IN;
229 if (Len == 7 && !memcmp(IdentStart, "include", 7)) {
230 if (LexInclude()) return YYERROR;
234 Filelval.StrVal = new std::string(IdentStart, CurPtr);
238 /// LexInclude - We just read the "include" token. Get the string token that
239 /// comes next and enter the include.
240 bool TGLexer::LexInclude() {
241 // The token after the include must be a string.
242 int Tok = LexToken();
243 if (Tok == YYERROR) return true;
245 PrintError(getTokenStart(), "Expected filename after include");
250 std::string Filename = *Filelval.StrVal;
251 delete Filelval.StrVal;
253 // Try to find the file.
254 MemoryBuffer *NewBuf = MemoryBuffer::getFile(&Filename[0], Filename.size());
256 // If the file didn't exist directly, see if it's in an include path.
257 for (unsigned i = 0, e = IncludeDirectories.size(); i != e && !NewBuf; ++i) {
258 std::string IncFile = IncludeDirectories[i] + "/" + Filename;
259 NewBuf = MemoryBuffer::getFile(&IncFile[0], IncFile.size());
263 PrintError(getTokenStart(),
264 "Could not find include file '" + Filename + "'");
268 // Save the line number and lex buffer of the includer.
269 IncludeStack.push_back(IncludeRec(CurBuf, CurPtr, CurLineNo));
271 CurLineNo = 1; // Reset line numbering.
273 CurPtr = CurBuf->getBufferStart();
277 void TGLexer::SkipBCPLComment() {
278 ++CurPtr; // skip the second slash.
283 return; // Newline is end of comment.
285 // If this is the end of the buffer, end the comment.
286 if (CurPtr == CurBuf->getBufferEnd())
290 // Otherwise, skip the character.
295 /// SkipCComment - This skips C-style /**/ comments. The only difference from C
296 /// is that we allow nesting.
297 bool TGLexer::SkipCComment() {
298 const char *CommentStart = CurPtr-1;
299 ++CurPtr; // skip the star.
300 unsigned CommentDepth = 1;
303 int CurChar = getNextChar();
306 PrintError(CommentStart, "Unterminated comment!");
309 // End of the comment?
310 if (CurPtr[0] != '/') break;
312 ++CurPtr; // End the */.
313 if (--CommentDepth == 0)
317 // Start of a nested comment?
318 if (CurPtr[0] != '*') break;
330 int TGLexer::LexNumber() {
331 const char *NumStart = CurPtr-1;
333 if (CurPtr[-1] == '0') {
334 if (CurPtr[0] == 'x') {
337 while (isxdigit(CurPtr[0]))
340 // Requires at least one hex digit.
341 if (CurPtr == NumStart)
342 return ReturnError(CurPtr-2, "Invalid hexadecimal number");
344 Filelval.IntVal = strtoll(NumStart, 0, 16);
346 } else if (CurPtr[0] == 'b') {
349 while (CurPtr[0] == '0' || CurPtr[0] == '1')
352 // Requires at least one binary digit.
353 if (CurPtr == NumStart)
354 return ReturnError(CurPtr-2, "Invalid binary number");
355 Filelval.IntVal = strtoll(NumStart, 0, 2);
360 // Check for a sign without a digit.
361 if (CurPtr[-1] == '-' || CurPtr[-1] == '+') {
362 if (!isdigit(CurPtr[0]))
366 while (isdigit(CurPtr[0]))
368 Filelval.IntVal = strtoll(NumStart, 0, 10);
372 /// LexBracket - We just read '['. If this is a code block, return it,
373 /// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
374 int TGLexer::LexBracket() {
375 if (CurPtr[0] != '{')
378 const char *CodeStart = CurPtr;
380 int Char = getNextChar();
381 if (Char == EOF) break;
383 if (Char != '}') continue;
385 Char = getNextChar();
386 if (Char == EOF) break;
388 Filelval.StrVal = new std::string(CodeStart, CurPtr-2);
393 return ReturnError(CodeStart-2, "Unterminated Code Block");
396 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
397 int TGLexer::LexExclaim() {
398 if (!isalpha(*CurPtr))
401 const char *Start = CurPtr++;
402 while (isalpha(*CurPtr))
405 // Check to see which operator this is.
406 unsigned Len = CurPtr-Start;
408 if (Len == 3 && !memcmp(Start, "con", 3)) return CONCATTOK;
409 if (Len == 3 && !memcmp(Start, "sra", 3)) return SRATOK;
410 if (Len == 3 && !memcmp(Start, "srl", 3)) return SRLTOK;
411 if (Len == 3 && !memcmp(Start, "shl", 3)) return SHLTOK;
412 if (Len == 9 && !memcmp(Start, "strconcat", 9)) return STRCONCATTOK;
414 return ReturnError(Start-1, "Unknown operator");
417 //===----------------------------------------------------------------------===//
418 // Interfaces used by the Bison parser.
419 //===----------------------------------------------------------------------===//
422 static TGLexer *TheLexer;
426 std::ostream &err() {
427 return TheLexer->err();
430 /// ParseFile - this function begins the parsing of the specified tablegen
433 void ParseFile(const std::string &Filename,
434 const std::vector<std::string> &IncludeDirs) {
435 std::string ErrorStr;
436 MemoryBuffer *F = MemoryBuffer::getFileOrSTDIN(&Filename[0], Filename.size(),
439 cerr << "Could not open input file '" + Filename + "': " << ErrorStr <<"\n";
443 assert(!TheLexer && "Lexer isn't reentrant yet!");
444 TheLexer = new TGLexer(F);
446 // Record the location of the include directory so that the lexer can find
448 TheLexer->setIncludeDirs(IncludeDirs);
456 } // End llvm namespace
460 assert(TheLexer && "No lexer setup yet!");
461 int Tok = TheLexer->LexToken();