utils/lit/lit/ShUtil.py

   1 import itertools
   2
   3 import Util
   4 from ShCommands import Command, Pipeline, Seq
   5
   6 class ShLexer:
   7     def __init__(self, data, win32Escapes = False):
   8         self.data = data
   9         self.pos = 0
  10         self.end = len(data)
  11         self.win32Escapes = win32Escapes
  12
  13     def eat(self):
  14         c = self.data[self.pos]
  15         self.pos += 1
  16         return c
  17
  18     def look(self):
  19         return self.data[self.pos]
  20
  21     def maybe_eat(self, c):
  22         """
  23         maybe_eat(c) - Consume the character c if it is the next character,
  24         returning True if a character was consumed. """
  25         if self.data[self.pos] == c:
  26             self.pos += 1
  27             return True
  28         return False
  29
  30     def lex_arg_fast(self, c):
  31         # Get the leading whitespace free section.
  32         chunk = self.data[self.pos - 1:].split(None, 1)[0]
  33
  34         # If it has special characters, the fast path failed.
  35         if ('|' in chunk or '&' in chunk or
  36             '<' in chunk or '>' in chunk or
  37             "'" in chunk or '"' in chunk or
  38             ';' in chunk or '\\' in chunk):
  39             return None
  40
  41         self.pos = self.pos - 1 + len(chunk)
  42         return chunk
  43
  44     def lex_arg_slow(self, c):
  45         if c in "'\"":
  46             str = self.lex_arg_quoted(c)
  47         else:
  48             str = c
  49         while self.pos != self.end:
  50             c = self.look()
  51             if c.isspace() or c in "|&;":
  52                 break
  53             elif c in '><':
  54                 # This is an annoying case; we treat '2>' as a single token so
  55                 # we don't have to track whitespace tokens.
  56
  57                 # If the parse string isn't an integer, do the usual thing.
  58                 if not str.isdigit():
  59                     break
  60
  61                 # Otherwise, lex the operator and convert to a redirection
  62                 # token.
  63                 num = int(str)
  64                 tok = self.lex_one_token()
  65                 assert isinstance(tok, tuple) and len(tok) == 1
  66                 return (tok[0], num)
  67             elif c == '"':
  68                 self.eat()
  69                 str += self.lex_arg_quoted('"')
  70             elif c == "'":
  71                 self.eat()
  72                 str += self.lex_arg_quoted("'")
  73             elif not self.win32Escapes and c == '\\':
  74                 # Outside of a string, '\\' escapes everything.
  75                 self.eat()
  76                 if self.pos == self.end:
  77                     Util.warning("escape at end of quoted argument in: %r" %
  78                                  self.data)
  79                     return str
  80                 str += self.eat()
  81             else:
  82                 str += self.eat()
  83         return str
  84
  85     def lex_arg_quoted(self, delim):
  86         str = ''
  87         while self.pos != self.end:
  88             c = self.eat()
  89             if c == delim:
  90                 return str
  91             elif c == '\\' and delim == '"':
  92                 # Inside a '"' quoted string, '\\' only escapes the quote
  93                 # character and backslash, otherwise it is preserved.
  94                 if self.pos == self.end:
  95                     Util.warning("escape at end of quoted argument in: %r" %
  96                                  self.data)
  97                     return str
  98                 c = self.eat()
  99                 if c == '"': #
 100                     str += '"'
 101                 elif c == '\\':
 102                     str += '\\'
 103                 else:
 104                     str += '\\' + c
 105             else:
 106                 str += c
 107         Util.warning("missing quote character in %r" % self.data)
 108         return str
 109
 110     def lex_arg_checked(self, c):
 111         pos = self.pos
 112         res = self.lex_arg_fast(c)
 113         end = self.pos
 114
 115         self.pos = pos
 116         reference = self.lex_arg_slow(c)
 117         if res is not None:
 118             if res != reference:
 119                 raise ValueError,"Fast path failure: %r != %r" % (res, reference)
 120             if self.pos != end:
 121                 raise ValueError,"Fast path failure: %r != %r" % (self.pos, end)
 122         return reference
 123
 124     def lex_arg(self, c):
 125         return self.lex_arg_fast(c) or self.lex_arg_slow(c)
 126
 127     def lex_one_token(self):
 128         """
 129         lex_one_token - Lex a single 'sh' token. """
 130
 131         c = self.eat()
 132         if c == ';':
 133             return (c,)
 134         if c == '|':
 135             if self.maybe_eat('|'):
 136                 return ('||',)
 137             return (c,)
 138         if c == '&':
 139             if self.maybe_eat('&'):
 140                 return ('&&',)
 141             if self.maybe_eat('>'):
 142                 return ('&>',)
 143             return (c,)
 144         if c == '>':
 145             if self.maybe_eat('&'):
 146                 return ('>&',)
 147             if self.maybe_eat('>'):
 148                 return ('>>',)
 149             return (c,)
 150         if c == '<':
 151             if self.maybe_eat('&'):
 152                 return ('<&',)
 153             if self.maybe_eat('>'):
 154                 return ('<<',)
 155             return (c,)
 156
 157         return self.lex_arg(c)
 158
 159     def lex(self):
 160         while self.pos != self.end:
 161             if self.look().isspace():
 162                 self.eat()
 163             else:
 164                 yield self.lex_one_token()
 165
 166 ###
 167
 168 class ShParser:
 169     def __init__(self, data, win32Escapes = False, pipefail = False):
 170         self.data = data
 171         self.pipefail = pipefail
 172         self.tokens = ShLexer(data, win32Escapes = win32Escapes).lex()
 173
 174     def lex(self):
 175         try:
 176             return self.tokens.next()
 177         except StopIteration:
 178             return None
 179
 180     def look(self):
 181         next = self.lex()
 182         if next is not None:
 183             self.tokens = itertools.chain([next], self.tokens)
 184         return next
 185
 186     def parse_command(self):
 187         tok = self.lex()
 188         if not tok:
 189             raise ValueError,"empty command!"
 190         if isinstance(tok, tuple):
 191             raise ValueError,"syntax error near unexpected token %r" % tok[0]
 192
 193         args = [tok]
 194         redirects = []
 195         while 1:
 196             tok = self.look()
 197
 198             # EOF?
 199             if tok is None:
 200                 break
 201
 202             # If this is an argument, just add it to the current command.
 203             if isinstance(tok, str):
 204                 args.append(self.lex())
 205                 continue
 206
 207             # Otherwise see if it is a terminator.
 208             assert isinstance(tok, tuple)
 209             if tok[0] in ('|',';','&','||','&&'):
 210                 break
 211
 212             # Otherwise it must be a redirection.
 213             op = self.lex()
 214             arg = self.lex()
 215             if not arg:
 216                 raise ValueError,"syntax error near token %r" % op[0]
 217             redirects.append((op, arg))
 218
 219         return Command(args, redirects)
 220
 221     def parse_pipeline(self):
 222         negate = False
 223
 224         commands = [self.parse_command()]
 225         while self.look() == ('|',):
 226             self.lex()
 227             commands.append(self.parse_command())
 228         return Pipeline(commands, negate, self.pipefail)
 229
 230     def parse(self):
 231         lhs = self.parse_pipeline()
 232
 233         while self.look():
 234             operator = self.lex()
 235             assert isinstance(operator, tuple) and len(operator) == 1
 236
 237             if not self.look():
 238                 raise ValueError, "missing argument to operator %r" % operator[0]
 239
 240             # FIXME: Operator precedence!!
 241             lhs = Seq(lhs, operator[0], self.parse_pipeline())
 242
 243         return lhs
 244
 245 ###
 246
 247 import unittest
 248
 249 class TestShLexer(unittest.TestCase):
 250     def lex(self, str, *args, **kwargs):
 251         return list(ShLexer(str, *args, **kwargs).lex())
 252
 253     def test_basic(self):
 254         self.assertEqual(self.lex('a|b>c&d<e;f'),
 255                          ['a', ('|',), 'b', ('>',), 'c', ('&',), 'd',
 256                           ('<',), 'e', (';',), 'f'])
 257
 258     def test_redirection_tokens(self):
 259         self.assertEqual(self.lex('a2>c'),
 260                          ['a2', ('>',), 'c'])
 261         self.assertEqual(self.lex('a 2>c'),
 262                          ['a', ('>',2), 'c'])
 263
 264     def test_quoting(self):
 265         self.assertEqual(self.lex(""" 'a' """),
 266                          ['a'])
 267         self.assertEqual(self.lex(""" "hello\\"world" """),
 268                          ['hello"world'])
 269         self.assertEqual(self.lex(""" "hello\\'world" """),
 270                          ["hello\\'world"])
 271         self.assertEqual(self.lex(""" "hello\\\\world" """),
 272                          ["hello\\world"])
 273         self.assertEqual(self.lex(""" he"llo wo"rld """),
 274                          ["hello world"])
 275         self.assertEqual(self.lex(""" a\\ b a\\\\b """),
 276                          ["a b", "a\\b"])
 277         self.assertEqual(self.lex(""" "" "" """),
 278                          ["", ""])
 279         self.assertEqual(self.lex(""" a\\ b """, win32Escapes = True),
 280                          ['a\\', 'b'])
 281
 282 class TestShParse(unittest.TestCase):
 283     def parse(self, str):
 284         return ShParser(str).parse()
 285
 286     def test_basic(self):
 287         self.assertEqual(self.parse('echo hello'),
 288                          Pipeline([Command(['echo', 'hello'], [])], False))
 289         self.assertEqual(self.parse('echo ""'),
 290                          Pipeline([Command(['echo', ''], [])], False))
 291         self.assertEqual(self.parse("""echo -DFOO='a'"""),
 292                          Pipeline([Command(['echo', '-DFOO=a'], [])], False))
 293         self.assertEqual(self.parse('echo -DFOO="a"'),
 294                          Pipeline([Command(['echo', '-DFOO=a'], [])], False))
 295
 296     def test_redirection(self):
 297         self.assertEqual(self.parse('echo hello > c'),
 298                          Pipeline([Command(['echo', 'hello'],
 299                                            [((('>'),), 'c')])], False))
 300         self.assertEqual(self.parse('echo hello > c >> d'),
 301                          Pipeline([Command(['echo', 'hello'], [(('>',), 'c'),
 302                                                      (('>>',), 'd')])], False))
 303         self.assertEqual(self.parse('a 2>&1'),
 304                          Pipeline([Command(['a'], [(('>&',2), '1')])], False))
 305
 306     def test_pipeline(self):
 307         self.assertEqual(self.parse('a | b'),
 308                          Pipeline([Command(['a'], []),
 309                                    Command(['b'], [])],
 310                                   False))
 311
 312         self.assertEqual(self.parse('a | b | c'),
 313                          Pipeline([Command(['a'], []),
 314                                    Command(['b'], []),
 315                                    Command(['c'], [])],
 316                                   False))
 317
 318     def test_list(self):
 319         self.assertEqual(self.parse('a ; b'),
 320                          Seq(Pipeline([Command(['a'], [])], False),
 321                              ';',
 322                              Pipeline([Command(['b'], [])], False)))
 323
 324         self.assertEqual(self.parse('a & b'),
 325                          Seq(Pipeline([Command(['a'], [])], False),
 326                              '&',
 327                              Pipeline([Command(['b'], [])], False)))
 328
 329         self.assertEqual(self.parse('a && b'),
 330                          Seq(Pipeline([Command(['a'], [])], False),
 331                              '&&',
 332                              Pipeline([Command(['b'], [])], False)))
 333
 334         self.assertEqual(self.parse('a || b'),
 335                          Seq(Pipeline([Command(['a'], [])], False),
 336                              '||',
 337                              Pipeline([Command(['b'], [])], False)))
 338
 339         self.assertEqual(self.parse('a && b || c'),
 340                          Seq(Seq(Pipeline([Command(['a'], [])], False),
 341                                  '&&',
 342                                  Pipeline([Command(['b'], [])], False)),
 343                              '||',
 344                              Pipeline([Command(['c'], [])], False)))
 345
 346         self.assertEqual(self.parse('a; b'),
 347                          Seq(Pipeline([Command(['a'], [])], False),
 348                              ';',
 349                              Pipeline([Command(['b'], [])], False)))
 350
 351 if __name__ == '__main__':
 352     unittest.main()