12: def scan_tokens tokens, options
13:
14: opened_tokens = []
15:
16: until eos?
17:
18: kind = nil
19: match = nil
20:
21: if scan(/\s+/)
22: tokens << [matched, :space]
23: next
24:
25: elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
26: kind = self[1].to_sym
27: match = self[2].gsub(/\\(.)/, '\1')
28:
29: elsif scan(/ (\w+) < /x)
30: kind = self[1].to_sym
31: opened_tokens << kind
32: match = :open
33:
34: elsif scan(/ > /x)
35: kind = opened_tokens.pop
36: match = :close
37:
38: else
39: kind = :error
40: getch
41:
42: end
43:
44: match ||= matched
45: if $DEBUG and not kind
46: raise_inspect 'Error token %p in line %d' %
47: [[match, kind], line], tokens
48: end
49: raise_inspect 'Empty token', tokens unless match
50:
51: tokens << [match, kind]
52:
53: end
54:
55: tokens
56: end