45: def scan_tokens tokens, options
46:
47: value_expected = nil
48: states = [:initial]
49:
50: until eos?
51:
52: kind = nil
53: match = nil
54:
55: if scan(/\s+/)
56: kind = :space
57:
58: elsif case states.last
59: when :initial, :media
60: if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
61: kind = :keyword
62: elsif scan RE::Class
63: kind = :class
64: elsif scan RE::Id
65: kind = :constant
66: elsif scan RE::PseudoClass
67: kind = :pseudo_class
68: elsif match = scan(RE::AttributeSelector)
69:
70: tokens << [:open, :string]
71: tokens << [match[0,1], :delimiter]
72: tokens << [match[1..-2], :content] if match.size > 2
73: tokens << [match[-1,1], :delimiter] if match[-1] == ?]
74: tokens << [:close, :string]
75: next
76: elsif match = scan(/@media/)
77: kind = :directive
78: states.push :media_before_name
79: end
80:
81: when :block
82: if scan(/(?>#{RE::Ident})(?!\()/ox)
83: if value_expected
84: kind = :value
85: else
86: kind = :key
87: end
88: end
89:
90: when :media_before_name
91: if scan RE::Ident
92: kind = :type
93: states[-1] = :media_after_name
94: end
95:
96: when :media_after_name
97: if scan(/\{/)
98: kind = :operator
99: states[-1] = :media
100: end
101:
102: when :comment
103: if scan(/(?:[^*\s]|\*(?!\/))+/)
104: kind = :comment
105: elsif scan(/\*\//)
106: kind = :comment
107: states.pop
108: elsif scan(/\s+/)
109: kind = :space
110: end
111:
112: else
113: raise_inspect 'Unknown state', tokens
114:
115: end
116:
117: elsif scan(/\/\*/)
118: kind = :comment
119: states.push :comment
120:
121: elsif scan(/\{/)
122: value_expected = false
123: kind = :operator
124: states.push :block
125:
126: elsif scan(/\}/)
127: value_expected = false
128: if states.last == :block || states.last == :media
129: kind = :operator
130: states.pop
131: else
132: kind = :error
133: end
134:
135: elsif match = scan(/#{RE::String}/o)
136: tokens << [:open, :string]
137: tokens << [match[0, 1], :delimiter]
138: tokens << [match[1..-2], :content] if match.size > 2
139: tokens << [match[-1, 1], :delimiter] if match.size >= 2
140: tokens << [:close, :string]
141: next
142:
143: elsif match = scan(/#{RE::Function}/o)
144: tokens << [:open, :string]
145: start = match[/^\w+\(/]
146: tokens << [start, :delimiter]
147: if match[-1] == ?)
148: tokens << [match[start.size..-2], :content]
149: tokens << [')', :delimiter]
150: else
151: tokens << [match[start.size..-1], :content]
152: end
153: tokens << [:close, :string]
154: next
155:
156: elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
157: kind = :float
158:
159: elsif scan(/#{RE::Color}/o)
160: kind = :color
161:
162: elsif scan(/! *important/)
163: kind = :important
164:
165: elsif scan(/rgb\([^()\n]*\)?/)
166: kind = :color
167:
168: elsif scan(/#{RE::AtKeyword}/o)
169: kind = :directive
170:
171: elsif match = scan(/ [+>:;,.=()\/] /x)
172: if match == ':'
173: value_expected = true
174: elsif match == ';'
175: value_expected = false
176: end
177: kind = :operator
178:
179: else
180: getch
181: kind = :error
182:
183: end
184:
185: match ||= matched
186: if $DEBUG and not kind
187: raise_inspect 'Error token %p in line %d' %
188: [[match, kind], line], tokens
189: end
190: raise_inspect 'Empty token', tokens unless match
191:
192: tokens << [match, kind]
193:
194: end
195:
196: tokens
197: end