Chris@13
|
1 <?php declare(strict_types=1);
|
Chris@0
|
2
|
Chris@0
|
3 namespace PhpParser;
|
Chris@0
|
4
|
Chris@0
|
5 use PhpParser\Parser\Tokens;
|
Chris@0
|
6
|
Chris@0
|
7 class Lexer
|
Chris@0
|
8 {
|
Chris@0
|
9 protected $code;
|
Chris@0
|
10 protected $tokens;
|
Chris@0
|
11 protected $pos;
|
Chris@0
|
12 protected $line;
|
Chris@0
|
13 protected $filePos;
|
Chris@0
|
14 protected $prevCloseTagHasNewline;
|
Chris@0
|
15
|
Chris@0
|
16 protected $tokenMap;
|
Chris@0
|
17 protected $dropTokens;
|
Chris@0
|
18
|
Chris@0
|
19 protected $usedAttributes;
|
Chris@0
|
20
|
Chris@0
|
21 /**
|
Chris@0
|
22 * Creates a Lexer.
|
Chris@0
|
23 *
|
Chris@0
|
24 * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
|
Chris@0
|
25 * which is an array of attributes to add to the AST nodes. Possible
|
Chris@0
|
26 * attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
|
Chris@0
|
27 * 'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
|
Chris@0
|
28 * first three. For more info see getNextToken() docs.
|
Chris@0
|
29 */
|
Chris@13
|
30 public function __construct(array $options = []) {
|
Chris@0
|
31 // map from internal tokens to PhpParser tokens
|
Chris@0
|
32 $this->tokenMap = $this->createTokenMap();
|
Chris@0
|
33
|
Chris@0
|
34 // map of tokens to drop while lexing (the map is only used for isset lookup,
|
Chris@0
|
35 // that's why the value is simply set to 1; the value is never actually used.)
|
Chris@0
|
36 $this->dropTokens = array_fill_keys(
|
Chris@13
|
37 [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT], 1
|
Chris@0
|
38 );
|
Chris@0
|
39
|
Chris@0
|
40 // the usedAttributes member is a map of the used attribute names to a dummy
|
Chris@0
|
41 // value (here "true")
|
Chris@13
|
42 $options += [
|
Chris@13
|
43 'usedAttributes' => ['comments', 'startLine', 'endLine'],
|
Chris@13
|
44 ];
|
Chris@0
|
45 $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
|
Chris@0
|
46 }
|
Chris@0
|
47
|
Chris@0
|
48 /**
|
Chris@0
|
49 * Initializes the lexer for lexing the provided source code.
|
Chris@0
|
50 *
|
Chris@0
|
51 * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
|
Chris@0
|
52 * the getErrors() method.
|
Chris@0
|
53 *
|
Chris@0
|
54 * @param string $code The source code to lex
|
Chris@0
|
55 * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
|
Chris@0
|
56 * ErrorHandler\Throwing
|
Chris@0
|
57 */
|
Chris@13
|
58 public function startLexing(string $code, ErrorHandler $errorHandler = null) {
|
Chris@0
|
59 if (null === $errorHandler) {
|
Chris@0
|
60 $errorHandler = new ErrorHandler\Throwing();
|
Chris@0
|
61 }
|
Chris@0
|
62
|
Chris@0
|
63 $this->code = $code; // keep the code around for __halt_compiler() handling
|
Chris@0
|
64 $this->pos = -1;
|
Chris@0
|
65 $this->line = 1;
|
Chris@0
|
66 $this->filePos = 0;
|
Chris@0
|
67
|
Chris@0
|
68 // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
|
Chris@0
|
69 // This ensures proper composability, because having a newline is the "safe" assumption.
|
Chris@0
|
70 $this->prevCloseTagHasNewline = true;
|
Chris@0
|
71
|
Chris@0
|
72 $scream = ini_set('xdebug.scream', '0');
|
Chris@0
|
73
|
Chris@13
|
74 error_clear_last();
|
Chris@0
|
75 $this->tokens = @token_get_all($code);
|
Chris@0
|
76 $this->handleErrors($errorHandler);
|
Chris@0
|
77
|
Chris@0
|
78 if (false !== $scream) {
|
Chris@0
|
79 ini_set('xdebug.scream', $scream);
|
Chris@0
|
80 }
|
Chris@0
|
81 }
|
Chris@0
|
82
|
Chris@0
|
83 private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
|
Chris@0
|
84 for ($i = $start; $i < $end; $i++) {
|
Chris@0
|
85 $chr = $this->code[$i];
|
Chris@0
|
86 if ($chr === 'b' || $chr === 'B') {
|
Chris@0
|
87 // HHVM does not treat b" tokens correctly, so ignore these
|
Chris@0
|
88 continue;
|
Chris@0
|
89 }
|
Chris@0
|
90
|
Chris@0
|
91 if ($chr === "\0") {
|
Chris@0
|
92 // PHP cuts error message after null byte, so need special case
|
Chris@0
|
93 $errorMsg = 'Unexpected null byte';
|
Chris@0
|
94 } else {
|
Chris@0
|
95 $errorMsg = sprintf(
|
Chris@0
|
96 'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
|
Chris@0
|
97 );
|
Chris@0
|
98 }
|
Chris@0
|
99
|
Chris@0
|
100 $errorHandler->handleError(new Error($errorMsg, [
|
Chris@0
|
101 'startLine' => $line,
|
Chris@0
|
102 'endLine' => $line,
|
Chris@0
|
103 'startFilePos' => $i,
|
Chris@0
|
104 'endFilePos' => $i,
|
Chris@0
|
105 ]));
|
Chris@0
|
106 }
|
Chris@0
|
107 }
|
Chris@0
|
108
|
Chris@13
|
109 /**
|
Chris@13
|
110 * Check whether comment token is unterminated.
|
Chris@13
|
111 *
|
Chris@13
|
112 * @return bool
|
Chris@13
|
113 */
|
Chris@13
|
114 private function isUnterminatedComment($token) : bool {
|
Chris@13
|
115 return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
|
Chris@0
|
116 && substr($token[1], 0, 2) === '/*'
|
Chris@0
|
117 && substr($token[1], -2) !== '*/';
|
Chris@0
|
118 }
|
Chris@0
|
119
|
Chris@13
|
120 /**
|
Chris@13
|
121 * Check whether an error *may* have occurred during tokenization.
|
Chris@13
|
122 *
|
Chris@13
|
123 * @return bool
|
Chris@13
|
124 */
|
Chris@13
|
125 private function errorMayHaveOccurred() : bool {
|
Chris@0
|
126 if (defined('HHVM_VERSION')) {
|
Chris@0
|
127 // In HHVM token_get_all() does not throw warnings, so we need to conservatively
|
Chris@0
|
128 // assume that an error occurred
|
Chris@0
|
129 return true;
|
Chris@0
|
130 }
|
Chris@0
|
131
|
Chris@13
|
132 return null !== error_get_last();
|
Chris@0
|
133 }
|
Chris@0
|
134
|
Chris@0
|
135 protected function handleErrors(ErrorHandler $errorHandler) {
|
Chris@0
|
136 if (!$this->errorMayHaveOccurred()) {
|
Chris@0
|
137 return;
|
Chris@0
|
138 }
|
Chris@0
|
139
|
Chris@0
|
140 // PHP's error handling for token_get_all() is rather bad, so if we want detailed
|
Chris@0
|
141 // error information we need to compute it ourselves. Invalid character errors are
|
Chris@0
|
142 // detected by finding "gaps" in the token array. Unterminated comments are detected
|
Chris@0
|
143 // by checking if a trailing comment has a "*/" at the end.
|
Chris@0
|
144
|
Chris@0
|
145 $filePos = 0;
|
Chris@0
|
146 $line = 1;
|
Chris@13
|
147 foreach ($this->tokens as $token) {
|
Chris@0
|
148 $tokenValue = \is_string($token) ? $token : $token[1];
|
Chris@0
|
149 $tokenLen = \strlen($tokenValue);
|
Chris@0
|
150
|
Chris@0
|
151 if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
|
Chris@0
|
152 // Something is missing, must be an invalid character
|
Chris@0
|
153 $nextFilePos = strpos($this->code, $tokenValue, $filePos);
|
Chris@0
|
154 $this->handleInvalidCharacterRange(
|
Chris@0
|
155 $filePos, $nextFilePos, $line, $errorHandler);
|
Chris@13
|
156 $filePos = (int) $nextFilePos;
|
Chris@0
|
157 }
|
Chris@0
|
158
|
Chris@0
|
159 $filePos += $tokenLen;
|
Chris@0
|
160 $line += substr_count($tokenValue, "\n");
|
Chris@0
|
161 }
|
Chris@0
|
162
|
Chris@0
|
163 if ($filePos !== \strlen($this->code)) {
|
Chris@0
|
164 if (substr($this->code, $filePos, 2) === '/*') {
|
Chris@0
|
165 // Unlike PHP, HHVM will drop unterminated comments entirely
|
Chris@0
|
166 $comment = substr($this->code, $filePos);
|
Chris@0
|
167 $errorHandler->handleError(new Error('Unterminated comment', [
|
Chris@0
|
168 'startLine' => $line,
|
Chris@0
|
169 'endLine' => $line + substr_count($comment, "\n"),
|
Chris@0
|
170 'startFilePos' => $filePos,
|
Chris@0
|
171 'endFilePos' => $filePos + \strlen($comment),
|
Chris@0
|
172 ]));
|
Chris@0
|
173
|
Chris@0
|
174 // Emulate the PHP behavior
|
Chris@0
|
175 $isDocComment = isset($comment[3]) && $comment[3] === '*';
|
Chris@13
|
176 $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
|
Chris@0
|
177 } else {
|
Chris@0
|
178 // Invalid characters at the end of the input
|
Chris@0
|
179 $this->handleInvalidCharacterRange(
|
Chris@0
|
180 $filePos, \strlen($this->code), $line, $errorHandler);
|
Chris@0
|
181 }
|
Chris@0
|
182 return;
|
Chris@0
|
183 }
|
Chris@0
|
184
|
Chris@0
|
185 if (count($this->tokens) > 0) {
|
Chris@0
|
186 // Check for unterminated comment
|
Chris@0
|
187 $lastToken = $this->tokens[count($this->tokens) - 1];
|
Chris@0
|
188 if ($this->isUnterminatedComment($lastToken)) {
|
Chris@0
|
189 $errorHandler->handleError(new Error('Unterminated comment', [
|
Chris@0
|
190 'startLine' => $line - substr_count($lastToken[1], "\n"),
|
Chris@0
|
191 'endLine' => $line,
|
Chris@0
|
192 'startFilePos' => $filePos - \strlen($lastToken[1]),
|
Chris@0
|
193 'endFilePos' => $filePos,
|
Chris@0
|
194 ]));
|
Chris@0
|
195 }
|
Chris@0
|
196 }
|
Chris@0
|
197 }
|
Chris@0
|
198
|
Chris@0
|
199 /**
|
Chris@0
|
200 * Fetches the next token.
|
Chris@0
|
201 *
|
Chris@0
|
202 * The available attributes are determined by the 'usedAttributes' option, which can
|
Chris@0
|
203 * be specified in the constructor. The following attributes are supported:
|
Chris@0
|
204 *
|
Chris@0
|
205 * * 'comments' => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
|
Chris@0
|
206 * representing all comments that occurred between the previous
|
Chris@0
|
207 * non-discarded token and the current one.
|
Chris@0
|
208 * * 'startLine' => Line in which the node starts.
|
Chris@0
|
209 * * 'endLine' => Line in which the node ends.
|
Chris@0
|
210 * * 'startTokenPos' => Offset into the token array of the first token in the node.
|
Chris@0
|
211 * * 'endTokenPos' => Offset into the token array of the last token in the node.
|
Chris@0
|
212 * * 'startFilePos' => Offset into the code string of the first character that is part of the node.
|
Chris@0
|
213 * * 'endFilePos' => Offset into the code string of the last character that is part of the node.
|
Chris@0
|
214 *
|
Chris@0
|
215 * @param mixed $value Variable to store token content in
|
Chris@0
|
216 * @param mixed $startAttributes Variable to store start attributes in
|
Chris@0
|
217 * @param mixed $endAttributes Variable to store end attributes in
|
Chris@0
|
218 *
|
Chris@0
|
219 * @return int Token id
|
Chris@0
|
220 */
|
Chris@13
|
221 public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
|
Chris@13
|
222 $startAttributes = [];
|
Chris@13
|
223 $endAttributes = [];
|
Chris@0
|
224
|
Chris@0
|
225 while (1) {
|
Chris@0
|
226 if (isset($this->tokens[++$this->pos])) {
|
Chris@0
|
227 $token = $this->tokens[$this->pos];
|
Chris@0
|
228 } else {
|
Chris@0
|
229 // EOF token with ID 0
|
Chris@0
|
230 $token = "\0";
|
Chris@0
|
231 }
|
Chris@0
|
232
|
Chris@0
|
233 if (isset($this->usedAttributes['startLine'])) {
|
Chris@0
|
234 $startAttributes['startLine'] = $this->line;
|
Chris@0
|
235 }
|
Chris@0
|
236 if (isset($this->usedAttributes['startTokenPos'])) {
|
Chris@0
|
237 $startAttributes['startTokenPos'] = $this->pos;
|
Chris@0
|
238 }
|
Chris@0
|
239 if (isset($this->usedAttributes['startFilePos'])) {
|
Chris@0
|
240 $startAttributes['startFilePos'] = $this->filePos;
|
Chris@0
|
241 }
|
Chris@0
|
242
|
Chris@0
|
243 if (\is_string($token)) {
|
Chris@0
|
244 $value = $token;
|
Chris@0
|
245 if (isset($token[1])) {
|
Chris@0
|
246 // bug in token_get_all
|
Chris@0
|
247 $this->filePos += 2;
|
Chris@0
|
248 $id = ord('"');
|
Chris@0
|
249 } else {
|
Chris@0
|
250 $this->filePos += 1;
|
Chris@0
|
251 $id = ord($token);
|
Chris@0
|
252 }
|
Chris@0
|
253 } elseif (!isset($this->dropTokens[$token[0]])) {
|
Chris@0
|
254 $value = $token[1];
|
Chris@0
|
255 $id = $this->tokenMap[$token[0]];
|
Chris@13
|
256 if (\T_CLOSE_TAG === $token[0]) {
|
Chris@0
|
257 $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
|
Chris@13
|
258 } elseif (\T_INLINE_HTML === $token[0]) {
|
Chris@0
|
259 $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
|
Chris@0
|
260 }
|
Chris@0
|
261
|
Chris@0
|
262 $this->line += substr_count($value, "\n");
|
Chris@0
|
263 $this->filePos += \strlen($value);
|
Chris@0
|
264 } else {
|
Chris@13
|
265 if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
|
Chris@0
|
266 if (isset($this->usedAttributes['comments'])) {
|
Chris@13
|
267 $comment = \T_DOC_COMMENT === $token[0]
|
Chris@13
|
268 ? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
|
Chris@13
|
269 : new Comment($token[1], $this->line, $this->filePos, $this->pos);
|
Chris@0
|
270 $startAttributes['comments'][] = $comment;
|
Chris@0
|
271 }
|
Chris@0
|
272 }
|
Chris@0
|
273
|
Chris@0
|
274 $this->line += substr_count($token[1], "\n");
|
Chris@0
|
275 $this->filePos += \strlen($token[1]);
|
Chris@0
|
276 continue;
|
Chris@0
|
277 }
|
Chris@0
|
278
|
Chris@0
|
279 if (isset($this->usedAttributes['endLine'])) {
|
Chris@0
|
280 $endAttributes['endLine'] = $this->line;
|
Chris@0
|
281 }
|
Chris@0
|
282 if (isset($this->usedAttributes['endTokenPos'])) {
|
Chris@0
|
283 $endAttributes['endTokenPos'] = $this->pos;
|
Chris@0
|
284 }
|
Chris@0
|
285 if (isset($this->usedAttributes['endFilePos'])) {
|
Chris@0
|
286 $endAttributes['endFilePos'] = $this->filePos - 1;
|
Chris@0
|
287 }
|
Chris@0
|
288
|
Chris@0
|
289 return $id;
|
Chris@0
|
290 }
|
Chris@0
|
291
|
Chris@0
|
292 throw new \RuntimeException('Reached end of lexer loop');
|
Chris@0
|
293 }
|
Chris@0
|
294
|
Chris@0
|
295 /**
|
Chris@0
|
296 * Returns the token array for current code.
|
Chris@0
|
297 *
|
Chris@0
|
298 * The token array is in the same format as provided by the
|
Chris@0
|
299 * token_get_all() function and does not discard tokens (i.e.
|
Chris@0
|
300 * whitespace and comments are included). The token position
|
Chris@0
|
301 * attributes are against this token array.
|
Chris@0
|
302 *
|
Chris@0
|
303 * @return array Array of tokens in token_get_all() format
|
Chris@0
|
304 */
|
Chris@13
|
305 public function getTokens() : array {
|
Chris@0
|
306 return $this->tokens;
|
Chris@0
|
307 }
|
Chris@0
|
308
|
Chris@0
|
309 /**
|
Chris@0
|
310 * Handles __halt_compiler() by returning the text after it.
|
Chris@0
|
311 *
|
Chris@0
|
312 * @return string Remaining text
|
Chris@0
|
313 */
|
Chris@13
|
314 public function handleHaltCompiler() : string {
|
Chris@0
|
315 // text after T_HALT_COMPILER, still including ();
|
Chris@0
|
316 $textAfter = substr($this->code, $this->filePos);
|
Chris@0
|
317
|
Chris@0
|
318 // ensure that it is followed by ();
|
Chris@0
|
319 // this simplifies the situation, by not allowing any comments
|
Chris@0
|
320 // in between of the tokens.
|
Chris@0
|
321 if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
|
Chris@0
|
322 throw new Error('__HALT_COMPILER must be followed by "();"');
|
Chris@0
|
323 }
|
Chris@0
|
324
|
Chris@0
|
325 // prevent the lexer from returning any further tokens
|
Chris@0
|
326 $this->pos = count($this->tokens);
|
Chris@0
|
327
|
Chris@0
|
328 // return with (); removed
|
Chris@13
|
329 return substr($textAfter, strlen($matches[0]));
|
Chris@0
|
330 }
|
Chris@0
|
331
|
Chris@0
|
332 /**
|
Chris@0
|
333 * Creates the token map.
|
Chris@0
|
334 *
|
Chris@0
|
335 * The token map maps the PHP internal token identifiers
|
Chris@0
|
336 * to the identifiers used by the Parser. Additionally it
|
Chris@0
|
337 * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
|
Chris@0
|
338 *
|
Chris@0
|
339 * @return array The token map
|
Chris@0
|
340 */
|
Chris@13
|
341 protected function createTokenMap() : array {
|
Chris@13
|
342 $tokenMap = [];
|
Chris@0
|
343
|
Chris@0
|
344 // 256 is the minimum possible token number, as everything below
|
Chris@0
|
345 // it is an ASCII value
|
Chris@0
|
346 for ($i = 256; $i < 1000; ++$i) {
|
Chris@13
|
347 if (\T_DOUBLE_COLON === $i) {
|
Chris@0
|
348 // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
|
Chris@0
|
349 $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
|
Chris@13
|
350 } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
|
Chris@0
|
351 // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
|
Chris@0
|
352 $tokenMap[$i] = Tokens::T_ECHO;
|
Chris@13
|
353 } elseif(\T_CLOSE_TAG === $i) {
|
Chris@0
|
354 // T_CLOSE_TAG is equivalent to ';'
|
Chris@0
|
355 $tokenMap[$i] = ord(';');
|
Chris@0
|
356 } elseif ('UNKNOWN' !== $name = token_name($i)) {
|
Chris@0
|
357 if ('T_HASHBANG' === $name) {
|
Chris@0
|
358 // HHVM uses a special token for #! hashbang lines
|
Chris@0
|
359 $tokenMap[$i] = Tokens::T_INLINE_HTML;
|
Chris@13
|
360 } elseif (defined($name = Tokens::class . '::' . $name)) {
|
Chris@0
|
361 // Other tokens can be mapped directly
|
Chris@0
|
362 $tokenMap[$i] = constant($name);
|
Chris@0
|
363 }
|
Chris@0
|
364 }
|
Chris@0
|
365 }
|
Chris@0
|
366
|
Chris@0
|
367 // HHVM uses a special token for numbers that overflow to double
|
Chris@0
|
368 if (defined('T_ONUMBER')) {
|
Chris@13
|
369 $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
|
Chris@0
|
370 }
|
Chris@0
|
371 // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
|
Chris@0
|
372 if (defined('T_COMPILER_HALT_OFFSET')) {
|
Chris@13
|
373 $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
|
Chris@0
|
374 }
|
Chris@0
|
375
|
Chris@0
|
376 return $tokenMap;
|
Chris@0
|
377 }
|
Chris@0
|
378 }
|