annotate vendor/nikic/php-parser/lib/PhpParser/Lexer.php @ 19:fa3358dc1485 tip

Add ndrum files
author Chris Cannam
date Wed, 28 Aug 2019 13:14:47 +0100
parents 5fb285c0d0e3
children
rev   line source
Chris@13 1 <?php declare(strict_types=1);
Chris@0 2
Chris@0 3 namespace PhpParser;
Chris@0 4
Chris@0 5 use PhpParser\Parser\Tokens;
Chris@0 6
Chris@0 7 class Lexer
Chris@0 8 {
Chris@0 9 protected $code;
Chris@0 10 protected $tokens;
Chris@0 11 protected $pos;
Chris@0 12 protected $line;
Chris@0 13 protected $filePos;
Chris@0 14 protected $prevCloseTagHasNewline;
Chris@0 15
Chris@0 16 protected $tokenMap;
Chris@0 17 protected $dropTokens;
Chris@0 18
Chris@0 19 protected $usedAttributes;
Chris@0 20
Chris@0 21 /**
Chris@0 22 * Creates a Lexer.
Chris@0 23 *
Chris@0 24 * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
Chris@0 25 * which is an array of attributes to add to the AST nodes. Possible
Chris@0 26 * attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
Chris@0 27 * 'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
Chris@0 28 * first three. For more info see getNextToken() docs.
Chris@0 29 */
Chris@13 30 public function __construct(array $options = []) {
Chris@0 31 // map from internal tokens to PhpParser tokens
Chris@0 32 $this->tokenMap = $this->createTokenMap();
Chris@0 33
Chris@0 34 // map of tokens to drop while lexing (the map is only used for isset lookup,
Chris@0 35 // that's why the value is simply set to 1; the value is never actually used.)
Chris@0 36 $this->dropTokens = array_fill_keys(
Chris@13 37 [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT], 1
Chris@0 38 );
Chris@0 39
Chris@0 40 // the usedAttributes member is a map of the used attribute names to a dummy
Chris@0 41 // value (here "true")
Chris@13 42 $options += [
Chris@13 43 'usedAttributes' => ['comments', 'startLine', 'endLine'],
Chris@13 44 ];
Chris@0 45 $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
Chris@0 46 }
Chris@0 47
Chris@0 48 /**
Chris@0 49 * Initializes the lexer for lexing the provided source code.
Chris@0 50 *
Chris@0 51 * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
Chris@0 52 * the getErrors() method.
Chris@0 53 *
Chris@0 54 * @param string $code The source code to lex
Chris@0 55 * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
Chris@0 56 * ErrorHandler\Throwing
Chris@0 57 */
Chris@13 58 public function startLexing(string $code, ErrorHandler $errorHandler = null) {
Chris@0 59 if (null === $errorHandler) {
Chris@0 60 $errorHandler = new ErrorHandler\Throwing();
Chris@0 61 }
Chris@0 62
Chris@0 63 $this->code = $code; // keep the code around for __halt_compiler() handling
Chris@0 64 $this->pos = -1;
Chris@0 65 $this->line = 1;
Chris@0 66 $this->filePos = 0;
Chris@0 67
Chris@0 68 // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
Chris@0 69 // This ensures proper composability, because having a newline is the "safe" assumption.
Chris@0 70 $this->prevCloseTagHasNewline = true;
Chris@0 71
Chris@0 72 $scream = ini_set('xdebug.scream', '0');
Chris@0 73
Chris@13 74 error_clear_last();
Chris@0 75 $this->tokens = @token_get_all($code);
Chris@0 76 $this->handleErrors($errorHandler);
Chris@0 77
Chris@0 78 if (false !== $scream) {
Chris@0 79 ini_set('xdebug.scream', $scream);
Chris@0 80 }
Chris@0 81 }
Chris@0 82
Chris@0 83 private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
Chris@0 84 for ($i = $start; $i < $end; $i++) {
Chris@0 85 $chr = $this->code[$i];
Chris@0 86 if ($chr === 'b' || $chr === 'B') {
Chris@0 87 // HHVM does not treat b" tokens correctly, so ignore these
Chris@0 88 continue;
Chris@0 89 }
Chris@0 90
Chris@0 91 if ($chr === "\0") {
Chris@0 92 // PHP cuts error message after null byte, so need special case
Chris@0 93 $errorMsg = 'Unexpected null byte';
Chris@0 94 } else {
Chris@0 95 $errorMsg = sprintf(
Chris@0 96 'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
Chris@0 97 );
Chris@0 98 }
Chris@0 99
Chris@0 100 $errorHandler->handleError(new Error($errorMsg, [
Chris@0 101 'startLine' => $line,
Chris@0 102 'endLine' => $line,
Chris@0 103 'startFilePos' => $i,
Chris@0 104 'endFilePos' => $i,
Chris@0 105 ]));
Chris@0 106 }
Chris@0 107 }
Chris@0 108
Chris@13 109 /**
Chris@13 110 * Check whether comment token is unterminated.
Chris@13 111 *
Chris@13 112 * @return bool
Chris@13 113 */
Chris@13 114 private function isUnterminatedComment($token) : bool {
Chris@13 115 return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
Chris@0 116 && substr($token[1], 0, 2) === '/*'
Chris@0 117 && substr($token[1], -2) !== '*/';
Chris@0 118 }
Chris@0 119
Chris@13 120 /**
Chris@13 121 * Check whether an error *may* have occurred during tokenization.
Chris@13 122 *
Chris@13 123 * @return bool
Chris@13 124 */
Chris@13 125 private function errorMayHaveOccurred() : bool {
Chris@0 126 if (defined('HHVM_VERSION')) {
Chris@0 127 // In HHVM token_get_all() does not throw warnings, so we need to conservatively
Chris@0 128 // assume that an error occurred
Chris@0 129 return true;
Chris@0 130 }
Chris@0 131
Chris@13 132 return null !== error_get_last();
Chris@0 133 }
Chris@0 134
Chris@0 135 protected function handleErrors(ErrorHandler $errorHandler) {
Chris@0 136 if (!$this->errorMayHaveOccurred()) {
Chris@0 137 return;
Chris@0 138 }
Chris@0 139
Chris@0 140 // PHP's error handling for token_get_all() is rather bad, so if we want detailed
Chris@0 141 // error information we need to compute it ourselves. Invalid character errors are
Chris@0 142 // detected by finding "gaps" in the token array. Unterminated comments are detected
Chris@0 143 // by checking if a trailing comment has a "*/" at the end.
Chris@0 144
Chris@0 145 $filePos = 0;
Chris@0 146 $line = 1;
Chris@13 147 foreach ($this->tokens as $token) {
Chris@0 148 $tokenValue = \is_string($token) ? $token : $token[1];
Chris@0 149 $tokenLen = \strlen($tokenValue);
Chris@0 150
Chris@0 151 if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
Chris@0 152 // Something is missing, must be an invalid character
Chris@0 153 $nextFilePos = strpos($this->code, $tokenValue, $filePos);
Chris@0 154 $this->handleInvalidCharacterRange(
Chris@0 155 $filePos, $nextFilePos, $line, $errorHandler);
Chris@13 156 $filePos = (int) $nextFilePos;
Chris@0 157 }
Chris@0 158
Chris@0 159 $filePos += $tokenLen;
Chris@0 160 $line += substr_count($tokenValue, "\n");
Chris@0 161 }
Chris@0 162
Chris@0 163 if ($filePos !== \strlen($this->code)) {
Chris@0 164 if (substr($this->code, $filePos, 2) === '/*') {
Chris@0 165 // Unlike PHP, HHVM will drop unterminated comments entirely
Chris@0 166 $comment = substr($this->code, $filePos);
Chris@0 167 $errorHandler->handleError(new Error('Unterminated comment', [
Chris@0 168 'startLine' => $line,
Chris@0 169 'endLine' => $line + substr_count($comment, "\n"),
Chris@0 170 'startFilePos' => $filePos,
Chris@0 171 'endFilePos' => $filePos + \strlen($comment),
Chris@0 172 ]));
Chris@0 173
Chris@0 174 // Emulate the PHP behavior
Chris@0 175 $isDocComment = isset($comment[3]) && $comment[3] === '*';
Chris@13 176 $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
Chris@0 177 } else {
Chris@0 178 // Invalid characters at the end of the input
Chris@0 179 $this->handleInvalidCharacterRange(
Chris@0 180 $filePos, \strlen($this->code), $line, $errorHandler);
Chris@0 181 }
Chris@0 182 return;
Chris@0 183 }
Chris@0 184
Chris@0 185 if (count($this->tokens) > 0) {
Chris@0 186 // Check for unterminated comment
Chris@0 187 $lastToken = $this->tokens[count($this->tokens) - 1];
Chris@0 188 if ($this->isUnterminatedComment($lastToken)) {
Chris@0 189 $errorHandler->handleError(new Error('Unterminated comment', [
Chris@0 190 'startLine' => $line - substr_count($lastToken[1], "\n"),
Chris@0 191 'endLine' => $line,
Chris@0 192 'startFilePos' => $filePos - \strlen($lastToken[1]),
Chris@0 193 'endFilePos' => $filePos,
Chris@0 194 ]));
Chris@0 195 }
Chris@0 196 }
Chris@0 197 }
Chris@0 198
Chris@0 199 /**
Chris@0 200 * Fetches the next token.
Chris@0 201 *
Chris@0 202 * The available attributes are determined by the 'usedAttributes' option, which can
Chris@0 203 * be specified in the constructor. The following attributes are supported:
Chris@0 204 *
Chris@0 205 * * 'comments' => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
Chris@0 206 * representing all comments that occurred between the previous
Chris@0 207 * non-discarded token and the current one.
Chris@0 208 * * 'startLine' => Line in which the node starts.
Chris@0 209 * * 'endLine' => Line in which the node ends.
Chris@0 210 * * 'startTokenPos' => Offset into the token array of the first token in the node.
Chris@0 211 * * 'endTokenPos' => Offset into the token array of the last token in the node.
Chris@0 212 * * 'startFilePos' => Offset into the code string of the first character that is part of the node.
Chris@0 213 * * 'endFilePos' => Offset into the code string of the last character that is part of the node.
Chris@0 214 *
Chris@0 215 * @param mixed $value Variable to store token content in
Chris@0 216 * @param mixed $startAttributes Variable to store start attributes in
Chris@0 217 * @param mixed $endAttributes Variable to store end attributes in
Chris@0 218 *
Chris@0 219 * @return int Token id
Chris@0 220 */
Chris@13 221 public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
Chris@13 222 $startAttributes = [];
Chris@13 223 $endAttributes = [];
Chris@0 224
Chris@0 225 while (1) {
Chris@0 226 if (isset($this->tokens[++$this->pos])) {
Chris@0 227 $token = $this->tokens[$this->pos];
Chris@0 228 } else {
Chris@0 229 // EOF token with ID 0
Chris@0 230 $token = "\0";
Chris@0 231 }
Chris@0 232
Chris@0 233 if (isset($this->usedAttributes['startLine'])) {
Chris@0 234 $startAttributes['startLine'] = $this->line;
Chris@0 235 }
Chris@0 236 if (isset($this->usedAttributes['startTokenPos'])) {
Chris@0 237 $startAttributes['startTokenPos'] = $this->pos;
Chris@0 238 }
Chris@0 239 if (isset($this->usedAttributes['startFilePos'])) {
Chris@0 240 $startAttributes['startFilePos'] = $this->filePos;
Chris@0 241 }
Chris@0 242
Chris@0 243 if (\is_string($token)) {
Chris@0 244 $value = $token;
Chris@0 245 if (isset($token[1])) {
Chris@0 246 // bug in token_get_all
Chris@0 247 $this->filePos += 2;
Chris@0 248 $id = ord('"');
Chris@0 249 } else {
Chris@0 250 $this->filePos += 1;
Chris@0 251 $id = ord($token);
Chris@0 252 }
Chris@0 253 } elseif (!isset($this->dropTokens[$token[0]])) {
Chris@0 254 $value = $token[1];
Chris@0 255 $id = $this->tokenMap[$token[0]];
Chris@13 256 if (\T_CLOSE_TAG === $token[0]) {
Chris@0 257 $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
Chris@13 258 } elseif (\T_INLINE_HTML === $token[0]) {
Chris@0 259 $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
Chris@0 260 }
Chris@0 261
Chris@0 262 $this->line += substr_count($value, "\n");
Chris@0 263 $this->filePos += \strlen($value);
Chris@0 264 } else {
Chris@13 265 if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
Chris@0 266 if (isset($this->usedAttributes['comments'])) {
Chris@13 267 $comment = \T_DOC_COMMENT === $token[0]
Chris@13 268 ? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
Chris@13 269 : new Comment($token[1], $this->line, $this->filePos, $this->pos);
Chris@0 270 $startAttributes['comments'][] = $comment;
Chris@0 271 }
Chris@0 272 }
Chris@0 273
Chris@0 274 $this->line += substr_count($token[1], "\n");
Chris@0 275 $this->filePos += \strlen($token[1]);
Chris@0 276 continue;
Chris@0 277 }
Chris@0 278
Chris@0 279 if (isset($this->usedAttributes['endLine'])) {
Chris@0 280 $endAttributes['endLine'] = $this->line;
Chris@0 281 }
Chris@0 282 if (isset($this->usedAttributes['endTokenPos'])) {
Chris@0 283 $endAttributes['endTokenPos'] = $this->pos;
Chris@0 284 }
Chris@0 285 if (isset($this->usedAttributes['endFilePos'])) {
Chris@0 286 $endAttributes['endFilePos'] = $this->filePos - 1;
Chris@0 287 }
Chris@0 288
Chris@0 289 return $id;
Chris@0 290 }
Chris@0 291
Chris@0 292 throw new \RuntimeException('Reached end of lexer loop');
Chris@0 293 }
Chris@0 294
Chris@0 295 /**
Chris@0 296 * Returns the token array for current code.
Chris@0 297 *
Chris@0 298 * The token array is in the same format as provided by the
Chris@0 299 * token_get_all() function and does not discard tokens (i.e.
Chris@0 300 * whitespace and comments are included). The token position
Chris@0 301 * attributes are against this token array.
Chris@0 302 *
Chris@0 303 * @return array Array of tokens in token_get_all() format
Chris@0 304 */
Chris@13 305 public function getTokens() : array {
Chris@0 306 return $this->tokens;
Chris@0 307 }
Chris@0 308
Chris@0 309 /**
Chris@0 310 * Handles __halt_compiler() by returning the text after it.
Chris@0 311 *
Chris@0 312 * @return string Remaining text
Chris@0 313 */
Chris@13 314 public function handleHaltCompiler() : string {
Chris@0 315 // text after T_HALT_COMPILER, still including ();
Chris@0 316 $textAfter = substr($this->code, $this->filePos);
Chris@0 317
Chris@0 318 // ensure that it is followed by ();
Chris@0 319 // this simplifies the situation, by not allowing any comments
Chris@0 320 // in between of the tokens.
Chris@0 321 if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
Chris@0 322 throw new Error('__HALT_COMPILER must be followed by "();"');
Chris@0 323 }
Chris@0 324
Chris@0 325 // prevent the lexer from returning any further tokens
Chris@0 326 $this->pos = count($this->tokens);
Chris@0 327
Chris@0 328 // return with (); removed
Chris@13 329 return substr($textAfter, strlen($matches[0]));
Chris@0 330 }
Chris@0 331
Chris@0 332 /**
Chris@0 333 * Creates the token map.
Chris@0 334 *
Chris@0 335 * The token map maps the PHP internal token identifiers
Chris@0 336 * to the identifiers used by the Parser. Additionally it
Chris@0 337 * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
Chris@0 338 *
Chris@0 339 * @return array The token map
Chris@0 340 */
Chris@13 341 protected function createTokenMap() : array {
Chris@13 342 $tokenMap = [];
Chris@0 343
Chris@0 344 // 256 is the minimum possible token number, as everything below
Chris@0 345 // it is an ASCII value
Chris@0 346 for ($i = 256; $i < 1000; ++$i) {
Chris@13 347 if (\T_DOUBLE_COLON === $i) {
Chris@0 348 // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
Chris@0 349 $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
Chris@13 350 } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
Chris@0 351 // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
Chris@0 352 $tokenMap[$i] = Tokens::T_ECHO;
Chris@13 353 } elseif(\T_CLOSE_TAG === $i) {
Chris@0 354 // T_CLOSE_TAG is equivalent to ';'
Chris@0 355 $tokenMap[$i] = ord(';');
Chris@0 356 } elseif ('UNKNOWN' !== $name = token_name($i)) {
Chris@0 357 if ('T_HASHBANG' === $name) {
Chris@0 358 // HHVM uses a special token for #! hashbang lines
Chris@0 359 $tokenMap[$i] = Tokens::T_INLINE_HTML;
Chris@13 360 } elseif (defined($name = Tokens::class . '::' . $name)) {
Chris@0 361 // Other tokens can be mapped directly
Chris@0 362 $tokenMap[$i] = constant($name);
Chris@0 363 }
Chris@0 364 }
Chris@0 365 }
Chris@0 366
Chris@0 367 // HHVM uses a special token for numbers that overflow to double
Chris@0 368 if (defined('T_ONUMBER')) {
Chris@13 369 $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
Chris@0 370 }
Chris@0 371 // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
Chris@0 372 if (defined('T_COMPILER_HALT_OFFSET')) {
Chris@13 373 $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
Chris@0 374 }
Chris@0 375
Chris@0 376 return $tokenMap;
Chris@0 377 }
Chris@0 378 }