Chris@13: <?php declare(strict_types=1);
Chris@0: 
Chris@0: namespace PhpParser;
Chris@0: 
Chris@0: use PhpParser\Parser\Tokens;
Chris@0: 
Chris@0: class Lexer
Chris@0: {
Chris@0:     protected $code;
Chris@0:     protected $tokens;
Chris@0:     protected $pos;
Chris@0:     protected $line;
Chris@0:     protected $filePos;
Chris@0:     protected $prevCloseTagHasNewline;
Chris@0: 
Chris@0:     protected $tokenMap;
Chris@0:     protected $dropTokens;
Chris@0: 
Chris@0:     protected $usedAttributes;
Chris@0: 
Chris@0:     /**
Chris@0:      * Creates a Lexer.
Chris@0:      *
Chris@0:      * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
Chris@0:      *                       which is an array of attributes to add to the AST nodes. Possible
Chris@0:      *                       attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
Chris@0:      *                       'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
Chris@0:      *                       first three. For more info see getNextToken() docs.
Chris@0:      */
Chris@13:     public function __construct(array $options = []) {
Chris@0:         // map from internal tokens to PhpParser tokens
Chris@0:         $this->tokenMap = $this->createTokenMap();
Chris@0: 
Chris@0:         // map of tokens to drop while lexing (the map is only used for isset lookup,
Chris@0:         // that's why the value is simply set to 1; the value is never actually used.)
Chris@0:         $this->dropTokens = array_fill_keys(
Chris@13:             [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT], 1
Chris@0:         );
Chris@0: 
Chris@0:         // the usedAttributes member is a map of the used attribute names to a dummy
Chris@0:         // value (here "true")
Chris@13:         $options += [
Chris@13:             'usedAttributes' => ['comments', 'startLine', 'endLine'],
Chris@13:         ];
Chris@0:         $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * Initializes the lexer for lexing the provided source code.
Chris@0:      *
Chris@0:      * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
Chris@0:      * the getErrors() method.
Chris@0:      *
Chris@0:      * @param string $code The source code to lex
Chris@0:      * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
Chris@0:      *                                        ErrorHandler\Throwing
Chris@0:      */
Chris@13:     public function startLexing(string $code, ErrorHandler $errorHandler = null) {
Chris@0:         if (null === $errorHandler) {
Chris@0:             $errorHandler = new ErrorHandler\Throwing();
Chris@0:         }
Chris@0: 
Chris@0:         $this->code = $code; // keep the code around for __halt_compiler() handling
Chris@0:         $this->pos  = -1;
Chris@0:         $this->line =  1;
Chris@0:         $this->filePos = 0;
Chris@0: 
Chris@0:         // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
Chris@0:         // This ensures proper composability, because having a newline is the "safe" assumption.
Chris@0:         $this->prevCloseTagHasNewline = true;
Chris@0: 
Chris@0:         $scream = ini_set('xdebug.scream', '0');
Chris@0: 
Chris@13:         error_clear_last();
Chris@0:         $this->tokens = @token_get_all($code);
Chris@0:         $this->handleErrors($errorHandler);
Chris@0: 
Chris@0:         if (false !== $scream) {
Chris@0:             ini_set('xdebug.scream', $scream);
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
Chris@0:         for ($i = $start; $i < $end; $i++) {
Chris@0:             $chr = $this->code[$i];
Chris@0:             if ($chr === 'b' || $chr === 'B') {
Chris@0:                 // HHVM does not treat b" tokens correctly, so ignore these
Chris@0:                 continue;
Chris@0:             }
Chris@0: 
Chris@0:             if ($chr === "\0") {
Chris@0:                 // PHP cuts error message after null byte, so need special case
Chris@0:                 $errorMsg = 'Unexpected null byte';
Chris@0:             } else {
Chris@0:                 $errorMsg = sprintf(
Chris@0:                     'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
Chris@0:                 );
Chris@0:             }
Chris@0: 
Chris@0:             $errorHandler->handleError(new Error($errorMsg, [
Chris@0:                 'startLine' => $line,
Chris@0:                 'endLine' => $line,
Chris@0:                 'startFilePos' => $i,
Chris@0:                 'endFilePos' => $i,
Chris@0:             ]));
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@13:     /**
Chris@13:      * Check whether comment token is unterminated.
Chris@13:      *
Chris@13:      * @return bool
Chris@13:      */
Chris@13:     private function isUnterminatedComment($token) : bool {
Chris@13:         return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
Chris@0:             && substr($token[1], 0, 2) === '/*'
Chris@0:             && substr($token[1], -2) !== '*/';
Chris@0:     }
Chris@0: 
Chris@13:     /**
Chris@13:      * Check whether an error *may* have occurred during tokenization.
Chris@13:      *
Chris@13:      * @return bool
Chris@13:      */
Chris@13:     private function errorMayHaveOccurred() : bool {
Chris@0:         if (defined('HHVM_VERSION')) {
Chris@0:             // In HHVM token_get_all() does not throw warnings, so we need to conservatively
Chris@0:             // assume that an error occurred
Chris@0:             return true;
Chris@0:         }
Chris@0: 
Chris@13:         return null !== error_get_last();
Chris@0:     }
Chris@0: 
Chris@0:     protected function handleErrors(ErrorHandler $errorHandler) {
Chris@0:         if (!$this->errorMayHaveOccurred()) {
Chris@0:             return;
Chris@0:         }
Chris@0: 
Chris@0:         // PHP's error handling for token_get_all() is rather bad, so if we want detailed
Chris@0:         // error information we need to compute it ourselves. Invalid character errors are
Chris@0:         // detected by finding "gaps" in the token array. Unterminated comments are detected
Chris@0:         // by checking if a trailing comment has a "*/" at the end.
Chris@0: 
Chris@0:         $filePos = 0;
Chris@0:         $line = 1;
Chris@13:         foreach ($this->tokens as $token) {
Chris@0:             $tokenValue = \is_string($token) ? $token : $token[1];
Chris@0:             $tokenLen = \strlen($tokenValue);
Chris@0: 
Chris@0:             if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
Chris@0:                 // Something is missing, must be an invalid character
Chris@0:                 $nextFilePos = strpos($this->code, $tokenValue, $filePos);
Chris@0:                 $this->handleInvalidCharacterRange(
Chris@0:                     $filePos, $nextFilePos, $line, $errorHandler);
Chris@13:                 $filePos = (int) $nextFilePos;
Chris@0:             }
Chris@0: 
Chris@0:             $filePos += $tokenLen;
Chris@0:             $line += substr_count($tokenValue, "\n");
Chris@0:         }
Chris@0: 
Chris@0:         if ($filePos !== \strlen($this->code)) {
Chris@0:             if (substr($this->code, $filePos, 2) === '/*') {
Chris@0:                 // Unlike PHP, HHVM will drop unterminated comments entirely
Chris@0:                 $comment = substr($this->code, $filePos);
Chris@0:                 $errorHandler->handleError(new Error('Unterminated comment', [
Chris@0:                     'startLine' => $line,
Chris@0:                     'endLine' => $line + substr_count($comment, "\n"),
Chris@0:                     'startFilePos' => $filePos,
Chris@0:                     'endFilePos' => $filePos + \strlen($comment),
Chris@0:                 ]));
Chris@0: 
Chris@0:                 // Emulate the PHP behavior
Chris@0:                 $isDocComment = isset($comment[3]) && $comment[3] === '*';
Chris@13:                 $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
Chris@0:             } else {
Chris@0:                 // Invalid characters at the end of the input
Chris@0:                 $this->handleInvalidCharacterRange(
Chris@0:                     $filePos, \strlen($this->code), $line, $errorHandler);
Chris@0:             }
Chris@0:             return;
Chris@0:         }
Chris@0: 
Chris@0:         if (count($this->tokens) > 0) {
Chris@0:             // Check for unterminated comment
Chris@0:             $lastToken = $this->tokens[count($this->tokens) - 1];
Chris@0:             if ($this->isUnterminatedComment($lastToken)) {
Chris@0:                 $errorHandler->handleError(new Error('Unterminated comment', [
Chris@0:                     'startLine' => $line - substr_count($lastToken[1], "\n"),
Chris@0:                     'endLine' => $line,
Chris@0:                     'startFilePos' => $filePos - \strlen($lastToken[1]),
Chris@0:                     'endFilePos' => $filePos,
Chris@0:                 ]));
Chris@0:             }
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * Fetches the next token.
Chris@0:      *
Chris@0:      * The available attributes are determined by the 'usedAttributes' option, which can
Chris@0:      * be specified in the constructor. The following attributes are supported:
Chris@0:      *
Chris@0:      *  * 'comments'      => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
Chris@0:      *                       representing all comments that occurred between the previous
Chris@0:      *                       non-discarded token and the current one.
Chris@0:      *  * 'startLine'     => Line in which the node starts.
Chris@0:      *  * 'endLine'       => Line in which the node ends.
Chris@0:      *  * 'startTokenPos' => Offset into the token array of the first token in the node.
Chris@0:      *  * 'endTokenPos'   => Offset into the token array of the last token in the node.
Chris@0:      *  * 'startFilePos'  => Offset into the code string of the first character that is part of the node.
Chris@0:      *  * 'endFilePos'    => Offset into the code string of the last character that is part of the node.
Chris@0:      *
Chris@0:      * @param mixed $value           Variable to store token content in
Chris@0:      * @param mixed $startAttributes Variable to store start attributes in
Chris@0:      * @param mixed $endAttributes   Variable to store end attributes in
Chris@0:      *
Chris@0:      * @return int Token id
Chris@0:      */
Chris@13:     public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
Chris@13:         $startAttributes = [];
Chris@13:         $endAttributes   = [];
Chris@0: 
Chris@0:         while (1) {
Chris@0:             if (isset($this->tokens[++$this->pos])) {
Chris@0:                 $token = $this->tokens[$this->pos];
Chris@0:             } else {
Chris@0:                 // EOF token with ID 0
Chris@0:                 $token = "\0";
Chris@0:             }
Chris@0: 
Chris@0:             if (isset($this->usedAttributes['startLine'])) {
Chris@0:                 $startAttributes['startLine'] = $this->line;
Chris@0:             }
Chris@0:             if (isset($this->usedAttributes['startTokenPos'])) {
Chris@0:                 $startAttributes['startTokenPos'] = $this->pos;
Chris@0:             }
Chris@0:             if (isset($this->usedAttributes['startFilePos'])) {
Chris@0:                 $startAttributes['startFilePos'] = $this->filePos;
Chris@0:             }
Chris@0: 
Chris@0:             if (\is_string($token)) {
Chris@0:                 $value = $token;
Chris@0:                 if (isset($token[1])) {
Chris@0:                     // bug in token_get_all
Chris@0:                     $this->filePos += 2;
Chris@0:                     $id = ord('"');
Chris@0:                 } else {
Chris@0:                     $this->filePos += 1;
Chris@0:                     $id = ord($token);
Chris@0:                 }
Chris@0:             } elseif (!isset($this->dropTokens[$token[0]])) {
Chris@0:                 $value = $token[1];
Chris@0:                 $id = $this->tokenMap[$token[0]];
Chris@13:                 if (\T_CLOSE_TAG === $token[0]) {
Chris@0:                     $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
Chris@13:                 } elseif (\T_INLINE_HTML === $token[0]) {
Chris@0:                     $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
Chris@0:                 }
Chris@0: 
Chris@0:                 $this->line += substr_count($value, "\n");
Chris@0:                 $this->filePos += \strlen($value);
Chris@0:             } else {
Chris@13:                 if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
Chris@0:                     if (isset($this->usedAttributes['comments'])) {
Chris@13:                         $comment = \T_DOC_COMMENT === $token[0]
Chris@13:                             ? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
Chris@13:                             : new Comment($token[1], $this->line, $this->filePos, $this->pos);
Chris@0:                         $startAttributes['comments'][] = $comment;
Chris@0:                     }
Chris@0:                 }
Chris@0: 
Chris@0:                 $this->line += substr_count($token[1], "\n");
Chris@0:                 $this->filePos += \strlen($token[1]);
Chris@0:                 continue;
Chris@0:             }
Chris@0: 
Chris@0:             if (isset($this->usedAttributes['endLine'])) {
Chris@0:                 $endAttributes['endLine'] = $this->line;
Chris@0:             }
Chris@0:             if (isset($this->usedAttributes['endTokenPos'])) {
Chris@0:                 $endAttributes['endTokenPos'] = $this->pos;
Chris@0:             }
Chris@0:             if (isset($this->usedAttributes['endFilePos'])) {
Chris@0:                 $endAttributes['endFilePos'] = $this->filePos - 1;
Chris@0:             }
Chris@0: 
Chris@0:             return $id;
Chris@0:         }
Chris@0: 
Chris@0:         throw new \RuntimeException('Reached end of lexer loop');
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * Returns the token array for current code.
Chris@0:      *
Chris@0:      * The token array is in the same format as provided by the
Chris@0:      * token_get_all() function and does not discard tokens (i.e.
Chris@0:      * whitespace and comments are included). The token position
Chris@0:      * attributes are against this token array.
Chris@0:      *
Chris@0:      * @return array Array of tokens in token_get_all() format
Chris@0:      */
Chris@13:     public function getTokens() : array {
Chris@0:         return $this->tokens;
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * Handles __halt_compiler() by returning the text after it.
Chris@0:      *
Chris@0:      * @return string Remaining text
Chris@0:      */
Chris@13:     public function handleHaltCompiler() : string {
Chris@0:         // text after T_HALT_COMPILER, still including ();
Chris@0:         $textAfter = substr($this->code, $this->filePos);
Chris@0: 
Chris@0:         // ensure that it is followed by ();
Chris@0:         // this simplifies the situation, by not allowing any comments
Chris@0:         // in between of the tokens.
Chris@0:         if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
Chris@0:             throw new Error('__HALT_COMPILER must be followed by "();"');
Chris@0:         }
Chris@0: 
Chris@0:         // prevent the lexer from returning any further tokens
Chris@0:         $this->pos = count($this->tokens);
Chris@0: 
Chris@0:         // return with (); removed
Chris@13:         return substr($textAfter, strlen($matches[0]));
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * Creates the token map.
Chris@0:      *
Chris@0:      * The token map maps the PHP internal token identifiers
Chris@0:      * to the identifiers used by the Parser. Additionally it
Chris@0:      * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
Chris@0:      *
Chris@0:      * @return array The token map
Chris@0:      */
Chris@13:     protected function createTokenMap() : array {
Chris@13:         $tokenMap = [];
Chris@0: 
Chris@0:         // 256 is the minimum possible token number, as everything below
Chris@0:         // it is an ASCII value
Chris@0:         for ($i = 256; $i < 1000; ++$i) {
Chris@13:             if (\T_DOUBLE_COLON === $i) {
Chris@0:                 // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
Chris@0:                 $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
Chris@13:             } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
Chris@0:                 // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
Chris@0:                 $tokenMap[$i] = Tokens::T_ECHO;
Chris@13:             } elseif(\T_CLOSE_TAG === $i) {
Chris@0:                 // T_CLOSE_TAG is equivalent to ';'
Chris@0:                 $tokenMap[$i] = ord(';');
Chris@0:             } elseif ('UNKNOWN' !== $name = token_name($i)) {
Chris@0:                 if ('T_HASHBANG' === $name) {
Chris@0:                     // HHVM uses a special token for #! hashbang lines
Chris@0:                     $tokenMap[$i] = Tokens::T_INLINE_HTML;
Chris@13:                 } elseif (defined($name = Tokens::class . '::' . $name)) {
Chris@0:                     // Other tokens can be mapped directly
Chris@0:                     $tokenMap[$i] = constant($name);
Chris@0:                 }
Chris@0:             }
Chris@0:         }
Chris@0: 
Chris@0:         // HHVM uses a special token for numbers that overflow to double
Chris@0:         if (defined('T_ONUMBER')) {
Chris@13:             $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
Chris@0:         }
Chris@0:         // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
Chris@0:         if (defined('T_COMPILER_HALT_OFFSET')) {
Chris@13:             $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
Chris@0:         }
Chris@0: 
Chris@0:         return $tokenMap;
Chris@0:     }
Chris@0: }