Chris@13: <?php declare(strict_types=1);
Chris@0: 
Chris@0: namespace PhpParser;
Chris@0: 
Chris@0: use PhpParser\Parser\Tokens;
Chris@0: 
Chris@17: class LexerTest extends \PHPUnit\Framework\TestCase
Chris@0: {
Chris@0:     /* To allow overwriting in parent class */
Chris@13:     protected function getLexer(array $options = []) {
Chris@0:         return new Lexer($options);
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * @dataProvider provideTestError
Chris@0:      */
Chris@0:     public function testError($code, $messages) {
Chris@0:         if (defined('HHVM_VERSION')) {
Chris@0:             $this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
Chris@0:         }
Chris@0: 
Chris@0:         $errorHandler = new ErrorHandler\Collecting();
Chris@0:         $lexer = $this->getLexer(['usedAttributes' => [
Chris@0:             'comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'
Chris@0:         ]]);
Chris@0:         $lexer->startLexing($code, $errorHandler);
Chris@0:         $errors = $errorHandler->getErrors();
Chris@0: 
Chris@13:         $this->assertCount(count($messages), $errors);
Chris@0:         for ($i = 0; $i < count($messages); $i++) {
Chris@0:             $this->assertSame($messages[$i], $errors[$i]->getMessageWithColumnInfo($code));
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     public function provideTestError() {
Chris@13:         return [
Chris@13:             ["<?php /*", ["Unterminated comment from 1:7 to 1:9"]],
Chris@13:             ["<?php \1", ["Unexpected character \"\1\" (ASCII 1) from 1:7 to 1:7"]],
Chris@13:             ["<?php \0", ["Unexpected null byte from 1:7 to 1:7"]],
Chris@0:             // Error with potentially emulated token
Chris@13:             ["<?php ?? \0", ["Unexpected null byte from 1:10 to 1:10"]],
Chris@13:             ["<?php\n\0\1 foo /* bar", [
Chris@0:                 "Unexpected null byte from 2:1 to 2:1",
Chris@0:                 "Unexpected character \"\1\" (ASCII 1) from 2:2 to 2:2",
Chris@0:                 "Unterminated comment from 2:8 to 2:14"
Chris@13:             ]],
Chris@13:         ];
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * @dataProvider provideTestLex
Chris@0:      */
Chris@0:     public function testLex($code, $options, $tokens) {
Chris@0:         $lexer = $this->getLexer($options);
Chris@0:         $lexer->startLexing($code);
Chris@0:         while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
Chris@0:             $token = array_shift($tokens);
Chris@0: 
Chris@0:             $this->assertSame($token[0], $id);
Chris@0:             $this->assertSame($token[1], $value);
Chris@0:             $this->assertEquals($token[2], $startAttributes);
Chris@0:             $this->assertEquals($token[3], $endAttributes);
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     public function provideTestLex() {
Chris@13:         return [
Chris@0:             // tests conversion of closing PHP tag and drop of whitespace and opening tags
Chris@13:             [
Chris@0:                 '<?php tokens ?>plaintext',
Chris@13:                 [],
Chris@13:                 [
Chris@13:                     [
Chris@0:                         Tokens::T_STRING, 'tokens',
Chris@13:                         ['startLine' => 1], ['endLine' => 1]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         ord(';'), '?>',
Chris@13:                         ['startLine' => 1], ['endLine' => 1]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         Tokens::T_INLINE_HTML, 'plaintext',
Chris@13:                         ['startLine' => 1, 'hasLeadingNewline' => false],
Chris@13:                         ['endLine' => 1]
Chris@13:                     ],
Chris@13:                 ]
Chris@13:             ],
Chris@0:             // tests line numbers
Chris@13:             [
Chris@0:                 '<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
Chris@13:                 [],
Chris@13:                 [
Chris@13:                     [
Chris@0:                         ord('$'), '$',
Chris@13:                         ['startLine' => 2], ['endLine' => 2]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         Tokens::T_STRING, 'token',
Chris@13:                         ['startLine' => 2], ['endLine' => 2]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         ord('$'), '$',
Chris@13:                         [
Chris@0:                             'startLine' => 3,
Chris@13:                             'comments' => [
Chris@13:                                 new Comment\Doc('/** doc' . "\n" . 'comment */', 2, 14, 5),
Chris@13:                             ]
Chris@13:                         ],
Chris@13:                         ['endLine' => 3]
Chris@13:                     ],
Chris@13:                 ]
Chris@13:             ],
Chris@0:             // tests comment extraction
Chris@13:             [
Chris@0:                 '<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
Chris@13:                 [],
Chris@13:                 [
Chris@13:                     [
Chris@0:                         Tokens::T_STRING, 'token',
Chris@13:                         [
Chris@0:                             'startLine' => 2,
Chris@13:                             'comments' => [
Chris@13:                                 new Comment('/* comment */', 1, 6, 1),
Chris@13:                                 new Comment('// comment' . "\n", 1, 20, 3),
Chris@13:                                 new Comment\Doc('/** docComment 1 */', 2, 31, 4),
Chris@13:                                 new Comment\Doc('/** docComment 2 */', 2, 50, 5),
Chris@13:                             ],
Chris@13:                         ],
Chris@13:                         ['endLine' => 2]
Chris@13:                     ],
Chris@13:                 ]
Chris@13:             ],
Chris@0:             // tests differing start and end line
Chris@13:             [
Chris@0:                 '<?php "foo' . "\n" . 'bar"',
Chris@13:                 [],
Chris@13:                 [
Chris@13:                     [
Chris@0:                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
Chris@13:                         ['startLine' => 1], ['endLine' => 2]
Chris@13:                     ],
Chris@13:                 ]
Chris@13:             ],
Chris@0:             // tests exact file offsets
Chris@13:             [
Chris@0:                 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
Chris@13:                 ['usedAttributes' => ['startFilePos', 'endFilePos']],
Chris@13:                 [
Chris@13:                     [
Chris@0:                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
Chris@13:                         ['startFilePos' => 6], ['endFilePos' => 8]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         ord(';'), ';',
Chris@13:                         ['startFilePos' => 9], ['endFilePos' => 9]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
Chris@13:                         ['startFilePos' => 18], ['endFilePos' => 20]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         ord(';'), ';',
Chris@13:                         ['startFilePos' => 21], ['endFilePos' => 21]
Chris@13:                     ],
Chris@13:                 ]
Chris@13:             ],
Chris@0:             // tests token offsets
Chris@13:             [
Chris@0:                 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
Chris@13:                 ['usedAttributes' => ['startTokenPos', 'endTokenPos']],
Chris@13:                 [
Chris@13:                     [
Chris@0:                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
Chris@13:                         ['startTokenPos' => 1], ['endTokenPos' => 1]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         ord(';'), ';',
Chris@13:                         ['startTokenPos' => 2], ['endTokenPos' => 2]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
Chris@13:                         ['startTokenPos' => 5], ['endTokenPos' => 5]
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         ord(';'), ';',
Chris@13:                         ['startTokenPos' => 6], ['endTokenPos' => 6]
Chris@13:                     ],
Chris@13:                 ]
Chris@13:             ],
Chris@0:             // tests all attributes being disabled
Chris@13:             [
Chris@0:                 '<?php /* foo */ $bar;',
Chris@13:                 ['usedAttributes' => []],
Chris@13:                 [
Chris@13:                     [
Chris@0:                         Tokens::T_VARIABLE, '$bar',
Chris@13:                         [], []
Chris@13:                     ],
Chris@13:                     [
Chris@0:                         ord(';'), ';',
Chris@13:                         [], []
Chris@13:                     ]
Chris@13:                 ]
Chris@13:             ],
Chris@0:             // tests no tokens
Chris@13:             [
Chris@0:                 '',
Chris@13:                 [],
Chris@13:                 []
Chris@13:             ],
Chris@13:         ];
Chris@0:     }
Chris@0: 
Chris@0:     /**
Chris@0:      * @dataProvider provideTestHaltCompiler
Chris@0:      */
Chris@0:     public function testHandleHaltCompiler($code, $remaining) {
Chris@0:         $lexer = $this->getLexer();
Chris@0:         $lexer->startLexing($code);
Chris@0: 
Chris@0:         while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
Chris@0: 
Chris@0:         $this->assertSame($remaining, $lexer->handleHaltCompiler());
Chris@0:         $this->assertSame(0, $lexer->getNextToken());
Chris@0:     }
Chris@0: 
Chris@0:     public function provideTestHaltCompiler() {
Chris@13:         return [
Chris@13:             ['<?php ... __halt_compiler();Remaining Text', 'Remaining Text'],
Chris@13:             ['<?php ... __halt_compiler ( ) ;Remaining Text', 'Remaining Text'],
Chris@13:             ['<?php ... __halt_compiler() ?>Remaining Text', 'Remaining Text'],
Chris@0:             //array('<?php ... __halt_compiler();' . "\0", "\0"),
Chris@0:             //array('<?php ... __halt_compiler /* */ ( ) ;Remaining Text', 'Remaining Text'),
Chris@13:         ];
Chris@0:     }
Chris@0: 
Chris@0:     public function testHandleHaltCompilerError() {
Chris@17:         $this->expectException(Error::class);
Chris@17:         $this->expectExceptionMessage('__HALT_COMPILER must be followed by "();"');
Chris@0:         $lexer = $this->getLexer();
Chris@0:         $lexer->startLexing('<?php ... __halt_compiler invalid ();');
Chris@0: 
Chris@0:         while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
Chris@0:         $lexer->handleHaltCompiler();
Chris@0:     }
Chris@0: 
Chris@0:     public function testGetTokens() {
Chris@0:         $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
Chris@13:         $expectedTokens = [
Chris@13:             [T_OPEN_TAG, '<?php ', 1],
Chris@13:             [T_CONSTANT_ENCAPSED_STRING, '"a"', 1],
Chris@0:             ';',
Chris@13:             [T_WHITESPACE, "\n", 1],
Chris@13:             [T_COMMENT, '// foo' . "\n", 2],
Chris@13:             [T_CONSTANT_ENCAPSED_STRING, '"b"', 3],
Chris@0:             ';',
Chris@13:         ];
Chris@0: 
Chris@0:         $lexer = $this->getLexer();
Chris@0:         $lexer->startLexing($code);
Chris@0:         $this->assertSame($expectedTokens, $lexer->getTokens());
Chris@0:     }
Chris@0: }