annotate vendor/nikic/php-parser/test/PhpParser/LexerTest.php @ 19:fa3358dc1485 tip

Add ndrum files
author Chris Cannam
date Wed, 28 Aug 2019 13:14:47 +0100
parents 129ea1e6d783
children
rev   line source
Chris@13 1 <?php declare(strict_types=1);
Chris@0 2
Chris@0 3 namespace PhpParser;
Chris@0 4
Chris@0 5 use PhpParser\Parser\Tokens;
Chris@0 6
Chris@17 7 class LexerTest extends \PHPUnit\Framework\TestCase
Chris@0 8 {
Chris@0 9 /* To allow overwriting in parent class */
Chris@13 10 protected function getLexer(array $options = []) {
Chris@0 11 return new Lexer($options);
Chris@0 12 }
Chris@0 13
Chris@0 14 /**
Chris@0 15 * @dataProvider provideTestError
Chris@0 16 */
Chris@0 17 public function testError($code, $messages) {
Chris@0 18 if (defined('HHVM_VERSION')) {
Chris@0 19 $this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
Chris@0 20 }
Chris@0 21
Chris@0 22 $errorHandler = new ErrorHandler\Collecting();
Chris@0 23 $lexer = $this->getLexer(['usedAttributes' => [
Chris@0 24 'comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'
Chris@0 25 ]]);
Chris@0 26 $lexer->startLexing($code, $errorHandler);
Chris@0 27 $errors = $errorHandler->getErrors();
Chris@0 28
Chris@13 29 $this->assertCount(count($messages), $errors);
Chris@0 30 for ($i = 0; $i < count($messages); $i++) {
Chris@0 31 $this->assertSame($messages[$i], $errors[$i]->getMessageWithColumnInfo($code));
Chris@0 32 }
Chris@0 33 }
Chris@0 34
Chris@0 35 public function provideTestError() {
Chris@13 36 return [
Chris@13 37 ["<?php /*", ["Unterminated comment from 1:7 to 1:9"]],
Chris@13 38 ["<?php \1", ["Unexpected character \"\1\" (ASCII 1) from 1:7 to 1:7"]],
Chris@13 39 ["<?php \0", ["Unexpected null byte from 1:7 to 1:7"]],
Chris@0 40 // Error with potentially emulated token
Chris@13 41 ["<?php ?? \0", ["Unexpected null byte from 1:10 to 1:10"]],
Chris@13 42 ["<?php\n\0\1 foo /* bar", [
Chris@0 43 "Unexpected null byte from 2:1 to 2:1",
Chris@0 44 "Unexpected character \"\1\" (ASCII 1) from 2:2 to 2:2",
Chris@0 45 "Unterminated comment from 2:8 to 2:14"
Chris@13 46 ]],
Chris@13 47 ];
Chris@0 48 }
Chris@0 49
Chris@0 50 /**
Chris@0 51 * @dataProvider provideTestLex
Chris@0 52 */
Chris@0 53 public function testLex($code, $options, $tokens) {
Chris@0 54 $lexer = $this->getLexer($options);
Chris@0 55 $lexer->startLexing($code);
Chris@0 56 while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
Chris@0 57 $token = array_shift($tokens);
Chris@0 58
Chris@0 59 $this->assertSame($token[0], $id);
Chris@0 60 $this->assertSame($token[1], $value);
Chris@0 61 $this->assertEquals($token[2], $startAttributes);
Chris@0 62 $this->assertEquals($token[3], $endAttributes);
Chris@0 63 }
Chris@0 64 }
Chris@0 65
Chris@0 66 public function provideTestLex() {
Chris@13 67 return [
Chris@0 68 // tests conversion of closing PHP tag and drop of whitespace and opening tags
Chris@13 69 [
Chris@0 70 '<?php tokens ?>plaintext',
Chris@13 71 [],
Chris@13 72 [
Chris@13 73 [
Chris@0 74 Tokens::T_STRING, 'tokens',
Chris@13 75 ['startLine' => 1], ['endLine' => 1]
Chris@13 76 ],
Chris@13 77 [
Chris@0 78 ord(';'), '?>',
Chris@13 79 ['startLine' => 1], ['endLine' => 1]
Chris@13 80 ],
Chris@13 81 [
Chris@0 82 Tokens::T_INLINE_HTML, 'plaintext',
Chris@13 83 ['startLine' => 1, 'hasLeadingNewline' => false],
Chris@13 84 ['endLine' => 1]
Chris@13 85 ],
Chris@13 86 ]
Chris@13 87 ],
Chris@0 88 // tests line numbers
Chris@13 89 [
Chris@0 90 '<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
Chris@13 91 [],
Chris@13 92 [
Chris@13 93 [
Chris@0 94 ord('$'), '$',
Chris@13 95 ['startLine' => 2], ['endLine' => 2]
Chris@13 96 ],
Chris@13 97 [
Chris@0 98 Tokens::T_STRING, 'token',
Chris@13 99 ['startLine' => 2], ['endLine' => 2]
Chris@13 100 ],
Chris@13 101 [
Chris@0 102 ord('$'), '$',
Chris@13 103 [
Chris@0 104 'startLine' => 3,
Chris@13 105 'comments' => [
Chris@13 106 new Comment\Doc('/** doc' . "\n" . 'comment */', 2, 14, 5),
Chris@13 107 ]
Chris@13 108 ],
Chris@13 109 ['endLine' => 3]
Chris@13 110 ],
Chris@13 111 ]
Chris@13 112 ],
Chris@0 113 // tests comment extraction
Chris@13 114 [
Chris@0 115 '<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
Chris@13 116 [],
Chris@13 117 [
Chris@13 118 [
Chris@0 119 Tokens::T_STRING, 'token',
Chris@13 120 [
Chris@0 121 'startLine' => 2,
Chris@13 122 'comments' => [
Chris@13 123 new Comment('/* comment */', 1, 6, 1),
Chris@13 124 new Comment('// comment' . "\n", 1, 20, 3),
Chris@13 125 new Comment\Doc('/** docComment 1 */', 2, 31, 4),
Chris@13 126 new Comment\Doc('/** docComment 2 */', 2, 50, 5),
Chris@13 127 ],
Chris@13 128 ],
Chris@13 129 ['endLine' => 2]
Chris@13 130 ],
Chris@13 131 ]
Chris@13 132 ],
Chris@0 133 // tests differing start and end line
Chris@13 134 [
Chris@0 135 '<?php "foo' . "\n" . 'bar"',
Chris@13 136 [],
Chris@13 137 [
Chris@13 138 [
Chris@0 139 Tokens::T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
Chris@13 140 ['startLine' => 1], ['endLine' => 2]
Chris@13 141 ],
Chris@13 142 ]
Chris@13 143 ],
Chris@0 144 // tests exact file offsets
Chris@13 145 [
Chris@0 146 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
Chris@13 147 ['usedAttributes' => ['startFilePos', 'endFilePos']],
Chris@13 148 [
Chris@13 149 [
Chris@0 150 Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
Chris@13 151 ['startFilePos' => 6], ['endFilePos' => 8]
Chris@13 152 ],
Chris@13 153 [
Chris@0 154 ord(';'), ';',
Chris@13 155 ['startFilePos' => 9], ['endFilePos' => 9]
Chris@13 156 ],
Chris@13 157 [
Chris@0 158 Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
Chris@13 159 ['startFilePos' => 18], ['endFilePos' => 20]
Chris@13 160 ],
Chris@13 161 [
Chris@0 162 ord(';'), ';',
Chris@13 163 ['startFilePos' => 21], ['endFilePos' => 21]
Chris@13 164 ],
Chris@13 165 ]
Chris@13 166 ],
Chris@0 167 // tests token offsets
Chris@13 168 [
Chris@0 169 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
Chris@13 170 ['usedAttributes' => ['startTokenPos', 'endTokenPos']],
Chris@13 171 [
Chris@13 172 [
Chris@0 173 Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
Chris@13 174 ['startTokenPos' => 1], ['endTokenPos' => 1]
Chris@13 175 ],
Chris@13 176 [
Chris@0 177 ord(';'), ';',
Chris@13 178 ['startTokenPos' => 2], ['endTokenPos' => 2]
Chris@13 179 ],
Chris@13 180 [
Chris@0 181 Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
Chris@13 182 ['startTokenPos' => 5], ['endTokenPos' => 5]
Chris@13 183 ],
Chris@13 184 [
Chris@0 185 ord(';'), ';',
Chris@13 186 ['startTokenPos' => 6], ['endTokenPos' => 6]
Chris@13 187 ],
Chris@13 188 ]
Chris@13 189 ],
Chris@0 190 // tests all attributes being disabled
Chris@13 191 [
Chris@0 192 '<?php /* foo */ $bar;',
Chris@13 193 ['usedAttributes' => []],
Chris@13 194 [
Chris@13 195 [
Chris@0 196 Tokens::T_VARIABLE, '$bar',
Chris@13 197 [], []
Chris@13 198 ],
Chris@13 199 [
Chris@0 200 ord(';'), ';',
Chris@13 201 [], []
Chris@13 202 ]
Chris@13 203 ]
Chris@13 204 ],
Chris@0 205 // tests no tokens
Chris@13 206 [
Chris@0 207 '',
Chris@13 208 [],
Chris@13 209 []
Chris@13 210 ],
Chris@13 211 ];
Chris@0 212 }
Chris@0 213
Chris@0 214 /**
Chris@0 215 * @dataProvider provideTestHaltCompiler
Chris@0 216 */
Chris@0 217 public function testHandleHaltCompiler($code, $remaining) {
Chris@0 218 $lexer = $this->getLexer();
Chris@0 219 $lexer->startLexing($code);
Chris@0 220
Chris@0 221 while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
Chris@0 222
Chris@0 223 $this->assertSame($remaining, $lexer->handleHaltCompiler());
Chris@0 224 $this->assertSame(0, $lexer->getNextToken());
Chris@0 225 }
Chris@0 226
Chris@0 227 public function provideTestHaltCompiler() {
Chris@13 228 return [
Chris@13 229 ['<?php ... __halt_compiler();Remaining Text', 'Remaining Text'],
Chris@13 230 ['<?php ... __halt_compiler ( ) ;Remaining Text', 'Remaining Text'],
Chris@13 231 ['<?php ... __halt_compiler() ?>Remaining Text', 'Remaining Text'],
Chris@0 232 //array('<?php ... __halt_compiler();' . "\0", "\0"),
Chris@0 233 //array('<?php ... __halt_compiler /* */ ( ) ;Remaining Text', 'Remaining Text'),
Chris@13 234 ];
Chris@0 235 }
Chris@0 236
Chris@0 237 public function testHandleHaltCompilerError() {
Chris@17 238 $this->expectException(Error::class);
Chris@17 239 $this->expectExceptionMessage('__HALT_COMPILER must be followed by "();"');
Chris@0 240 $lexer = $this->getLexer();
Chris@0 241 $lexer->startLexing('<?php ... __halt_compiler invalid ();');
Chris@0 242
Chris@0 243 while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
Chris@0 244 $lexer->handleHaltCompiler();
Chris@0 245 }
Chris@0 246
Chris@0 247 public function testGetTokens() {
Chris@0 248 $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
Chris@13 249 $expectedTokens = [
Chris@13 250 [T_OPEN_TAG, '<?php ', 1],
Chris@13 251 [T_CONSTANT_ENCAPSED_STRING, '"a"', 1],
Chris@0 252 ';',
Chris@13 253 [T_WHITESPACE, "\n", 1],
Chris@13 254 [T_COMMENT, '// foo' . "\n", 2],
Chris@13 255 [T_CONSTANT_ENCAPSED_STRING, '"b"', 3],
Chris@0 256 ';',
Chris@13 257 ];
Chris@0 258
Chris@0 259 $lexer = $this->getLexer();
Chris@0 260 $lexer->startLexing($code);
Chris@0 261 $this->assertSame($expectedTokens, $lexer->getTokens());
Chris@0 262 }
Chris@0 263 }