Chris@13
|
1 <?php declare(strict_types=1);
|
Chris@0
|
2
|
Chris@0
|
3 namespace PhpParser;
|
Chris@0
|
4
|
Chris@0
|
5 use PhpParser\Parser\Tokens;
|
Chris@0
|
6
|
Chris@17
|
7 class LexerTest extends \PHPUnit\Framework\TestCase
|
Chris@0
|
8 {
|
Chris@0
|
9 /* To allow overwriting in parent class */
|
Chris@13
|
10 protected function getLexer(array $options = []) {
|
Chris@0
|
11 return new Lexer($options);
|
Chris@0
|
12 }
|
Chris@0
|
13
|
Chris@0
|
14 /**
|
Chris@0
|
15 * @dataProvider provideTestError
|
Chris@0
|
16 */
|
Chris@0
|
17 public function testError($code, $messages) {
|
Chris@0
|
18 if (defined('HHVM_VERSION')) {
|
Chris@0
|
19 $this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
|
Chris@0
|
20 }
|
Chris@0
|
21
|
Chris@0
|
22 $errorHandler = new ErrorHandler\Collecting();
|
Chris@0
|
23 $lexer = $this->getLexer(['usedAttributes' => [
|
Chris@0
|
24 'comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'
|
Chris@0
|
25 ]]);
|
Chris@0
|
26 $lexer->startLexing($code, $errorHandler);
|
Chris@0
|
27 $errors = $errorHandler->getErrors();
|
Chris@0
|
28
|
Chris@13
|
29 $this->assertCount(count($messages), $errors);
|
Chris@0
|
30 for ($i = 0; $i < count($messages); $i++) {
|
Chris@0
|
31 $this->assertSame($messages[$i], $errors[$i]->getMessageWithColumnInfo($code));
|
Chris@0
|
32 }
|
Chris@0
|
33 }
|
Chris@0
|
34
|
Chris@0
|
35 public function provideTestError() {
|
Chris@13
|
36 return [
|
Chris@13
|
37 ["<?php /*", ["Unterminated comment from 1:7 to 1:9"]],
|
Chris@13
|
38 ["<?php \1", ["Unexpected character \"\1\" (ASCII 1) from 1:7 to 1:7"]],
|
Chris@13
|
39 ["<?php \0", ["Unexpected null byte from 1:7 to 1:7"]],
|
Chris@0
|
40 // Error with potentially emulated token
|
Chris@13
|
41 ["<?php ?? \0", ["Unexpected null byte from 1:10 to 1:10"]],
|
Chris@13
|
42 ["<?php\n\0\1 foo /* bar", [
|
Chris@0
|
43 "Unexpected null byte from 2:1 to 2:1",
|
Chris@0
|
44 "Unexpected character \"\1\" (ASCII 1) from 2:2 to 2:2",
|
Chris@0
|
45 "Unterminated comment from 2:8 to 2:14"
|
Chris@13
|
46 ]],
|
Chris@13
|
47 ];
|
Chris@0
|
48 }
|
Chris@0
|
49
|
Chris@0
|
50 /**
|
Chris@0
|
51 * @dataProvider provideTestLex
|
Chris@0
|
52 */
|
Chris@0
|
53 public function testLex($code, $options, $tokens) {
|
Chris@0
|
54 $lexer = $this->getLexer($options);
|
Chris@0
|
55 $lexer->startLexing($code);
|
Chris@0
|
56 while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
|
Chris@0
|
57 $token = array_shift($tokens);
|
Chris@0
|
58
|
Chris@0
|
59 $this->assertSame($token[0], $id);
|
Chris@0
|
60 $this->assertSame($token[1], $value);
|
Chris@0
|
61 $this->assertEquals($token[2], $startAttributes);
|
Chris@0
|
62 $this->assertEquals($token[3], $endAttributes);
|
Chris@0
|
63 }
|
Chris@0
|
64 }
|
Chris@0
|
65
|
Chris@0
|
66 public function provideTestLex() {
|
Chris@13
|
67 return [
|
Chris@0
|
68 // tests conversion of closing PHP tag and drop of whitespace and opening tags
|
Chris@13
|
69 [
|
Chris@0
|
70 '<?php tokens ?>plaintext',
|
Chris@13
|
71 [],
|
Chris@13
|
72 [
|
Chris@13
|
73 [
|
Chris@0
|
74 Tokens::T_STRING, 'tokens',
|
Chris@13
|
75 ['startLine' => 1], ['endLine' => 1]
|
Chris@13
|
76 ],
|
Chris@13
|
77 [
|
Chris@0
|
78 ord(';'), '?>',
|
Chris@13
|
79 ['startLine' => 1], ['endLine' => 1]
|
Chris@13
|
80 ],
|
Chris@13
|
81 [
|
Chris@0
|
82 Tokens::T_INLINE_HTML, 'plaintext',
|
Chris@13
|
83 ['startLine' => 1, 'hasLeadingNewline' => false],
|
Chris@13
|
84 ['endLine' => 1]
|
Chris@13
|
85 ],
|
Chris@13
|
86 ]
|
Chris@13
|
87 ],
|
Chris@0
|
88 // tests line numbers
|
Chris@13
|
89 [
|
Chris@0
|
90 '<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
|
Chris@13
|
91 [],
|
Chris@13
|
92 [
|
Chris@13
|
93 [
|
Chris@0
|
94 ord('$'), '$',
|
Chris@13
|
95 ['startLine' => 2], ['endLine' => 2]
|
Chris@13
|
96 ],
|
Chris@13
|
97 [
|
Chris@0
|
98 Tokens::T_STRING, 'token',
|
Chris@13
|
99 ['startLine' => 2], ['endLine' => 2]
|
Chris@13
|
100 ],
|
Chris@13
|
101 [
|
Chris@0
|
102 ord('$'), '$',
|
Chris@13
|
103 [
|
Chris@0
|
104 'startLine' => 3,
|
Chris@13
|
105 'comments' => [
|
Chris@13
|
106 new Comment\Doc('/** doc' . "\n" . 'comment */', 2, 14, 5),
|
Chris@13
|
107 ]
|
Chris@13
|
108 ],
|
Chris@13
|
109 ['endLine' => 3]
|
Chris@13
|
110 ],
|
Chris@13
|
111 ]
|
Chris@13
|
112 ],
|
Chris@0
|
113 // tests comment extraction
|
Chris@13
|
114 [
|
Chris@0
|
115 '<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
|
Chris@13
|
116 [],
|
Chris@13
|
117 [
|
Chris@13
|
118 [
|
Chris@0
|
119 Tokens::T_STRING, 'token',
|
Chris@13
|
120 [
|
Chris@0
|
121 'startLine' => 2,
|
Chris@13
|
122 'comments' => [
|
Chris@13
|
123 new Comment('/* comment */', 1, 6, 1),
|
Chris@13
|
124 new Comment('// comment' . "\n", 1, 20, 3),
|
Chris@13
|
125 new Comment\Doc('/** docComment 1 */', 2, 31, 4),
|
Chris@13
|
126 new Comment\Doc('/** docComment 2 */', 2, 50, 5),
|
Chris@13
|
127 ],
|
Chris@13
|
128 ],
|
Chris@13
|
129 ['endLine' => 2]
|
Chris@13
|
130 ],
|
Chris@13
|
131 ]
|
Chris@13
|
132 ],
|
Chris@0
|
133 // tests differing start and end line
|
Chris@13
|
134 [
|
Chris@0
|
135 '<?php "foo' . "\n" . 'bar"',
|
Chris@13
|
136 [],
|
Chris@13
|
137 [
|
Chris@13
|
138 [
|
Chris@0
|
139 Tokens::T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
|
Chris@13
|
140 ['startLine' => 1], ['endLine' => 2]
|
Chris@13
|
141 ],
|
Chris@13
|
142 ]
|
Chris@13
|
143 ],
|
Chris@0
|
144 // tests exact file offsets
|
Chris@13
|
145 [
|
Chris@0
|
146 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
|
Chris@13
|
147 ['usedAttributes' => ['startFilePos', 'endFilePos']],
|
Chris@13
|
148 [
|
Chris@13
|
149 [
|
Chris@0
|
150 Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
|
Chris@13
|
151 ['startFilePos' => 6], ['endFilePos' => 8]
|
Chris@13
|
152 ],
|
Chris@13
|
153 [
|
Chris@0
|
154 ord(';'), ';',
|
Chris@13
|
155 ['startFilePos' => 9], ['endFilePos' => 9]
|
Chris@13
|
156 ],
|
Chris@13
|
157 [
|
Chris@0
|
158 Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
|
Chris@13
|
159 ['startFilePos' => 18], ['endFilePos' => 20]
|
Chris@13
|
160 ],
|
Chris@13
|
161 [
|
Chris@0
|
162 ord(';'), ';',
|
Chris@13
|
163 ['startFilePos' => 21], ['endFilePos' => 21]
|
Chris@13
|
164 ],
|
Chris@13
|
165 ]
|
Chris@13
|
166 ],
|
Chris@0
|
167 // tests token offsets
|
Chris@13
|
168 [
|
Chris@0
|
169 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
|
Chris@13
|
170 ['usedAttributes' => ['startTokenPos', 'endTokenPos']],
|
Chris@13
|
171 [
|
Chris@13
|
172 [
|
Chris@0
|
173 Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
|
Chris@13
|
174 ['startTokenPos' => 1], ['endTokenPos' => 1]
|
Chris@13
|
175 ],
|
Chris@13
|
176 [
|
Chris@0
|
177 ord(';'), ';',
|
Chris@13
|
178 ['startTokenPos' => 2], ['endTokenPos' => 2]
|
Chris@13
|
179 ],
|
Chris@13
|
180 [
|
Chris@0
|
181 Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
|
Chris@13
|
182 ['startTokenPos' => 5], ['endTokenPos' => 5]
|
Chris@13
|
183 ],
|
Chris@13
|
184 [
|
Chris@0
|
185 ord(';'), ';',
|
Chris@13
|
186 ['startTokenPos' => 6], ['endTokenPos' => 6]
|
Chris@13
|
187 ],
|
Chris@13
|
188 ]
|
Chris@13
|
189 ],
|
Chris@0
|
190 // tests all attributes being disabled
|
Chris@13
|
191 [
|
Chris@0
|
192 '<?php /* foo */ $bar;',
|
Chris@13
|
193 ['usedAttributes' => []],
|
Chris@13
|
194 [
|
Chris@13
|
195 [
|
Chris@0
|
196 Tokens::T_VARIABLE, '$bar',
|
Chris@13
|
197 [], []
|
Chris@13
|
198 ],
|
Chris@13
|
199 [
|
Chris@0
|
200 ord(';'), ';',
|
Chris@13
|
201 [], []
|
Chris@13
|
202 ]
|
Chris@13
|
203 ]
|
Chris@13
|
204 ],
|
Chris@0
|
205 // tests no tokens
|
Chris@13
|
206 [
|
Chris@0
|
207 '',
|
Chris@13
|
208 [],
|
Chris@13
|
209 []
|
Chris@13
|
210 ],
|
Chris@13
|
211 ];
|
Chris@0
|
212 }
|
Chris@0
|
213
|
Chris@0
|
214 /**
|
Chris@0
|
215 * @dataProvider provideTestHaltCompiler
|
Chris@0
|
216 */
|
Chris@0
|
217 public function testHandleHaltCompiler($code, $remaining) {
|
Chris@0
|
218 $lexer = $this->getLexer();
|
Chris@0
|
219 $lexer->startLexing($code);
|
Chris@0
|
220
|
Chris@0
|
221 while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
|
Chris@0
|
222
|
Chris@0
|
223 $this->assertSame($remaining, $lexer->handleHaltCompiler());
|
Chris@0
|
224 $this->assertSame(0, $lexer->getNextToken());
|
Chris@0
|
225 }
|
Chris@0
|
226
|
Chris@0
|
227 public function provideTestHaltCompiler() {
|
Chris@13
|
228 return [
|
Chris@13
|
229 ['<?php ... __halt_compiler();Remaining Text', 'Remaining Text'],
|
Chris@13
|
230 ['<?php ... __halt_compiler ( ) ;Remaining Text', 'Remaining Text'],
|
Chris@13
|
231 ['<?php ... __halt_compiler() ?>Remaining Text', 'Remaining Text'],
|
Chris@0
|
232 //array('<?php ... __halt_compiler();' . "\0", "\0"),
|
Chris@0
|
233 //array('<?php ... __halt_compiler /* */ ( ) ;Remaining Text', 'Remaining Text'),
|
Chris@13
|
234 ];
|
Chris@0
|
235 }
|
Chris@0
|
236
|
Chris@0
|
237 public function testHandleHaltCompilerError() {
|
Chris@17
|
238 $this->expectException(Error::class);
|
Chris@17
|
239 $this->expectExceptionMessage('__HALT_COMPILER must be followed by "();"');
|
Chris@0
|
240 $lexer = $this->getLexer();
|
Chris@0
|
241 $lexer->startLexing('<?php ... __halt_compiler invalid ();');
|
Chris@0
|
242
|
Chris@0
|
243 while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
|
Chris@0
|
244 $lexer->handleHaltCompiler();
|
Chris@0
|
245 }
|
Chris@0
|
246
|
Chris@0
|
247 public function testGetTokens() {
|
Chris@0
|
248 $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
|
Chris@13
|
249 $expectedTokens = [
|
Chris@13
|
250 [T_OPEN_TAG, '<?php ', 1],
|
Chris@13
|
251 [T_CONSTANT_ENCAPSED_STRING, '"a"', 1],
|
Chris@0
|
252 ';',
|
Chris@13
|
253 [T_WHITESPACE, "\n", 1],
|
Chris@13
|
254 [T_COMMENT, '// foo' . "\n", 2],
|
Chris@13
|
255 [T_CONSTANT_ENCAPSED_STRING, '"b"', 3],
|
Chris@0
|
256 ';',
|
Chris@13
|
257 ];
|
Chris@0
|
258
|
Chris@0
|
259 $lexer = $this->getLexer();
|
Chris@0
|
260 $lexer->startLexing($code);
|
Chris@0
|
261 $this->assertSame($expectedTokens, $lexer->getTokens());
|
Chris@0
|
262 }
|
Chris@0
|
263 }
|