annotate vendor/squizlabs/php_codesniffer/src/Tokenizers/PHP.php @ 19:fa3358dc1485 tip

Add ndrum files
author Chris Cannam
date Wed, 28 Aug 2019 13:14:47 +0100
parents af1871eacc83
children
rev   line source
Chris@17 1 <?php
Chris@17 2 /**
Chris@17 3 * Tokenizes PHP code.
Chris@17 4 *
Chris@17 5 * @author Greg Sherwood <gsherwood@squiz.net>
Chris@17 6 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
Chris@17 7 * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
Chris@17 8 */
Chris@17 9
Chris@17 10 namespace PHP_CodeSniffer\Tokenizers;
Chris@17 11
Chris@17 12 use PHP_CodeSniffer\Util;
Chris@17 13
Chris@17 14 class PHP extends Tokenizer
Chris@17 15 {
Chris@17 16
Chris@17 17
Chris@17 18 /**
Chris@17 19 * A list of tokens that are allowed to open a scope.
Chris@17 20 *
Chris@17 21 * This array also contains information about what kind of token the scope
Chris@17 22 * opener uses to open and close the scope, if the token strictly requires
Chris@17 23 * an opener, if the token can share a scope closer, and who it can be shared
Chris@17 24 * with. An example of a token that shares a scope closer is a CASE scope.
Chris@17 25 *
Chris@17 26 * @var array
Chris@17 27 */
Chris@17 28 public $scopeOpeners = [
Chris@17 29 T_IF => [
Chris@17 30 'start' => [
Chris@17 31 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
Chris@17 32 T_COLON => T_COLON,
Chris@17 33 ],
Chris@17 34 'end' => [
Chris@17 35 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 36 T_ENDIF => T_ENDIF,
Chris@17 37 T_ELSE => T_ELSE,
Chris@17 38 T_ELSEIF => T_ELSEIF,
Chris@17 39 ],
Chris@17 40 'strict' => false,
Chris@17 41 'shared' => false,
Chris@17 42 'with' => [
Chris@17 43 T_ELSE => T_ELSE,
Chris@17 44 T_ELSEIF => T_ELSEIF,
Chris@17 45 ],
Chris@17 46 ],
Chris@17 47 T_TRY => [
Chris@17 48 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 49 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 50 'strict' => true,
Chris@17 51 'shared' => false,
Chris@17 52 'with' => [],
Chris@17 53 ],
Chris@17 54 T_CATCH => [
Chris@17 55 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 56 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 57 'strict' => true,
Chris@17 58 'shared' => false,
Chris@17 59 'with' => [],
Chris@17 60 ],
Chris@17 61 T_FINALLY => [
Chris@17 62 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 63 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 64 'strict' => true,
Chris@17 65 'shared' => false,
Chris@17 66 'with' => [],
Chris@17 67 ],
Chris@17 68 T_ELSE => [
Chris@17 69 'start' => [
Chris@17 70 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
Chris@17 71 T_COLON => T_COLON,
Chris@17 72 ],
Chris@17 73 'end' => [
Chris@17 74 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 75 T_ENDIF => T_ENDIF,
Chris@17 76 ],
Chris@17 77 'strict' => false,
Chris@17 78 'shared' => false,
Chris@17 79 'with' => [
Chris@17 80 T_IF => T_IF,
Chris@17 81 T_ELSEIF => T_ELSEIF,
Chris@17 82 ],
Chris@17 83 ],
Chris@17 84 T_ELSEIF => [
Chris@17 85 'start' => [
Chris@17 86 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
Chris@17 87 T_COLON => T_COLON,
Chris@17 88 ],
Chris@17 89 'end' => [
Chris@17 90 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 91 T_ENDIF => T_ENDIF,
Chris@17 92 T_ELSE => T_ELSE,
Chris@17 93 T_ELSEIF => T_ELSEIF,
Chris@17 94 ],
Chris@17 95 'strict' => false,
Chris@17 96 'shared' => false,
Chris@17 97 'with' => [
Chris@17 98 T_IF => T_IF,
Chris@17 99 T_ELSE => T_ELSE,
Chris@17 100 ],
Chris@17 101 ],
Chris@17 102 T_FOR => [
Chris@17 103 'start' => [
Chris@17 104 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
Chris@17 105 T_COLON => T_COLON,
Chris@17 106 ],
Chris@17 107 'end' => [
Chris@17 108 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 109 T_ENDFOR => T_ENDFOR,
Chris@17 110 ],
Chris@17 111 'strict' => false,
Chris@17 112 'shared' => false,
Chris@17 113 'with' => [],
Chris@17 114 ],
Chris@17 115 T_FOREACH => [
Chris@17 116 'start' => [
Chris@17 117 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
Chris@17 118 T_COLON => T_COLON,
Chris@17 119 ],
Chris@17 120 'end' => [
Chris@17 121 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 122 T_ENDFOREACH => T_ENDFOREACH,
Chris@17 123 ],
Chris@17 124 'strict' => false,
Chris@17 125 'shared' => false,
Chris@17 126 'with' => [],
Chris@17 127 ],
Chris@17 128 T_INTERFACE => [
Chris@17 129 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 130 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 131 'strict' => true,
Chris@17 132 'shared' => false,
Chris@17 133 'with' => [],
Chris@17 134 ],
Chris@17 135 T_FUNCTION => [
Chris@17 136 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 137 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 138 'strict' => true,
Chris@17 139 'shared' => false,
Chris@17 140 'with' => [],
Chris@17 141 ],
Chris@17 142 T_CLASS => [
Chris@17 143 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 144 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 145 'strict' => true,
Chris@17 146 'shared' => false,
Chris@17 147 'with' => [],
Chris@17 148 ],
Chris@17 149 T_TRAIT => [
Chris@17 150 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 151 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 152 'strict' => true,
Chris@17 153 'shared' => false,
Chris@17 154 'with' => [],
Chris@17 155 ],
Chris@17 156 T_USE => [
Chris@17 157 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 158 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 159 'strict' => false,
Chris@17 160 'shared' => false,
Chris@17 161 'with' => [],
Chris@17 162 ],
Chris@17 163 T_DECLARE => [
Chris@17 164 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 165 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 166 'strict' => false,
Chris@17 167 'shared' => false,
Chris@17 168 'with' => [],
Chris@17 169 ],
Chris@17 170 T_NAMESPACE => [
Chris@17 171 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 172 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 173 'strict' => false,
Chris@17 174 'shared' => false,
Chris@17 175 'with' => [],
Chris@17 176 ],
Chris@17 177 T_WHILE => [
Chris@17 178 'start' => [
Chris@17 179 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
Chris@17 180 T_COLON => T_COLON,
Chris@17 181 ],
Chris@17 182 'end' => [
Chris@17 183 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 184 T_ENDWHILE => T_ENDWHILE,
Chris@17 185 ],
Chris@17 186 'strict' => false,
Chris@17 187 'shared' => false,
Chris@17 188 'with' => [],
Chris@17 189 ],
Chris@17 190 T_DO => [
Chris@17 191 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
Chris@17 192 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
Chris@17 193 'strict' => true,
Chris@17 194 'shared' => false,
Chris@17 195 'with' => [],
Chris@17 196 ],
Chris@17 197 T_SWITCH => [
Chris@17 198 'start' => [
Chris@17 199 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
Chris@17 200 T_COLON => T_COLON,
Chris@17 201 ],
Chris@17 202 'end' => [
Chris@17 203 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 204 T_ENDSWITCH => T_ENDSWITCH,
Chris@17 205 ],
Chris@17 206 'strict' => true,
Chris@17 207 'shared' => false,
Chris@17 208 'with' => [],
Chris@17 209 ],
Chris@17 210 T_CASE => [
Chris@17 211 'start' => [
Chris@17 212 T_COLON => T_COLON,
Chris@17 213 T_SEMICOLON => T_SEMICOLON,
Chris@17 214 ],
Chris@17 215 'end' => [
Chris@17 216 T_BREAK => T_BREAK,
Chris@17 217 T_RETURN => T_RETURN,
Chris@17 218 T_CONTINUE => T_CONTINUE,
Chris@17 219 T_THROW => T_THROW,
Chris@17 220 T_EXIT => T_EXIT,
Chris@17 221 ],
Chris@17 222 'strict' => true,
Chris@17 223 'shared' => true,
Chris@17 224 'with' => [
Chris@17 225 T_DEFAULT => T_DEFAULT,
Chris@17 226 T_CASE => T_CASE,
Chris@17 227 T_SWITCH => T_SWITCH,
Chris@17 228 ],
Chris@17 229 ],
Chris@17 230 T_DEFAULT => [
Chris@17 231 'start' => [
Chris@17 232 T_COLON => T_COLON,
Chris@17 233 T_SEMICOLON => T_SEMICOLON,
Chris@17 234 ],
Chris@17 235 'end' => [
Chris@17 236 T_BREAK => T_BREAK,
Chris@17 237 T_RETURN => T_RETURN,
Chris@17 238 T_CONTINUE => T_CONTINUE,
Chris@17 239 T_THROW => T_THROW,
Chris@17 240 T_EXIT => T_EXIT,
Chris@17 241 ],
Chris@17 242 'strict' => true,
Chris@17 243 'shared' => true,
Chris@17 244 'with' => [
Chris@17 245 T_CASE => T_CASE,
Chris@17 246 T_SWITCH => T_SWITCH,
Chris@17 247 ],
Chris@17 248 ],
Chris@17 249 T_START_HEREDOC => [
Chris@17 250 'start' => [T_START_HEREDOC => T_START_HEREDOC],
Chris@17 251 'end' => [T_END_HEREDOC => T_END_HEREDOC],
Chris@17 252 'strict' => true,
Chris@17 253 'shared' => false,
Chris@17 254 'with' => [],
Chris@17 255 ],
Chris@17 256 T_START_NOWDOC => [
Chris@17 257 'start' => [T_START_NOWDOC => T_START_NOWDOC],
Chris@17 258 'end' => [T_END_NOWDOC => T_END_NOWDOC],
Chris@17 259 'strict' => true,
Chris@17 260 'shared' => false,
Chris@17 261 'with' => [],
Chris@17 262 ],
Chris@17 263 ];
Chris@17 264
Chris@17 265 /**
Chris@17 266 * A list of tokens that end the scope.
Chris@17 267 *
Chris@17 268 * This array is just a unique collection of the end tokens
Chris@18 269 * from the scopeOpeners array. The data is duplicated here to
Chris@17 270 * save time during parsing of the file.
Chris@17 271 *
Chris@17 272 * @var array
Chris@17 273 */
Chris@17 274 public $endScopeTokens = [
Chris@17 275 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 276 T_ENDIF => T_ENDIF,
Chris@17 277 T_ENDFOR => T_ENDFOR,
Chris@17 278 T_ENDFOREACH => T_ENDFOREACH,
Chris@17 279 T_ENDWHILE => T_ENDWHILE,
Chris@17 280 T_ENDSWITCH => T_ENDSWITCH,
Chris@17 281 T_BREAK => T_BREAK,
Chris@17 282 T_END_HEREDOC => T_END_HEREDOC,
Chris@17 283 ];
Chris@17 284
Chris@17 285 /**
Chris@17 286 * Known lengths of tokens.
Chris@17 287 *
Chris@17 288 * @var array<int, int>
Chris@17 289 */
Chris@17 290 public $knownLengths = [
Chris@17 291 T_ABSTRACT => 8,
Chris@17 292 T_AND_EQUAL => 2,
Chris@17 293 T_ARRAY => 5,
Chris@17 294 T_AS => 2,
Chris@17 295 T_BOOLEAN_AND => 2,
Chris@17 296 T_BOOLEAN_OR => 2,
Chris@17 297 T_BREAK => 5,
Chris@17 298 T_CALLABLE => 8,
Chris@17 299 T_CASE => 4,
Chris@17 300 T_CATCH => 5,
Chris@17 301 T_CLASS => 5,
Chris@17 302 T_CLASS_C => 9,
Chris@17 303 T_CLONE => 5,
Chris@17 304 T_CONCAT_EQUAL => 2,
Chris@17 305 T_CONST => 5,
Chris@17 306 T_CONTINUE => 8,
Chris@17 307 T_CURLY_OPEN => 2,
Chris@17 308 T_DEC => 2,
Chris@17 309 T_DECLARE => 7,
Chris@17 310 T_DEFAULT => 7,
Chris@17 311 T_DIR => 7,
Chris@17 312 T_DIV_EQUAL => 2,
Chris@17 313 T_DO => 2,
Chris@17 314 T_DOLLAR_OPEN_CURLY_BRACES => 2,
Chris@17 315 T_DOUBLE_ARROW => 2,
Chris@17 316 T_DOUBLE_COLON => 2,
Chris@17 317 T_ECHO => 4,
Chris@17 318 T_ELSE => 4,
Chris@17 319 T_ELSEIF => 6,
Chris@17 320 T_EMPTY => 5,
Chris@17 321 T_ENDDECLARE => 10,
Chris@17 322 T_ENDFOR => 6,
Chris@17 323 T_ENDFOREACH => 10,
Chris@17 324 T_ENDIF => 5,
Chris@17 325 T_ENDSWITCH => 9,
Chris@17 326 T_ENDWHILE => 8,
Chris@17 327 T_EVAL => 4,
Chris@17 328 T_EXTENDS => 7,
Chris@17 329 T_FILE => 8,
Chris@17 330 T_FINAL => 5,
Chris@17 331 T_FINALLY => 7,
Chris@17 332 T_FOR => 3,
Chris@17 333 T_FOREACH => 7,
Chris@17 334 T_FUNCTION => 8,
Chris@17 335 T_FUNC_C => 12,
Chris@17 336 T_GLOBAL => 6,
Chris@17 337 T_GOTO => 4,
Chris@17 338 T_HALT_COMPILER => 15,
Chris@17 339 T_IF => 2,
Chris@17 340 T_IMPLEMENTS => 10,
Chris@17 341 T_INC => 2,
Chris@17 342 T_INCLUDE => 7,
Chris@17 343 T_INCLUDE_ONCE => 12,
Chris@17 344 T_INSTANCEOF => 10,
Chris@17 345 T_INSTEADOF => 9,
Chris@17 346 T_INTERFACE => 9,
Chris@17 347 T_ISSET => 5,
Chris@17 348 T_IS_EQUAL => 2,
Chris@17 349 T_IS_GREATER_OR_EQUAL => 2,
Chris@17 350 T_IS_IDENTICAL => 3,
Chris@17 351 T_IS_NOT_EQUAL => 2,
Chris@17 352 T_IS_NOT_IDENTICAL => 3,
Chris@17 353 T_IS_SMALLER_OR_EQUAL => 2,
Chris@17 354 T_LINE => 8,
Chris@17 355 T_LIST => 4,
Chris@17 356 T_LOGICAL_AND => 3,
Chris@17 357 T_LOGICAL_OR => 2,
Chris@17 358 T_LOGICAL_XOR => 3,
Chris@17 359 T_METHOD_C => 10,
Chris@17 360 T_MINUS_EQUAL => 2,
Chris@17 361 T_POW_EQUAL => 3,
Chris@17 362 T_MOD_EQUAL => 2,
Chris@17 363 T_MUL_EQUAL => 2,
Chris@17 364 T_NAMESPACE => 9,
Chris@17 365 T_NS_C => 13,
Chris@17 366 T_NS_SEPARATOR => 1,
Chris@17 367 T_NEW => 3,
Chris@17 368 T_OBJECT_OPERATOR => 2,
Chris@17 369 T_OPEN_TAG_WITH_ECHO => 3,
Chris@17 370 T_OR_EQUAL => 2,
Chris@17 371 T_PLUS_EQUAL => 2,
Chris@17 372 T_PRINT => 5,
Chris@17 373 T_PRIVATE => 7,
Chris@17 374 T_PUBLIC => 6,
Chris@17 375 T_PROTECTED => 9,
Chris@17 376 T_REQUIRE => 7,
Chris@17 377 T_REQUIRE_ONCE => 12,
Chris@17 378 T_RETURN => 6,
Chris@17 379 T_STATIC => 6,
Chris@17 380 T_SWITCH => 6,
Chris@17 381 T_THROW => 5,
Chris@17 382 T_TRAIT => 5,
Chris@17 383 T_TRAIT_C => 9,
Chris@17 384 T_TRY => 3,
Chris@17 385 T_UNSET => 5,
Chris@17 386 T_USE => 3,
Chris@17 387 T_VAR => 3,
Chris@17 388 T_WHILE => 5,
Chris@17 389 T_XOR_EQUAL => 2,
Chris@17 390 T_YIELD => 5,
Chris@17 391 T_OPEN_CURLY_BRACKET => 1,
Chris@17 392 T_CLOSE_CURLY_BRACKET => 1,
Chris@17 393 T_OPEN_SQUARE_BRACKET => 1,
Chris@17 394 T_CLOSE_SQUARE_BRACKET => 1,
Chris@17 395 T_OPEN_PARENTHESIS => 1,
Chris@17 396 T_CLOSE_PARENTHESIS => 1,
Chris@17 397 T_COLON => 1,
Chris@17 398 T_STRING_CONCAT => 1,
Chris@17 399 T_INLINE_THEN => 1,
Chris@17 400 T_INLINE_ELSE => 1,
Chris@17 401 T_NULLABLE => 1,
Chris@17 402 T_NULL => 4,
Chris@17 403 T_FALSE => 5,
Chris@17 404 T_TRUE => 4,
Chris@17 405 T_SEMICOLON => 1,
Chris@17 406 T_EQUAL => 1,
Chris@17 407 T_MULTIPLY => 1,
Chris@17 408 T_DIVIDE => 1,
Chris@17 409 T_PLUS => 1,
Chris@17 410 T_MINUS => 1,
Chris@17 411 T_MODULUS => 1,
Chris@17 412 T_POW => 2,
Chris@17 413 T_SPACESHIP => 3,
Chris@17 414 T_COALESCE => 2,
Chris@17 415 T_COALESCE_EQUAL => 3,
Chris@17 416 T_BITWISE_AND => 1,
Chris@17 417 T_BITWISE_OR => 1,
Chris@17 418 T_BITWISE_XOR => 1,
Chris@17 419 T_SL => 2,
Chris@17 420 T_SR => 2,
Chris@17 421 T_SL_EQUAL => 3,
Chris@17 422 T_SR_EQUAL => 3,
Chris@17 423 T_GREATER_THAN => 1,
Chris@17 424 T_LESS_THAN => 1,
Chris@17 425 T_BOOLEAN_NOT => 1,
Chris@17 426 T_SELF => 4,
Chris@17 427 T_PARENT => 6,
Chris@17 428 T_COMMA => 1,
Chris@17 429 T_THIS => 4,
Chris@17 430 T_CLOSURE => 8,
Chris@17 431 T_BACKTICK => 1,
Chris@17 432 T_OPEN_SHORT_ARRAY => 1,
Chris@17 433 T_CLOSE_SHORT_ARRAY => 1,
Chris@17 434 ];
Chris@17 435
Chris@17 436
Chris@17 437 /**
Chris@17 438 * A cache of different token types, resolved into arrays.
Chris@17 439 *
Chris@17 440 * @var array
Chris@17 441 * @see standardiseToken()
Chris@17 442 */
Chris@17 443 private static $resolveTokenCache = [];
Chris@17 444
Chris@17 445
Chris@17 446 /**
Chris@17 447 * Creates an array of tokens when given some PHP code.
Chris@17 448 *
Chris@17 449 * Starts by using token_get_all() but does a lot of extra processing
Chris@17 450 * to insert information about the context of the token.
Chris@17 451 *
Chris@17 452 * @param string $string The string to tokenize.
Chris@17 453 *
Chris@17 454 * @return array
Chris@17 455 */
Chris@17 456 protected function tokenize($string)
Chris@17 457 {
Chris@17 458 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 459 echo "\t*** START PHP TOKENIZING ***".PHP_EOL;
Chris@17 460 $isWin = false;
Chris@17 461 if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') {
Chris@17 462 $isWin = true;
Chris@17 463 }
Chris@17 464 }
Chris@17 465
Chris@17 466 $tokens = @token_get_all($string);
Chris@17 467 $finalTokens = [];
Chris@17 468
Chris@17 469 $newStackPtr = 0;
Chris@17 470 $numTokens = count($tokens);
Chris@17 471 $lastNotEmptyToken = 0;
Chris@17 472
Chris@17 473 $insideInlineIf = [];
Chris@17 474 $insideUseGroup = false;
Chris@17 475
Chris@17 476 $commentTokenizer = new Comment();
Chris@17 477
Chris@17 478 for ($stackPtr = 0; $stackPtr < $numTokens; $stackPtr++) {
Chris@17 479 // Special case for tokens we have needed to blank out.
Chris@17 480 if ($tokens[$stackPtr] === null) {
Chris@17 481 continue;
Chris@17 482 }
Chris@17 483
Chris@17 484 $token = (array) $tokens[$stackPtr];
Chris@17 485 $tokenIsArray = isset($token[1]);
Chris@17 486
Chris@17 487 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 488 if ($tokenIsArray === true) {
Chris@17 489 $type = Util\Tokens::tokenName($token[0]);
Chris@17 490 $content = Util\Common::prepareForOutput($token[1]);
Chris@17 491 } else {
Chris@17 492 $newToken = self::resolveSimpleToken($token[0]);
Chris@17 493 $type = $newToken['type'];
Chris@17 494 $content = Util\Common::prepareForOutput($token[0]);
Chris@17 495 }
Chris@17 496
Chris@17 497 echo "\tProcess token ";
Chris@17 498 if ($tokenIsArray === true) {
Chris@17 499 echo "[$stackPtr]";
Chris@17 500 } else {
Chris@17 501 echo " $stackPtr ";
Chris@17 502 }
Chris@17 503
Chris@17 504 echo ": $type => $content";
Chris@17 505 }//end if
Chris@17 506
Chris@18 507 if ($newStackPtr > 0
Chris@18 508 && isset(Util\Tokens::$emptyTokens[$finalTokens[($newStackPtr - 1)]['code']]) === false
Chris@18 509 ) {
Chris@17 510 $lastNotEmptyToken = ($newStackPtr - 1);
Chris@17 511 }
Chris@17 512
Chris@17 513 /*
Chris@17 514 If we are using \r\n newline characters, the \r and \n are sometimes
Chris@17 515 split over two tokens. This normally occurs after comments. We need
Chris@17 516 to merge these two characters together so that our line endings are
Chris@17 517 consistent for all lines.
Chris@17 518 */
Chris@17 519
Chris@17 520 if ($tokenIsArray === true && substr($token[1], -1) === "\r") {
Chris@17 521 if (isset($tokens[($stackPtr + 1)]) === true
Chris@17 522 && is_array($tokens[($stackPtr + 1)]) === true
Chris@17 523 && $tokens[($stackPtr + 1)][1][0] === "\n"
Chris@17 524 ) {
Chris@17 525 $token[1] .= "\n";
Chris@17 526 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 527 if ($isWin === true) {
Chris@17 528 echo '\n';
Chris@17 529 } else {
Chris@17 530 echo "\033[30;1m\\n\033[0m";
Chris@17 531 }
Chris@17 532 }
Chris@17 533
Chris@17 534 if ($tokens[($stackPtr + 1)][1] === "\n") {
Chris@17 535 // This token's content has been merged into the previous,
Chris@17 536 // so we can skip it.
Chris@17 537 $tokens[($stackPtr + 1)] = '';
Chris@17 538 } else {
Chris@17 539 $tokens[($stackPtr + 1)][1] = substr($tokens[($stackPtr + 1)][1], 1);
Chris@17 540 }
Chris@17 541 }
Chris@17 542 }//end if
Chris@17 543
Chris@17 544 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 545 echo PHP_EOL;
Chris@17 546 }
Chris@17 547
Chris@17 548 /*
Chris@17 549 Parse doc blocks into something that can be easily iterated over.
Chris@17 550 */
Chris@17 551
Chris@17 552 if ($tokenIsArray === true
Chris@17 553 && ($token[0] === T_DOC_COMMENT
Chris@17 554 || ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0))
Chris@17 555 ) {
Chris@17 556 $commentTokens = $commentTokenizer->tokenizeString($token[1], $this->eolChar, $newStackPtr);
Chris@17 557 foreach ($commentTokens as $commentToken) {
Chris@17 558 $finalTokens[$newStackPtr] = $commentToken;
Chris@17 559 $newStackPtr++;
Chris@17 560 }
Chris@17 561
Chris@17 562 continue;
Chris@17 563 }
Chris@17 564
Chris@17 565 /*
Chris@17 566 If this is a double quoted string, PHP will tokenize the whole
Chris@17 567 thing which causes problems with the scope map when braces are
Chris@17 568 within the string. So we need to merge the tokens together to
Chris@17 569 provide a single string.
Chris@17 570 */
Chris@17 571
Chris@17 572 if ($tokenIsArray === false && ($token[0] === '"' || $token[0] === 'b"')) {
Chris@17 573 // Binary casts need a special token.
Chris@17 574 if ($token[0] === 'b"') {
Chris@17 575 $finalTokens[$newStackPtr] = [
Chris@17 576 'code' => T_BINARY_CAST,
Chris@17 577 'type' => 'T_BINARY_CAST',
Chris@17 578 'content' => 'b',
Chris@17 579 ];
Chris@17 580 $newStackPtr++;
Chris@17 581 }
Chris@17 582
Chris@17 583 $tokenContent = '"';
Chris@17 584 $nestedVars = [];
Chris@17 585 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
Chris@17 586 $subToken = (array) $tokens[$i];
Chris@17 587 $subTokenIsArray = isset($subToken[1]);
Chris@17 588
Chris@17 589 if ($subTokenIsArray === true) {
Chris@17 590 $tokenContent .= $subToken[1];
Chris@17 591 if ($subToken[1] === '{'
Chris@17 592 && $subToken[0] !== T_ENCAPSED_AND_WHITESPACE
Chris@17 593 ) {
Chris@17 594 $nestedVars[] = $i;
Chris@17 595 }
Chris@17 596 } else {
Chris@17 597 $tokenContent .= $subToken[0];
Chris@17 598 if ($subToken[0] === '}') {
Chris@17 599 array_pop($nestedVars);
Chris@17 600 }
Chris@17 601 }
Chris@17 602
Chris@17 603 if ($subTokenIsArray === false
Chris@17 604 && $subToken[0] === '"'
Chris@17 605 && empty($nestedVars) === true
Chris@17 606 ) {
Chris@17 607 // We found the other end of the double quoted string.
Chris@17 608 break;
Chris@17 609 }
Chris@17 610 }//end for
Chris@17 611
Chris@17 612 $stackPtr = $i;
Chris@17 613
Chris@17 614 // Convert each line within the double quoted string to a
Chris@17 615 // new token, so it conforms with other multiple line tokens.
Chris@17 616 $tokenLines = explode($this->eolChar, $tokenContent);
Chris@17 617 $numLines = count($tokenLines);
Chris@17 618 $newToken = [];
Chris@17 619
Chris@17 620 for ($j = 0; $j < $numLines; $j++) {
Chris@17 621 $newToken['content'] = $tokenLines[$j];
Chris@17 622 if ($j === ($numLines - 1)) {
Chris@17 623 if ($tokenLines[$j] === '') {
Chris@17 624 break;
Chris@17 625 }
Chris@17 626 } else {
Chris@17 627 $newToken['content'] .= $this->eolChar;
Chris@17 628 }
Chris@17 629
Chris@17 630 $newToken['code'] = T_DOUBLE_QUOTED_STRING;
Chris@17 631 $newToken['type'] = 'T_DOUBLE_QUOTED_STRING';
Chris@17 632 $finalTokens[$newStackPtr] = $newToken;
Chris@17 633 $newStackPtr++;
Chris@17 634 }
Chris@17 635
Chris@17 636 // Continue, as we're done with this token.
Chris@17 637 continue;
Chris@17 638 }//end if
Chris@17 639
Chris@17 640 /*
Chris@17 641 Detect binary casting and assign the casts their own token.
Chris@17 642 */
Chris@17 643
Chris@17 644 if ($tokenIsArray === true
Chris@17 645 && $token[0] === T_CONSTANT_ENCAPSED_STRING
Chris@17 646 && (substr($token[1], 0, 2) === 'b"'
Chris@17 647 || substr($token[1], 0, 2) === "b'")
Chris@17 648 ) {
Chris@17 649 $finalTokens[$newStackPtr] = [
Chris@17 650 'code' => T_BINARY_CAST,
Chris@17 651 'type' => 'T_BINARY_CAST',
Chris@17 652 'content' => 'b',
Chris@17 653 ];
Chris@17 654 $newStackPtr++;
Chris@17 655 $token[1] = substr($token[1], 1);
Chris@17 656 }
Chris@17 657
Chris@17 658 if ($tokenIsArray === true
Chris@17 659 && $token[0] === T_STRING_CAST
Chris@17 660 && preg_match('`^\(\s*binary\s*\)$`i', $token[1]) === 1
Chris@17 661 ) {
Chris@17 662 $finalTokens[$newStackPtr] = [
Chris@17 663 'code' => T_BINARY_CAST,
Chris@17 664 'type' => 'T_BINARY_CAST',
Chris@17 665 'content' => $token[1],
Chris@17 666 ];
Chris@17 667 $newStackPtr++;
Chris@17 668 continue;
Chris@17 669 }
Chris@17 670
Chris@17 671 /*
Chris@17 672 If this is a heredoc, PHP will tokenize the whole
Chris@17 673 thing which causes problems when heredocs don't
Chris@17 674 contain real PHP code, which is almost never.
Chris@17 675 We want to leave the start and end heredoc tokens
Chris@17 676 alone though.
Chris@17 677 */
Chris@17 678
Chris@17 679 if ($tokenIsArray === true && $token[0] === T_START_HEREDOC) {
Chris@17 680 // Add the start heredoc token to the final array.
Chris@17 681 $finalTokens[$newStackPtr] = self::standardiseToken($token);
Chris@17 682
Chris@17 683 // Check if this is actually a nowdoc and use a different token
Chris@17 684 // to help the sniffs.
Chris@17 685 $nowdoc = false;
Chris@17 686 if (strpos($token[1], "'") !== false) {
Chris@17 687 $finalTokens[$newStackPtr]['code'] = T_START_NOWDOC;
Chris@17 688 $finalTokens[$newStackPtr]['type'] = 'T_START_NOWDOC';
Chris@17 689 $nowdoc = true;
Chris@17 690 }
Chris@17 691
Chris@17 692 $tokenContent = '';
Chris@17 693 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
Chris@17 694 $subTokenIsArray = is_array($tokens[$i]);
Chris@17 695 if ($subTokenIsArray === true
Chris@17 696 && $tokens[$i][0] === T_END_HEREDOC
Chris@17 697 ) {
Chris@17 698 // We found the other end of the heredoc.
Chris@17 699 break;
Chris@17 700 }
Chris@17 701
Chris@17 702 if ($subTokenIsArray === true) {
Chris@17 703 $tokenContent .= $tokens[$i][1];
Chris@17 704 } else {
Chris@17 705 $tokenContent .= $tokens[$i];
Chris@17 706 }
Chris@17 707 }
Chris@17 708
Chris@17 709 if ($i === $numTokens) {
Chris@17 710 // We got to the end of the file and never
Chris@17 711 // found the closing token, so this probably wasn't
Chris@17 712 // a heredoc.
Chris@17 713 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 714 $type = $finalTokens[$newStackPtr]['type'];
Chris@17 715 echo "\t\t* failed to find the end of the here/nowdoc".PHP_EOL;
Chris@17 716 echo "\t\t* token $stackPtr changed from $type to T_STRING".PHP_EOL;
Chris@17 717 }
Chris@17 718
Chris@17 719 $finalTokens[$newStackPtr]['code'] = T_STRING;
Chris@17 720 $finalTokens[$newStackPtr]['type'] = 'T_STRING';
Chris@17 721 $newStackPtr++;
Chris@17 722 continue;
Chris@17 723 }
Chris@17 724
Chris@17 725 $stackPtr = $i;
Chris@17 726 $newStackPtr++;
Chris@17 727
Chris@17 728 // Convert each line within the heredoc to a
Chris@17 729 // new token, so it conforms with other multiple line tokens.
Chris@17 730 $tokenLines = explode($this->eolChar, $tokenContent);
Chris@17 731 $numLines = count($tokenLines);
Chris@17 732 $newToken = [];
Chris@17 733
Chris@17 734 for ($j = 0; $j < $numLines; $j++) {
Chris@17 735 $newToken['content'] = $tokenLines[$j];
Chris@17 736 if ($j === ($numLines - 1)) {
Chris@17 737 if ($tokenLines[$j] === '') {
Chris@17 738 break;
Chris@17 739 }
Chris@17 740 } else {
Chris@17 741 $newToken['content'] .= $this->eolChar;
Chris@17 742 }
Chris@17 743
Chris@17 744 if ($nowdoc === true) {
Chris@17 745 $newToken['code'] = T_NOWDOC;
Chris@17 746 $newToken['type'] = 'T_NOWDOC';
Chris@17 747 } else {
Chris@17 748 $newToken['code'] = T_HEREDOC;
Chris@17 749 $newToken['type'] = 'T_HEREDOC';
Chris@17 750 }
Chris@17 751
Chris@17 752 $finalTokens[$newStackPtr] = $newToken;
Chris@17 753 $newStackPtr++;
Chris@17 754 }//end for
Chris@17 755
Chris@17 756 // Add the end heredoc token to the final array.
Chris@17 757 $finalTokens[$newStackPtr] = self::standardiseToken($tokens[$stackPtr]);
Chris@17 758
Chris@17 759 if ($nowdoc === true) {
Chris@17 760 $finalTokens[$newStackPtr]['code'] = T_END_NOWDOC;
Chris@17 761 $finalTokens[$newStackPtr]['type'] = 'T_END_NOWDOC';
Chris@17 762 }
Chris@17 763
Chris@17 764 $newStackPtr++;
Chris@17 765
Chris@17 766 // Continue, as we're done with this token.
Chris@17 767 continue;
Chris@17 768 }//end if
Chris@17 769
Chris@17 770 /*
Chris@17 771 Before PHP 7.0, the "yield from" was tokenized as
Chris@17 772 T_YIELD, T_WHITESPACE and T_STRING. So look for
Chris@17 773 and change this token in earlier versions.
Chris@17 774 */
Chris@17 775
Chris@17 776 if (PHP_VERSION_ID < 70000
Chris@17 777 && PHP_VERSION_ID >= 50500
Chris@17 778 && $tokenIsArray === true
Chris@17 779 && $token[0] === T_YIELD
Chris@17 780 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 781 && isset($tokens[($stackPtr + 2)]) === true
Chris@17 782 && $tokens[($stackPtr + 1)][0] === T_WHITESPACE
Chris@17 783 && $tokens[($stackPtr + 2)][0] === T_STRING
Chris@17 784 && strtolower($tokens[($stackPtr + 2)][1]) === 'from'
Chris@17 785 ) {
Chris@17 786 // Could be multi-line, so just the token stack.
Chris@18 787 $token[0] = T_YIELD_FROM;
Chris@18 788 $token[1] .= $tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1];
Chris@17 789
Chris@17 790 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 791 for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) {
Chris@17 792 $type = Util\Tokens::tokenName($tokens[$i][0]);
Chris@17 793 $content = Util\Common::prepareForOutput($tokens[$i][1]);
Chris@17 794 echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content".PHP_EOL;
Chris@17 795 }
Chris@17 796 }
Chris@17 797
Chris@17 798 $tokens[($stackPtr + 1)] = null;
Chris@17 799 $tokens[($stackPtr + 2)] = null;
Chris@17 800 }
Chris@17 801
Chris@17 802 /*
Chris@17 803 Before PHP 5.5, the yield keyword was tokenized as
Chris@17 804 T_STRING. So look for and change this token in
Chris@17 805 earlier versions.
Chris@17 806 Checks also if it is just "yield" or "yield from".
Chris@17 807 */
Chris@17 808
Chris@17 809 if (PHP_VERSION_ID < 50500
Chris@17 810 && $tokenIsArray === true
Chris@17 811 && $token[0] === T_STRING
Chris@17 812 && strtolower($token[1]) === 'yield'
Chris@17 813 ) {
Chris@17 814 if (isset($tokens[($stackPtr + 1)]) === true
Chris@17 815 && isset($tokens[($stackPtr + 2)]) === true
Chris@17 816 && $tokens[($stackPtr + 1)][0] === T_WHITESPACE
Chris@17 817 && $tokens[($stackPtr + 2)][0] === T_STRING
Chris@17 818 && strtolower($tokens[($stackPtr + 2)][1]) === 'from'
Chris@17 819 ) {
Chris@17 820 // Could be multi-line, so just just the token stack.
Chris@18 821 $token[0] = T_YIELD_FROM;
Chris@18 822 $token[1] .= $tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1];
Chris@17 823
Chris@17 824 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 825 for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) {
Chris@17 826 $type = Util\Tokens::tokenName($tokens[$i][0]);
Chris@17 827 $content = Util\Common::prepareForOutput($tokens[$i][1]);
Chris@17 828 echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content".PHP_EOL;
Chris@17 829 }
Chris@17 830 }
Chris@17 831
Chris@17 832 $tokens[($stackPtr + 1)] = null;
Chris@17 833 $tokens[($stackPtr + 2)] = null;
Chris@17 834 } else {
Chris@17 835 $newToken = [];
Chris@17 836 $newToken['code'] = T_YIELD;
Chris@17 837 $newToken['type'] = 'T_YIELD';
Chris@17 838 $newToken['content'] = $token[1];
Chris@17 839 $finalTokens[$newStackPtr] = $newToken;
Chris@17 840
Chris@17 841 $newStackPtr++;
Chris@17 842 continue;
Chris@17 843 }//end if
Chris@17 844 }//end if
Chris@17 845
Chris@17 846 /*
Chris@17 847 Before PHP 5.6, the ... operator was tokenized as three
Chris@17 848 T_STRING_CONCAT tokens in a row. So look for and combine
Chris@17 849 these tokens in earlier versions.
Chris@17 850 */
Chris@17 851
Chris@17 852 if ($tokenIsArray === false
Chris@17 853 && $token[0] === '.'
Chris@17 854 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 855 && isset($tokens[($stackPtr + 2)]) === true
Chris@17 856 && $tokens[($stackPtr + 1)] === '.'
Chris@17 857 && $tokens[($stackPtr + 2)] === '.'
Chris@17 858 ) {
Chris@17 859 $newToken = [];
Chris@17 860 $newToken['code'] = T_ELLIPSIS;
Chris@17 861 $newToken['type'] = 'T_ELLIPSIS';
Chris@17 862 $newToken['content'] = '...';
Chris@17 863 $finalTokens[$newStackPtr] = $newToken;
Chris@17 864
Chris@17 865 $newStackPtr++;
Chris@17 866 $stackPtr += 2;
Chris@17 867 continue;
Chris@17 868 }
Chris@17 869
Chris@17 870 /*
Chris@17 871 Before PHP 5.6, the ** operator was tokenized as two
Chris@17 872 T_MULTIPLY tokens in a row. So look for and combine
Chris@17 873 these tokens in earlier versions.
Chris@17 874 */
Chris@17 875
Chris@17 876 if ($tokenIsArray === false
Chris@17 877 && $token[0] === '*'
Chris@17 878 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 879 && $tokens[($stackPtr + 1)] === '*'
Chris@17 880 ) {
Chris@17 881 $newToken = [];
Chris@17 882 $newToken['code'] = T_POW;
Chris@17 883 $newToken['type'] = 'T_POW';
Chris@17 884 $newToken['content'] = '**';
Chris@17 885 $finalTokens[$newStackPtr] = $newToken;
Chris@17 886
Chris@17 887 $newStackPtr++;
Chris@17 888 $stackPtr++;
Chris@17 889 continue;
Chris@17 890 }
Chris@17 891
Chris@17 892 /*
Chris@17 893 Before PHP 5.6, the **= operator was tokenized as
Chris@17 894 T_MULTIPLY followed by T_MUL_EQUAL. So look for and combine
Chris@17 895 these tokens in earlier versions.
Chris@17 896 */
Chris@17 897
Chris@17 898 if ($tokenIsArray === false
Chris@17 899 && $token[0] === '*'
Chris@17 900 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 901 && is_array($tokens[($stackPtr + 1)]) === true
Chris@17 902 && $tokens[($stackPtr + 1)][1] === '*='
Chris@17 903 ) {
Chris@17 904 $newToken = [];
Chris@17 905 $newToken['code'] = T_POW_EQUAL;
Chris@17 906 $newToken['type'] = 'T_POW_EQUAL';
Chris@17 907 $newToken['content'] = '**=';
Chris@17 908 $finalTokens[$newStackPtr] = $newToken;
Chris@17 909
Chris@17 910 $newStackPtr++;
Chris@17 911 $stackPtr++;
Chris@17 912 continue;
Chris@17 913 }
Chris@17 914
Chris@17 915 /*
Chris@17 916 Before PHP 7, the ??= operator was tokenized as
Chris@17 917 T_INLINE_THEN, T_INLINE_THEN, T_EQUAL.
Chris@17 918 Between PHP 7.0 and 7.2, the ??= operator was tokenized as
Chris@17 919 T_COALESCE, T_EQUAL.
Chris@17 920 So look for and combine these tokens in earlier versions.
Chris@17 921 */
Chris@17 922
Chris@17 923 if (($tokenIsArray === false
Chris@17 924 && $token[0] === '?'
Chris@17 925 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 926 && $tokens[($stackPtr + 1)][0] === '?'
Chris@17 927 && isset($tokens[($stackPtr + 2)]) === true
Chris@17 928 && $tokens[($stackPtr + 2)][0] === '=')
Chris@17 929 || ($tokenIsArray === true
Chris@17 930 && $token[0] === T_COALESCE
Chris@17 931 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 932 && $tokens[($stackPtr + 1)][0] === '=')
Chris@17 933 ) {
Chris@17 934 $newToken = [];
Chris@17 935 $newToken['code'] = T_COALESCE_EQUAL;
Chris@17 936 $newToken['type'] = 'T_COALESCE_EQUAL';
Chris@17 937 $newToken['content'] = '??=';
Chris@17 938 $finalTokens[$newStackPtr] = $newToken;
Chris@17 939
Chris@17 940 $newStackPtr++;
Chris@17 941 $stackPtr++;
Chris@17 942
Chris@17 943 if ($tokenIsArray === false) {
Chris@17 944 // Pre PHP 7.
Chris@17 945 $stackPtr++;
Chris@17 946 }
Chris@17 947
Chris@17 948 continue;
Chris@17 949 }
Chris@17 950
Chris@17 951 /*
Chris@17 952 Before PHP 7, the ?? operator was tokenized as
Chris@17 953 T_INLINE_THEN followed by T_INLINE_THEN.
Chris@17 954 So look for and combine these tokens in earlier versions.
Chris@17 955 */
Chris@17 956
Chris@17 957 if ($tokenIsArray === false
Chris@17 958 && $token[0] === '?'
Chris@17 959 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 960 && $tokens[($stackPtr + 1)][0] === '?'
Chris@17 961 ) {
Chris@17 962 $newToken = [];
Chris@17 963 $newToken['code'] = T_COALESCE;
Chris@17 964 $newToken['type'] = 'T_COALESCE';
Chris@17 965 $newToken['content'] = '??';
Chris@17 966 $finalTokens[$newStackPtr] = $newToken;
Chris@17 967
Chris@17 968 $newStackPtr++;
Chris@17 969 $stackPtr++;
Chris@17 970 continue;
Chris@17 971 }
Chris@17 972
Chris@17 973 /*
Chris@17 974 Convert ? to T_NULLABLE OR T_INLINE_THEN
Chris@17 975 */
Chris@17 976
Chris@17 977 if ($tokenIsArray === false && $token[0] === '?') {
Chris@17 978 $newToken = [];
Chris@17 979 $newToken['content'] = '?';
Chris@17 980
Chris@17 981 $prevNonEmpty = null;
Chris@17 982 for ($i = ($stackPtr - 1); $i >= 0; $i--) {
Chris@17 983 if (is_array($tokens[$i]) === true) {
Chris@17 984 $tokenType = $tokens[$i][0];
Chris@17 985 } else {
Chris@17 986 $tokenType = $tokens[$i];
Chris@17 987 }
Chris@17 988
Chris@17 989 if ($prevNonEmpty === null
Chris@17 990 && isset(Util\Tokens::$emptyTokens[$tokenType]) === false
Chris@17 991 ) {
Chris@17 992 // Found the previous non-empty token.
Chris@17 993 if ($tokenType === ':' || $tokenType === ',') {
Chris@17 994 $newToken['code'] = T_NULLABLE;
Chris@17 995 $newToken['type'] = 'T_NULLABLE';
Chris@17 996 break;
Chris@17 997 }
Chris@17 998
Chris@17 999 $prevNonEmpty = $tokenType;
Chris@17 1000 }
Chris@17 1001
Chris@17 1002 if ($tokenType === T_FUNCTION) {
Chris@17 1003 $newToken['code'] = T_NULLABLE;
Chris@17 1004 $newToken['type'] = 'T_NULLABLE';
Chris@17 1005 break;
Chris@18 1006 } else if (in_array($tokenType, [T_OPEN_TAG, T_OPEN_TAG_WITH_ECHO, '=', '{', ';'], true) === true) {
Chris@17 1007 $newToken['code'] = T_INLINE_THEN;
Chris@17 1008 $newToken['type'] = 'T_INLINE_THEN';
Chris@17 1009
Chris@17 1010 $insideInlineIf[] = $stackPtr;
Chris@17 1011 break;
Chris@17 1012 }
Chris@17 1013 }//end for
Chris@17 1014
Chris@17 1015 $finalTokens[$newStackPtr] = $newToken;
Chris@17 1016 $newStackPtr++;
Chris@17 1017 continue;
Chris@17 1018 }//end if
Chris@17 1019
Chris@17 1020 /*
Chris@17 1021 Tokens after a double colon may be look like scope openers,
Chris@17 1022 such as when writing code like Foo::NAMESPACE, but they are
Chris@17 1023 only ever variables or strings.
Chris@17 1024 */
Chris@17 1025
Chris@17 1026 if ($stackPtr > 1
Chris@17 1027 && (is_array($tokens[($stackPtr - 1)]) === true
Chris@17 1028 && $tokens[($stackPtr - 1)][0] === T_PAAMAYIM_NEKUDOTAYIM)
Chris@17 1029 && $tokenIsArray === true
Chris@17 1030 && $token[0] !== T_STRING
Chris@17 1031 && $token[0] !== T_VARIABLE
Chris@17 1032 && $token[0] !== T_DOLLAR
Chris@17 1033 && isset(Util\Tokens::$emptyTokens[$token[0]]) === false
Chris@17 1034 ) {
Chris@17 1035 $newToken = [];
Chris@17 1036 $newToken['code'] = T_STRING;
Chris@17 1037 $newToken['type'] = 'T_STRING';
Chris@17 1038 $newToken['content'] = $token[1];
Chris@17 1039 $finalTokens[$newStackPtr] = $newToken;
Chris@17 1040
Chris@17 1041 $newStackPtr++;
Chris@17 1042 continue;
Chris@17 1043 }
Chris@17 1044
Chris@17 1045 /*
Chris@17 1046 The string-like token after a function keyword should always be
Chris@17 1047 tokenized as T_STRING even if it appears to be a different token,
Chris@17 1048 such as when writing code like: function default(): foo
Chris@17 1049 so go forward and change the token type before it is processed.
Chris@17 1050 */
Chris@17 1051
Chris@17 1052 if ($tokenIsArray === true
Chris@17 1053 && $token[0] === T_FUNCTION
Chris@17 1054 && $finalTokens[$lastNotEmptyToken]['code'] !== T_USE
Chris@17 1055 ) {
Chris@17 1056 for ($x = ($stackPtr + 1); $x < $numTokens; $x++) {
Chris@17 1057 if (is_array($tokens[$x]) === false
Chris@17 1058 || isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false
Chris@17 1059 ) {
Chris@17 1060 // Non-empty content.
Chris@17 1061 break;
Chris@17 1062 }
Chris@17 1063 }
Chris@17 1064
Chris@17 1065 if ($x < $numTokens && is_array($tokens[$x]) === true) {
Chris@17 1066 $tokens[$x][0] = T_STRING;
Chris@17 1067 }
Chris@17 1068
Chris@17 1069 /*
Chris@17 1070 This is a special condition for T_ARRAY tokens used for
Chris@17 1071 function return types. We want to keep the parenthesis map clean,
Chris@17 1072 so let's tag these tokens as T_STRING.
Chris@17 1073 */
Chris@17 1074
Chris@17 1075 // Go looking for the colon to start the return type hint.
Chris@17 1076 // Start by finding the closing parenthesis of the function.
Chris@17 1077 $parenthesisStack = [];
Chris@17 1078 $parenthesisCloser = false;
Chris@17 1079 for ($x = ($stackPtr + 1); $x < $numTokens; $x++) {
Chris@17 1080 if (is_array($tokens[$x]) === false && $tokens[$x] === '(') {
Chris@17 1081 $parenthesisStack[] = $x;
Chris@17 1082 } else if (is_array($tokens[$x]) === false && $tokens[$x] === ')') {
Chris@17 1083 array_pop($parenthesisStack);
Chris@17 1084 if (empty($parenthesisStack) === true) {
Chris@17 1085 $parenthesisCloser = $x;
Chris@17 1086 break;
Chris@17 1087 }
Chris@17 1088 }
Chris@17 1089 }
Chris@17 1090
Chris@17 1091 if ($parenthesisCloser !== false) {
Chris@17 1092 for ($x = ($parenthesisCloser + 1); $x < $numTokens; $x++) {
Chris@17 1093 if (is_array($tokens[$x]) === false
Chris@17 1094 || isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false
Chris@17 1095 ) {
Chris@17 1096 // Non-empty content.
Chris@17 1097 if (is_array($tokens[$x]) === true && $tokens[$x][0] === T_USE) {
Chris@17 1098 // Found a use statements, so search ahead for the closing parenthesis.
Chris@17 1099 for ($x += 1; $x < $numTokens; $x++) {
Chris@17 1100 if (is_array($tokens[$x]) === false && $tokens[$x] === ')') {
Chris@17 1101 continue(2);
Chris@17 1102 }
Chris@17 1103 }
Chris@17 1104 }
Chris@17 1105
Chris@17 1106 break;
Chris@17 1107 }
Chris@17 1108 }
Chris@17 1109
Chris@17 1110 if (isset($tokens[$x]) === true
Chris@17 1111 && is_array($tokens[$x]) === false
Chris@17 1112 && $tokens[$x] === ':'
Chris@17 1113 ) {
Chris@17 1114 $allowed = [
Chris@17 1115 T_STRING => T_STRING,
Chris@17 1116 T_ARRAY => T_ARRAY,
Chris@17 1117 T_CALLABLE => T_CALLABLE,
Chris@17 1118 T_SELF => T_SELF,
Chris@17 1119 T_PARENT => T_PARENT,
Chris@17 1120 T_NS_SEPARATOR => T_NS_SEPARATOR,
Chris@17 1121 ];
Chris@17 1122
Chris@17 1123 $allowed += Util\Tokens::$emptyTokens;
Chris@17 1124
Chris@17 1125 // Find the start of the return type.
Chris@17 1126 for ($x += 1; $x < $numTokens; $x++) {
Chris@17 1127 if (is_array($tokens[$x]) === true
Chris@17 1128 && isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true
Chris@17 1129 ) {
Chris@18 1130 // Whitespace or comments before the return type.
Chris@17 1131 continue;
Chris@17 1132 }
Chris@17 1133
Chris@17 1134 if (is_array($tokens[$x]) === false && $tokens[$x] === '?') {
Chris@17 1135 // Found a nullable operator, so skip it.
Chris@17 1136 // But also covert the token to save the tokenizer
Chris@17 1137 // a bit of time later on.
Chris@17 1138 $tokens[$x] = [
Chris@17 1139 T_NULLABLE,
Chris@17 1140 '?',
Chris@17 1141 ];
Chris@17 1142
Chris@17 1143 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1144 echo "\t\t* token $x changed from ? to T_NULLABLE".PHP_EOL;
Chris@17 1145 }
Chris@17 1146
Chris@17 1147 continue;
Chris@17 1148 }
Chris@17 1149
Chris@17 1150 break;
Chris@17 1151 }//end for
Chris@17 1152
Chris@17 1153 // Any T_ARRAY tokens we find between here and the next
Chris@17 1154 // token that can't be part of the return type need to be
Chris@17 1155 // converted to T_STRING tokens.
Chris@17 1156 for ($x; $x < $numTokens; $x++) {
Chris@17 1157 if (is_array($tokens[$x]) === false || isset($allowed[$tokens[$x][0]]) === false) {
Chris@17 1158 break;
Chris@17 1159 } else if ($tokens[$x][0] === T_ARRAY) {
Chris@17 1160 $tokens[$x][0] = T_STRING;
Chris@17 1161
Chris@17 1162 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1163 echo "\t\t* token $x changed from T_ARRAY to T_STRING".PHP_EOL;
Chris@17 1164 }
Chris@17 1165 }
Chris@17 1166 }
Chris@17 1167 }//end if
Chris@17 1168 }//end if
Chris@17 1169 }//end if
Chris@17 1170
Chris@17 1171 /*
Chris@17 1172 Before PHP 7, the <=> operator was tokenized as
Chris@17 1173 T_IS_SMALLER_OR_EQUAL followed by T_GREATER_THAN.
Chris@17 1174 So look for and combine these tokens in earlier versions.
Chris@17 1175 */
Chris@17 1176
Chris@17 1177 if ($tokenIsArray === true
Chris@17 1178 && $token[0] === T_IS_SMALLER_OR_EQUAL
Chris@17 1179 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 1180 && $tokens[($stackPtr + 1)][0] === '>'
Chris@17 1181 ) {
Chris@17 1182 $newToken = [];
Chris@17 1183 $newToken['code'] = T_SPACESHIP;
Chris@17 1184 $newToken['type'] = 'T_SPACESHIP';
Chris@17 1185 $newToken['content'] = '<=>';
Chris@17 1186 $finalTokens[$newStackPtr] = $newToken;
Chris@17 1187
Chris@17 1188 $newStackPtr++;
Chris@17 1189 $stackPtr++;
Chris@17 1190 continue;
Chris@17 1191 }
Chris@17 1192
Chris@17 1193 /*
Chris@17 1194 PHP doesn't assign a token to goto labels, so we have to.
Chris@17 1195 These are just string tokens with a single colon after them. Double
Chris@17 1196 colons are already tokenized and so don't interfere with this check.
Chris@17 1197 But we do have to account for CASE statements, that look just like
Chris@17 1198 goto labels.
Chris@17 1199 */
Chris@17 1200
Chris@17 1201 if ($tokenIsArray === true
Chris@17 1202 && $token[0] === T_STRING
Chris@17 1203 && isset($tokens[($stackPtr + 1)]) === true
Chris@17 1204 && $tokens[($stackPtr + 1)] === ':'
Chris@17 1205 && $tokens[($stackPtr - 1)][0] !== T_PAAMAYIM_NEKUDOTAYIM
Chris@17 1206 ) {
Chris@17 1207 $stopTokens = [
Chris@17 1208 T_CASE => true,
Chris@17 1209 T_SEMICOLON => true,
Chris@17 1210 T_OPEN_CURLY_BRACKET => true,
Chris@17 1211 T_INLINE_THEN => true,
Chris@17 1212 ];
Chris@17 1213
Chris@17 1214 for ($x = ($newStackPtr - 1); $x > 0; $x--) {
Chris@17 1215 if (isset($stopTokens[$finalTokens[$x]['code']]) === true) {
Chris@17 1216 break;
Chris@17 1217 }
Chris@17 1218 }
Chris@17 1219
Chris@17 1220 if ($finalTokens[$x]['code'] !== T_CASE
Chris@17 1221 && $finalTokens[$x]['code'] !== T_INLINE_THEN
Chris@17 1222 ) {
Chris@17 1223 $finalTokens[$newStackPtr] = [
Chris@17 1224 'content' => $token[1].':',
Chris@17 1225 'code' => T_GOTO_LABEL,
Chris@17 1226 'type' => 'T_GOTO_LABEL',
Chris@17 1227 ];
Chris@17 1228
Chris@17 1229 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1230 echo "\t\t* token $stackPtr changed from T_STRING to T_GOTO_LABEL".PHP_EOL;
Chris@17 1231 echo "\t\t* skipping T_COLON token ".($stackPtr + 1).PHP_EOL;
Chris@17 1232 }
Chris@17 1233
Chris@17 1234 $newStackPtr++;
Chris@17 1235 $stackPtr++;
Chris@17 1236 continue;
Chris@17 1237 }
Chris@17 1238 }//end if
Chris@17 1239
Chris@17 1240 /*
Chris@17 1241 If this token has newlines in its content, split each line up
Chris@17 1242 and create a new token for each line. We do this so it's easier
Chris@17 1243 to ascertain where errors occur on a line.
Chris@17 1244 Note that $token[1] is the token's content.
Chris@17 1245 */
Chris@17 1246
Chris@17 1247 if ($tokenIsArray === true && strpos($token[1], $this->eolChar) !== false) {
Chris@17 1248 $tokenLines = explode($this->eolChar, $token[1]);
Chris@17 1249 $numLines = count($tokenLines);
Chris@17 1250 $newToken = [
Chris@17 1251 'type' => Util\Tokens::tokenName($token[0]),
Chris@17 1252 'code' => $token[0],
Chris@17 1253 'content' => '',
Chris@17 1254 ];
Chris@17 1255
Chris@17 1256 for ($i = 0; $i < $numLines; $i++) {
Chris@17 1257 $newToken['content'] = $tokenLines[$i];
Chris@17 1258 if ($i === ($numLines - 1)) {
Chris@17 1259 if ($tokenLines[$i] === '') {
Chris@17 1260 break;
Chris@17 1261 }
Chris@17 1262 } else {
Chris@17 1263 $newToken['content'] .= $this->eolChar;
Chris@17 1264 }
Chris@17 1265
Chris@17 1266 $finalTokens[$newStackPtr] = $newToken;
Chris@17 1267 $newStackPtr++;
Chris@17 1268 }
Chris@17 1269 } else {
Chris@17 1270 if ($tokenIsArray === true && $token[0] === T_STRING) {
Chris@17 1271 // Some T_STRING tokens should remain that way
Chris@17 1272 // due to their context.
Chris@17 1273 $context = [
Chris@17 1274 T_OBJECT_OPERATOR => true,
Chris@17 1275 T_FUNCTION => true,
Chris@17 1276 T_CLASS => true,
Chris@17 1277 T_EXTENDS => true,
Chris@17 1278 T_IMPLEMENTS => true,
Chris@17 1279 T_NEW => true,
Chris@17 1280 T_CONST => true,
Chris@17 1281 T_NS_SEPARATOR => true,
Chris@17 1282 T_USE => true,
Chris@17 1283 T_NAMESPACE => true,
Chris@17 1284 T_PAAMAYIM_NEKUDOTAYIM => true,
Chris@17 1285 ];
Chris@18 1286
Chris@17 1287 if (isset($context[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
Chris@17 1288 // Special case for syntax like: return new self
Chris@17 1289 // where self should not be a string.
Chris@17 1290 if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW
Chris@17 1291 && strtolower($token[1]) === 'self'
Chris@17 1292 ) {
Chris@17 1293 $finalTokens[$newStackPtr] = [
Chris@17 1294 'content' => $token[1],
Chris@17 1295 'code' => T_SELF,
Chris@17 1296 'type' => 'T_SELF',
Chris@17 1297 ];
Chris@17 1298 } else {
Chris@17 1299 $finalTokens[$newStackPtr] = [
Chris@17 1300 'content' => $token[1],
Chris@17 1301 'code' => T_STRING,
Chris@17 1302 'type' => 'T_STRING',
Chris@17 1303 ];
Chris@17 1304 }
Chris@17 1305
Chris@17 1306 $newStackPtr++;
Chris@17 1307 continue;
Chris@17 1308 }//end if
Chris@17 1309 }//end if
Chris@17 1310
Chris@17 1311 $newToken = null;
Chris@17 1312 if ($tokenIsArray === false) {
Chris@17 1313 if (isset(self::$resolveTokenCache[$token[0]]) === true) {
Chris@17 1314 $newToken = self::$resolveTokenCache[$token[0]];
Chris@17 1315 }
Chris@17 1316 } else {
Chris@17 1317 $cacheKey = null;
Chris@17 1318 if ($token[0] === T_STRING) {
Chris@17 1319 $cacheKey = strtolower($token[1]);
Chris@17 1320 } else if ($token[0] !== T_CURLY_OPEN) {
Chris@17 1321 $cacheKey = $token[0];
Chris@17 1322 }
Chris@17 1323
Chris@17 1324 if ($cacheKey !== null && isset(self::$resolveTokenCache[$cacheKey]) === true) {
Chris@17 1325 $newToken = self::$resolveTokenCache[$cacheKey];
Chris@17 1326 $newToken['content'] = $token[1];
Chris@17 1327 }
Chris@17 1328 }
Chris@17 1329
Chris@17 1330 if ($newToken === null) {
Chris@17 1331 $newToken = self::standardiseToken($token);
Chris@17 1332 }
Chris@17 1333
Chris@17 1334 // Convert colons that are actually the ELSE component of an
Chris@17 1335 // inline IF statement.
Chris@17 1336 if (empty($insideInlineIf) === false && $newToken['code'] === T_COLON) {
Chris@17 1337 // Make sure this isn't the return type separator of a closure.
Chris@17 1338 $isReturnType = false;
Chris@17 1339 for ($i = ($stackPtr - 1); $i > 0; $i--) {
Chris@17 1340 if (is_array($tokens[$i]) === false
Chris@17 1341 || ($tokens[$i][0] !== T_DOC_COMMENT
Chris@17 1342 && $tokens[$i][0] !== T_COMMENT
Chris@17 1343 && $tokens[$i][0] !== T_WHITESPACE)
Chris@17 1344 ) {
Chris@17 1345 break;
Chris@17 1346 }
Chris@17 1347 }
Chris@17 1348
Chris@17 1349 if ($tokens[$i] === ')') {
Chris@17 1350 $parenCount = 1;
Chris@17 1351 for ($i--; $i > 0; $i--) {
Chris@17 1352 if ($tokens[$i] === '(') {
Chris@17 1353 $parenCount--;
Chris@17 1354 if ($parenCount === 0) {
Chris@17 1355 break;
Chris@17 1356 }
Chris@17 1357 } else if ($tokens[$i] === ')') {
Chris@17 1358 $parenCount++;
Chris@17 1359 }
Chris@17 1360 }
Chris@17 1361
Chris@17 1362 // We've found the open parenthesis, so if the previous
Chris@17 1363 // non-empty token is FUNCTION or USE, this is a closure.
Chris@17 1364 for ($i--; $i > 0; $i--) {
Chris@17 1365 if (is_array($tokens[$i]) === false
Chris@17 1366 || ($tokens[$i][0] !== T_DOC_COMMENT
Chris@17 1367 && $tokens[$i][0] !== T_COMMENT
Chris@17 1368 && $tokens[$i][0] !== T_WHITESPACE)
Chris@17 1369 ) {
Chris@17 1370 break;
Chris@17 1371 }
Chris@17 1372 }
Chris@17 1373
Chris@17 1374 if ($tokens[$i][0] === T_FUNCTION || $tokens[$i][0] === T_USE) {
Chris@17 1375 $isReturnType = true;
Chris@17 1376 }
Chris@17 1377 }//end if
Chris@17 1378
Chris@17 1379 if ($isReturnType === false) {
Chris@17 1380 array_pop($insideInlineIf);
Chris@17 1381 $newToken['code'] = T_INLINE_ELSE;
Chris@17 1382 $newToken['type'] = 'T_INLINE_ELSE';
Chris@17 1383 }
Chris@17 1384 }//end if
Chris@17 1385
Chris@17 1386 // This is a special condition for T_ARRAY tokens used for
Chris@17 1387 // type hinting function arguments as being arrays. We want to keep
Chris@17 1388 // the parenthesis map clean, so let's tag these tokens as
Chris@17 1389 // T_STRING.
Chris@17 1390 if ($newToken['code'] === T_ARRAY) {
Chris@17 1391 for ($i = $stackPtr; $i < $numTokens; $i++) {
Chris@17 1392 if ($tokens[$i] === '(') {
Chris@17 1393 break;
Chris@17 1394 } else if ($tokens[$i][0] === T_VARIABLE) {
Chris@17 1395 $newToken['code'] = T_STRING;
Chris@17 1396 $newToken['type'] = 'T_STRING';
Chris@17 1397 break;
Chris@17 1398 }
Chris@17 1399 }
Chris@17 1400 }
Chris@17 1401
Chris@17 1402 // This is a special case when checking PHP 5.5+ code in PHP < 5.5
Chris@17 1403 // where "finally" should be T_FINALLY instead of T_STRING.
Chris@17 1404 if ($newToken['code'] === T_STRING
Chris@17 1405 && strtolower($newToken['content']) === 'finally'
Chris@17 1406 ) {
Chris@17 1407 $newToken['code'] = T_FINALLY;
Chris@17 1408 $newToken['type'] = 'T_FINALLY';
Chris@17 1409 }
Chris@17 1410
Chris@17 1411 // This is a special case for the PHP 5.5 classname::class syntax
Chris@17 1412 // where "class" should be T_STRING instead of T_CLASS.
Chris@17 1413 if (($newToken['code'] === T_CLASS
Chris@17 1414 || $newToken['code'] === T_FUNCTION)
Chris@17 1415 && $finalTokens[$lastNotEmptyToken]['code'] === T_DOUBLE_COLON
Chris@17 1416 ) {
Chris@17 1417 $newToken['code'] = T_STRING;
Chris@17 1418 $newToken['type'] = 'T_STRING';
Chris@17 1419 }
Chris@17 1420
Chris@17 1421 // This is a special case for PHP 5.6 use function and use const
Chris@17 1422 // where "function" and "const" should be T_STRING instead of T_FUNCTION
Chris@17 1423 // and T_CONST.
Chris@17 1424 if (($newToken['code'] === T_FUNCTION
Chris@17 1425 || $newToken['code'] === T_CONST)
Chris@17 1426 && ($finalTokens[$lastNotEmptyToken]['code'] === T_USE || $insideUseGroup === true)
Chris@17 1427 ) {
Chris@17 1428 $newToken['code'] = T_STRING;
Chris@17 1429 $newToken['type'] = 'T_STRING';
Chris@17 1430 }
Chris@17 1431
Chris@17 1432 // This is a special case for use groups in PHP 7+ where leaving
Chris@17 1433 // the curly braces as their normal tokens would confuse
Chris@17 1434 // the scope map and sniffs.
Chris@17 1435 if ($newToken['code'] === T_OPEN_CURLY_BRACKET
Chris@17 1436 && $finalTokens[$lastNotEmptyToken]['code'] === T_NS_SEPARATOR
Chris@17 1437 ) {
Chris@17 1438 $newToken['code'] = T_OPEN_USE_GROUP;
Chris@17 1439 $newToken['type'] = 'T_OPEN_USE_GROUP';
Chris@17 1440 $insideUseGroup = true;
Chris@17 1441 }
Chris@17 1442
Chris@17 1443 if ($insideUseGroup === true && $newToken['code'] === T_CLOSE_CURLY_BRACKET) {
Chris@17 1444 $newToken['code'] = T_CLOSE_USE_GROUP;
Chris@17 1445 $newToken['type'] = 'T_CLOSE_USE_GROUP';
Chris@17 1446 $insideUseGroup = false;
Chris@17 1447 }
Chris@17 1448
Chris@17 1449 $finalTokens[$newStackPtr] = $newToken;
Chris@17 1450 $newStackPtr++;
Chris@17 1451 }//end if
Chris@17 1452 }//end for
Chris@17 1453
Chris@17 1454 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1455 echo "\t*** END PHP TOKENIZING ***".PHP_EOL;
Chris@17 1456 }
Chris@17 1457
Chris@17 1458 return $finalTokens;
Chris@17 1459
Chris@17 1460 }//end tokenize()
Chris@17 1461
Chris@17 1462
Chris@17 1463 /**
Chris@17 1464 * Performs additional processing after main tokenizing.
Chris@17 1465 *
Chris@17 1466 * This additional processing checks for CASE statements that are using curly
Chris@17 1467 * braces for scope openers and closers. It also turns some T_FUNCTION tokens
Chris@17 1468 * into T_CLOSURE when they are not standard function definitions. It also
Chris@17 1469 * detects short array syntax and converts those square brackets into new tokens.
Chris@17 1470 * It also corrects some usage of the static and class keywords. It also
Chris@17 1471 * assigns tokens to function return types.
Chris@17 1472 *
Chris@17 1473 * @return void
Chris@17 1474 */
Chris@17 1475 protected function processAdditional()
Chris@17 1476 {
Chris@17 1477 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1478 echo "\t*** START ADDITIONAL PHP PROCESSING ***".PHP_EOL;
Chris@17 1479 }
Chris@17 1480
Chris@17 1481 $numTokens = count($this->tokens);
Chris@17 1482 for ($i = ($numTokens - 1); $i >= 0; $i--) {
Chris@17 1483 // Check for any unset scope conditions due to alternate IF/ENDIF syntax.
Chris@17 1484 if (isset($this->tokens[$i]['scope_opener']) === true
Chris@17 1485 && isset($this->tokens[$i]['scope_condition']) === false
Chris@17 1486 ) {
Chris@17 1487 $this->tokens[$i]['scope_condition'] = $this->tokens[$this->tokens[$i]['scope_opener']]['scope_condition'];
Chris@17 1488 }
Chris@17 1489
Chris@17 1490 if ($this->tokens[$i]['code'] === T_FUNCTION) {
Chris@17 1491 /*
Chris@17 1492 Detect functions that are actually closures and
Chris@17 1493 assign them a different token.
Chris@17 1494 */
Chris@17 1495
Chris@17 1496 if (isset($this->tokens[$i]['scope_opener']) === true) {
Chris@17 1497 for ($x = ($i + 1); $x < $numTokens; $x++) {
Chris@17 1498 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false
Chris@17 1499 && $this->tokens[$x]['code'] !== T_BITWISE_AND
Chris@17 1500 ) {
Chris@17 1501 break;
Chris@17 1502 }
Chris@17 1503 }
Chris@17 1504
Chris@17 1505 if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) {
Chris@17 1506 $this->tokens[$i]['code'] = T_CLOSURE;
Chris@17 1507 $this->tokens[$i]['type'] = 'T_CLOSURE';
Chris@17 1508 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1509 $line = $this->tokens[$i]['line'];
Chris@17 1510 echo "\t* token $i on line $line changed from T_FUNCTION to T_CLOSURE".PHP_EOL;
Chris@17 1511 }
Chris@17 1512
Chris@17 1513 for ($x = ($this->tokens[$i]['scope_opener'] + 1); $x < $this->tokens[$i]['scope_closer']; $x++) {
Chris@17 1514 if (isset($this->tokens[$x]['conditions'][$i]) === false) {
Chris@17 1515 continue;
Chris@17 1516 }
Chris@17 1517
Chris@17 1518 $this->tokens[$x]['conditions'][$i] = T_CLOSURE;
Chris@17 1519 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1520 $type = $this->tokens[$x]['type'];
Chris@17 1521 echo "\t\t* cleaned $x ($type) *".PHP_EOL;
Chris@17 1522 }
Chris@17 1523 }
Chris@17 1524 }
Chris@17 1525 }//end if
Chris@17 1526
Chris@17 1527 continue;
Chris@17 1528 } else if ($this->tokens[$i]['code'] === T_CLASS && isset($this->tokens[$i]['scope_opener']) === true) {
Chris@17 1529 /*
Chris@17 1530 Detect anonymous classes and assign them a different token.
Chris@17 1531 */
Chris@17 1532
Chris@17 1533 for ($x = ($i + 1); $x < $numTokens; $x++) {
Chris@17 1534 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
Chris@17 1535 break;
Chris@17 1536 }
Chris@17 1537 }
Chris@17 1538
Chris@17 1539 if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS
Chris@17 1540 || $this->tokens[$x]['code'] === T_OPEN_CURLY_BRACKET
Chris@17 1541 || $this->tokens[$x]['code'] === T_EXTENDS
Chris@17 1542 || $this->tokens[$x]['code'] === T_IMPLEMENTS
Chris@17 1543 ) {
Chris@17 1544 $this->tokens[$i]['code'] = T_ANON_CLASS;
Chris@17 1545 $this->tokens[$i]['type'] = 'T_ANON_CLASS';
Chris@17 1546 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1547 $line = $this->tokens[$i]['line'];
Chris@17 1548 echo "\t* token $i on line $line changed from T_CLASS to T_ANON_CLASS".PHP_EOL;
Chris@17 1549 }
Chris@17 1550
Chris@17 1551 for ($x = ($this->tokens[$i]['scope_opener'] + 1); $x < $this->tokens[$i]['scope_closer']; $x++) {
Chris@17 1552 if (isset($this->tokens[$x]['conditions'][$i]) === false) {
Chris@17 1553 continue;
Chris@17 1554 }
Chris@17 1555
Chris@17 1556 $this->tokens[$x]['conditions'][$i] = T_ANON_CLASS;
Chris@17 1557 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1558 $type = $this->tokens[$x]['type'];
Chris@17 1559 echo "\t\t* cleaned $x ($type) *".PHP_EOL;
Chris@17 1560 }
Chris@17 1561 }
Chris@17 1562 }
Chris@17 1563
Chris@17 1564 continue;
Chris@17 1565 } else if ($this->tokens[$i]['code'] === T_OPEN_SQUARE_BRACKET) {
Chris@17 1566 if (isset($this->tokens[$i]['bracket_closer']) === false) {
Chris@17 1567 continue;
Chris@17 1568 }
Chris@17 1569
Chris@17 1570 // Unless there is a variable or a bracket before this token,
Chris@17 1571 // it is the start of an array being defined using the short syntax.
Chris@17 1572 $isShortArray = false;
Chris@17 1573 $allowed = [
Chris@17 1574 T_CLOSE_SQUARE_BRACKET => T_CLOSE_SQUARE_BRACKET,
Chris@17 1575 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
Chris@17 1576 T_CLOSE_PARENTHESIS => T_CLOSE_PARENTHESIS,
Chris@17 1577 T_VARIABLE => T_VARIABLE,
Chris@17 1578 T_OBJECT_OPERATOR => T_OBJECT_OPERATOR,
Chris@17 1579 T_STRING => T_STRING,
Chris@17 1580 T_CONSTANT_ENCAPSED_STRING => T_CONSTANT_ENCAPSED_STRING,
Chris@17 1581 ];
Chris@17 1582
Chris@17 1583 for ($x = ($i - 1); $x >= 0; $x--) {
Chris@17 1584 // If we hit a scope opener, the statement has ended
Chris@17 1585 // without finding anything, so it's probably an array
Chris@17 1586 // using PHP 7.1 short list syntax.
Chris@17 1587 if (isset($this->tokens[$x]['scope_opener']) === true) {
Chris@17 1588 $isShortArray = true;
Chris@17 1589 break;
Chris@17 1590 }
Chris@17 1591
Chris@17 1592 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
Chris@17 1593 if (isset($allowed[$this->tokens[$x]['code']]) === false) {
Chris@17 1594 $isShortArray = true;
Chris@17 1595 }
Chris@17 1596
Chris@17 1597 break;
Chris@17 1598 }
Chris@17 1599 }
Chris@17 1600
Chris@17 1601 if ($isShortArray === true) {
Chris@17 1602 $this->tokens[$i]['code'] = T_OPEN_SHORT_ARRAY;
Chris@17 1603 $this->tokens[$i]['type'] = 'T_OPEN_SHORT_ARRAY';
Chris@17 1604
Chris@17 1605 $closer = $this->tokens[$i]['bracket_closer'];
Chris@17 1606 $this->tokens[$closer]['code'] = T_CLOSE_SHORT_ARRAY;
Chris@17 1607 $this->tokens[$closer]['type'] = 'T_CLOSE_SHORT_ARRAY';
Chris@17 1608 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1609 $line = $this->tokens[$i]['line'];
Chris@17 1610 echo "\t* token $i on line $line changed from T_OPEN_SQUARE_BRACKET to T_OPEN_SHORT_ARRAY".PHP_EOL;
Chris@17 1611 $line = $this->tokens[$closer]['line'];
Chris@17 1612 echo "\t* token $closer on line $line changed from T_CLOSE_SQUARE_BRACKET to T_CLOSE_SHORT_ARRAY".PHP_EOL;
Chris@17 1613 }
Chris@17 1614 }
Chris@17 1615
Chris@17 1616 continue;
Chris@17 1617 } else if ($this->tokens[$i]['code'] === T_STATIC) {
Chris@17 1618 for ($x = ($i - 1); $x > 0; $x--) {
Chris@17 1619 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
Chris@17 1620 break;
Chris@17 1621 }
Chris@17 1622 }
Chris@17 1623
Chris@17 1624 if ($this->tokens[$x]['code'] === T_INSTANCEOF) {
Chris@17 1625 $this->tokens[$i]['code'] = T_STRING;
Chris@17 1626 $this->tokens[$i]['type'] = 'T_STRING';
Chris@17 1627
Chris@17 1628 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1629 $line = $this->tokens[$i]['line'];
Chris@17 1630 echo "\t* token $i on line $line changed from T_STATIC to T_STRING".PHP_EOL;
Chris@17 1631 }
Chris@17 1632 }
Chris@17 1633
Chris@17 1634 continue;
Chris@17 1635 } else if ($this->tokens[$i]['code'] === T_TRUE
Chris@17 1636 || $this->tokens[$i]['code'] === T_FALSE
Chris@17 1637 || $this->tokens[$i]['code'] === T_NULL
Chris@17 1638 ) {
Chris@17 1639 for ($x = ($i + 1); $i < $numTokens; $x++) {
Chris@17 1640 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
Chris@17 1641 // Non-whitespace content.
Chris@17 1642 break;
Chris@17 1643 }
Chris@17 1644 }
Chris@17 1645
Chris@17 1646 $context = [
Chris@17 1647 T_OBJECT_OPERATOR => true,
Chris@17 1648 T_NS_SEPARATOR => true,
Chris@17 1649 T_PAAMAYIM_NEKUDOTAYIM => true,
Chris@17 1650 ];
Chris@17 1651 if (isset($context[$this->tokens[$x]['code']]) === true) {
Chris@17 1652 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1653 $line = $this->tokens[$i]['line'];
Chris@17 1654 $type = $this->tokens[$i]['type'];
Chris@17 1655 echo "\t* token $i on line $line changed from $type to T_STRING".PHP_EOL;
Chris@17 1656 }
Chris@17 1657
Chris@17 1658 $this->tokens[$i]['code'] = T_STRING;
Chris@17 1659 $this->tokens[$i]['type'] = 'T_STRING';
Chris@17 1660 }
Chris@17 1661 } else if ($this->tokens[$i]['code'] === T_CONST) {
Chris@17 1662 // Context sensitive keywords support.
Chris@17 1663 for ($x = ($i + 1); $i < $numTokens; $x++) {
Chris@17 1664 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
Chris@17 1665 // Non-whitespace content.
Chris@17 1666 break;
Chris@17 1667 }
Chris@17 1668 }
Chris@17 1669
Chris@17 1670 if ($this->tokens[$x]['code'] !== T_STRING) {
Chris@17 1671 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1672 $line = $this->tokens[$x]['line'];
Chris@17 1673 $type = $this->tokens[$x]['type'];
Chris@17 1674 echo "\t* token $x on line $line changed from $type to T_STRING".PHP_EOL;
Chris@17 1675 }
Chris@17 1676
Chris@17 1677 $this->tokens[$x]['code'] = T_STRING;
Chris@17 1678 $this->tokens[$x]['type'] = 'T_STRING';
Chris@17 1679 }
Chris@17 1680 }//end if
Chris@17 1681
Chris@17 1682 if (($this->tokens[$i]['code'] !== T_CASE
Chris@17 1683 && $this->tokens[$i]['code'] !== T_DEFAULT)
Chris@17 1684 || isset($this->tokens[$i]['scope_opener']) === false
Chris@17 1685 ) {
Chris@17 1686 // Only interested in CASE and DEFAULT statements from here on in.
Chris@17 1687 continue;
Chris@17 1688 }
Chris@17 1689
Chris@17 1690 $scopeOpener = $this->tokens[$i]['scope_opener'];
Chris@17 1691 $scopeCloser = $this->tokens[$i]['scope_closer'];
Chris@17 1692
Chris@17 1693 // If the first char after the opener is a curly brace
Chris@17 1694 // and that brace has been ignored, it is actually
Chris@17 1695 // opening this case statement and the opener and closer are
Chris@17 1696 // probably set incorrectly.
Chris@17 1697 for ($x = ($scopeOpener + 1); $x < $numTokens; $x++) {
Chris@17 1698 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
Chris@17 1699 // Non-whitespace content.
Chris@17 1700 break;
Chris@17 1701 }
Chris@17 1702 }
Chris@17 1703
Chris@17 1704 if ($this->tokens[$x]['code'] === T_CASE || $this->tokens[$x]['code'] === T_DEFAULT) {
Chris@17 1705 // Special case for multiple CASE statements that share the same
Chris@17 1706 // closer. Because we are going backwards through the file, this next
Chris@17 1707 // CASE statement is already fixed, so just use its closer and don't
Chris@17 1708 // worry about fixing anything.
Chris@17 1709 $newCloser = $this->tokens[$x]['scope_closer'];
Chris@17 1710 $this->tokens[$i]['scope_closer'] = $newCloser;
Chris@17 1711 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1712 $oldType = $this->tokens[$scopeCloser]['type'];
Chris@17 1713 $newType = $this->tokens[$newCloser]['type'];
Chris@17 1714 $line = $this->tokens[$i]['line'];
Chris@17 1715 echo "\t* token $i (T_CASE) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)".PHP_EOL;
Chris@17 1716 }
Chris@17 1717
Chris@17 1718 continue;
Chris@17 1719 }
Chris@17 1720
Chris@17 1721 if ($this->tokens[$x]['code'] !== T_OPEN_CURLY_BRACKET
Chris@17 1722 || isset($this->tokens[$x]['scope_condition']) === true
Chris@17 1723 ) {
Chris@17 1724 // Not a CASE/DEFAULT with a curly brace opener.
Chris@17 1725 continue;
Chris@17 1726 }
Chris@17 1727
Chris@17 1728 // The closer for this CASE/DEFAULT should be the closing curly brace and
Chris@17 1729 // not whatever it already is. The opener needs to be the opening curly
Chris@17 1730 // brace so everything matches up.
Chris@17 1731 $newCloser = $this->tokens[$x]['bracket_closer'];
Chris@17 1732 foreach ([$i, $x, $newCloser] as $index) {
Chris@17 1733 $this->tokens[$index]['scope_condition'] = $i;
Chris@17 1734 $this->tokens[$index]['scope_opener'] = $x;
Chris@17 1735 $this->tokens[$index]['scope_closer'] = $newCloser;
Chris@17 1736 }
Chris@17 1737
Chris@17 1738 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1739 $line = $this->tokens[$i]['line'];
Chris@17 1740 $tokenType = $this->tokens[$i]['type'];
Chris@17 1741
Chris@17 1742 $oldType = $this->tokens[$scopeOpener]['type'];
Chris@17 1743 $newType = $this->tokens[$x]['type'];
Chris@17 1744 echo "\t* token $i ($tokenType) on line $line opener changed from $scopeOpener ($oldType) to $x ($newType)".PHP_EOL;
Chris@17 1745
Chris@17 1746 $oldType = $this->tokens[$scopeCloser]['type'];
Chris@17 1747 $newType = $this->tokens[$newCloser]['type'];
Chris@17 1748 echo "\t* token $i ($tokenType) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)".PHP_EOL;
Chris@17 1749 }
Chris@17 1750
Chris@17 1751 if ($this->tokens[$scopeOpener]['scope_condition'] === $i) {
Chris@17 1752 unset($this->tokens[$scopeOpener]['scope_condition']);
Chris@17 1753 unset($this->tokens[$scopeOpener]['scope_opener']);
Chris@17 1754 unset($this->tokens[$scopeOpener]['scope_closer']);
Chris@17 1755 }
Chris@17 1756
Chris@17 1757 if ($this->tokens[$scopeCloser]['scope_condition'] === $i) {
Chris@17 1758 unset($this->tokens[$scopeCloser]['scope_condition']);
Chris@17 1759 unset($this->tokens[$scopeCloser]['scope_opener']);
Chris@17 1760 unset($this->tokens[$scopeCloser]['scope_closer']);
Chris@17 1761 } else {
Chris@17 1762 // We were using a shared closer. All tokens that were
Chris@17 1763 // sharing this closer with us, except for the scope condition
Chris@17 1764 // and it's opener, need to now point to the new closer.
Chris@17 1765 $condition = $this->tokens[$scopeCloser]['scope_condition'];
Chris@17 1766 $start = ($this->tokens[$condition]['scope_opener'] + 1);
Chris@17 1767 for ($y = $start; $y < $scopeCloser; $y++) {
Chris@17 1768 if (isset($this->tokens[$y]['scope_closer']) === true
Chris@17 1769 && $this->tokens[$y]['scope_closer'] === $scopeCloser
Chris@17 1770 ) {
Chris@17 1771 $this->tokens[$y]['scope_closer'] = $newCloser;
Chris@17 1772
Chris@17 1773 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1774 $line = $this->tokens[$y]['line'];
Chris@17 1775 $tokenType = $this->tokens[$y]['type'];
Chris@17 1776 $oldType = $this->tokens[$scopeCloser]['type'];
Chris@17 1777 $newType = $this->tokens[$newCloser]['type'];
Chris@17 1778 echo "\t\t* token $y ($tokenType) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)".PHP_EOL;
Chris@17 1779 }
Chris@17 1780 }
Chris@17 1781 }
Chris@17 1782 }//end if
Chris@17 1783
Chris@17 1784 unset($this->tokens[$x]['bracket_opener']);
Chris@17 1785 unset($this->tokens[$x]['bracket_closer']);
Chris@17 1786 unset($this->tokens[$newCloser]['bracket_opener']);
Chris@17 1787 unset($this->tokens[$newCloser]['bracket_closer']);
Chris@17 1788 $this->tokens[$scopeCloser]['conditions'][] = $i;
Chris@17 1789
Chris@17 1790 // Now fix up all the tokens that think they are
Chris@17 1791 // inside the CASE/DEFAULT statement when they are really outside.
Chris@17 1792 for ($x = $newCloser; $x < $scopeCloser; $x++) {
Chris@17 1793 foreach ($this->tokens[$x]['conditions'] as $num => $oldCond) {
Chris@17 1794 if ($oldCond === $this->tokens[$i]['code']) {
Chris@17 1795 $oldConditions = $this->tokens[$x]['conditions'];
Chris@17 1796 unset($this->tokens[$x]['conditions'][$num]);
Chris@17 1797
Chris@17 1798 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1799 $type = $this->tokens[$x]['type'];
Chris@17 1800 $oldConds = '';
Chris@17 1801 foreach ($oldConditions as $condition) {
Chris@17 1802 $oldConds .= Util\Tokens::tokenName($condition).',';
Chris@17 1803 }
Chris@17 1804
Chris@17 1805 $oldConds = rtrim($oldConds, ',');
Chris@17 1806
Chris@17 1807 $newConds = '';
Chris@17 1808 foreach ($this->tokens[$x]['conditions'] as $condition) {
Chris@17 1809 $newConds .= Util\Tokens::tokenName($condition).',';
Chris@17 1810 }
Chris@17 1811
Chris@17 1812 $newConds = rtrim($newConds, ',');
Chris@17 1813
Chris@17 1814 echo "\t\t* cleaned $x ($type) *".PHP_EOL;
Chris@17 1815 echo "\t\t\t=> conditions changed from $oldConds to $newConds".PHP_EOL;
Chris@17 1816 }
Chris@17 1817
Chris@17 1818 break;
Chris@17 1819 }//end if
Chris@17 1820 }//end foreach
Chris@17 1821 }//end for
Chris@17 1822 }//end for
Chris@17 1823
Chris@17 1824 if (PHP_CODESNIFFER_VERBOSITY > 1) {
Chris@17 1825 echo "\t*** END ADDITIONAL PHP PROCESSING ***".PHP_EOL;
Chris@17 1826 }
Chris@17 1827
Chris@17 1828 }//end processAdditional()
Chris@17 1829
Chris@17 1830
Chris@17 1831 /**
Chris@17 1832 * Takes a token produced from <code>token_get_all()</code> and produces a
Chris@17 1833 * more uniform token.
Chris@17 1834 *
Chris@17 1835 * @param string|array $token The token to convert.
Chris@17 1836 *
Chris@17 1837 * @return array The new token.
Chris@17 1838 */
Chris@17 1839 public static function standardiseToken($token)
Chris@17 1840 {
Chris@17 1841 if (isset($token[1]) === false) {
Chris@17 1842 if (isset(self::$resolveTokenCache[$token[0]]) === true) {
Chris@17 1843 return self::$resolveTokenCache[$token[0]];
Chris@17 1844 }
Chris@17 1845 } else {
Chris@17 1846 $cacheKey = null;
Chris@17 1847 if ($token[0] === T_STRING) {
Chris@17 1848 $cacheKey = strtolower($token[1]);
Chris@17 1849 } else if ($token[0] !== T_CURLY_OPEN) {
Chris@17 1850 $cacheKey = $token[0];
Chris@17 1851 }
Chris@17 1852
Chris@17 1853 if ($cacheKey !== null && isset(self::$resolveTokenCache[$cacheKey]) === true) {
Chris@17 1854 $newToken = self::$resolveTokenCache[$cacheKey];
Chris@17 1855 $newToken['content'] = $token[1];
Chris@17 1856 return $newToken;
Chris@17 1857 }
Chris@17 1858 }
Chris@17 1859
Chris@17 1860 if (isset($token[1]) === false) {
Chris@17 1861 return self::resolveSimpleToken($token[0]);
Chris@17 1862 }
Chris@17 1863
Chris@17 1864 if ($token[0] === T_STRING) {
Chris@17 1865 switch ($cacheKey) {
Chris@17 1866 case 'false':
Chris@17 1867 $newToken['type'] = 'T_FALSE';
Chris@17 1868 break;
Chris@17 1869 case 'true':
Chris@17 1870 $newToken['type'] = 'T_TRUE';
Chris@17 1871 break;
Chris@17 1872 case 'null':
Chris@17 1873 $newToken['type'] = 'T_NULL';
Chris@17 1874 break;
Chris@17 1875 case 'self':
Chris@17 1876 $newToken['type'] = 'T_SELF';
Chris@17 1877 break;
Chris@17 1878 case 'parent':
Chris@17 1879 $newToken['type'] = 'T_PARENT';
Chris@17 1880 break;
Chris@17 1881 default:
Chris@17 1882 $newToken['type'] = 'T_STRING';
Chris@17 1883 break;
Chris@17 1884 }
Chris@17 1885
Chris@17 1886 $newToken['code'] = constant($newToken['type']);
Chris@17 1887
Chris@17 1888 self::$resolveTokenCache[$cacheKey] = $newToken;
Chris@17 1889 } else if ($token[0] === T_CURLY_OPEN) {
Chris@17 1890 $newToken = [
Chris@17 1891 'code' => T_OPEN_CURLY_BRACKET,
Chris@17 1892 'type' => 'T_OPEN_CURLY_BRACKET',
Chris@17 1893 ];
Chris@17 1894 } else {
Chris@17 1895 $newToken = [
Chris@17 1896 'code' => $token[0],
Chris@17 1897 'type' => Util\Tokens::tokenName($token[0]),
Chris@17 1898 ];
Chris@17 1899
Chris@17 1900 self::$resolveTokenCache[$token[0]] = $newToken;
Chris@17 1901 }//end if
Chris@17 1902
Chris@17 1903 $newToken['content'] = $token[1];
Chris@17 1904 return $newToken;
Chris@17 1905
Chris@17 1906 }//end standardiseToken()
Chris@17 1907
Chris@17 1908
Chris@17 1909 /**
Chris@17 1910 * Converts simple tokens into a format that conforms to complex tokens
Chris@17 1911 * produced by token_get_all().
Chris@17 1912 *
Chris@17 1913 * Simple tokens are tokens that are not in array form when produced from
Chris@17 1914 * token_get_all().
Chris@17 1915 *
Chris@17 1916 * @param string $token The simple token to convert.
Chris@17 1917 *
Chris@17 1918 * @return array The new token in array format.
Chris@17 1919 */
Chris@17 1920 public static function resolveSimpleToken($token)
Chris@17 1921 {
Chris@17 1922 $newToken = [];
Chris@17 1923
Chris@17 1924 switch ($token) {
Chris@17 1925 case '{':
Chris@17 1926 $newToken['type'] = 'T_OPEN_CURLY_BRACKET';
Chris@17 1927 break;
Chris@17 1928 case '}':
Chris@17 1929 $newToken['type'] = 'T_CLOSE_CURLY_BRACKET';
Chris@17 1930 break;
Chris@17 1931 case '[':
Chris@17 1932 $newToken['type'] = 'T_OPEN_SQUARE_BRACKET';
Chris@17 1933 break;
Chris@17 1934 case ']':
Chris@17 1935 $newToken['type'] = 'T_CLOSE_SQUARE_BRACKET';
Chris@17 1936 break;
Chris@17 1937 case '(':
Chris@17 1938 $newToken['type'] = 'T_OPEN_PARENTHESIS';
Chris@17 1939 break;
Chris@17 1940 case ')':
Chris@17 1941 $newToken['type'] = 'T_CLOSE_PARENTHESIS';
Chris@17 1942 break;
Chris@17 1943 case ':':
Chris@17 1944 $newToken['type'] = 'T_COLON';
Chris@17 1945 break;
Chris@17 1946 case '.':
Chris@17 1947 $newToken['type'] = 'T_STRING_CONCAT';
Chris@17 1948 break;
Chris@17 1949 case ';':
Chris@17 1950 $newToken['type'] = 'T_SEMICOLON';
Chris@17 1951 break;
Chris@17 1952 case '=':
Chris@17 1953 $newToken['type'] = 'T_EQUAL';
Chris@17 1954 break;
Chris@17 1955 case '*':
Chris@17 1956 $newToken['type'] = 'T_MULTIPLY';
Chris@17 1957 break;
Chris@17 1958 case '/':
Chris@17 1959 $newToken['type'] = 'T_DIVIDE';
Chris@17 1960 break;
Chris@17 1961 case '+':
Chris@17 1962 $newToken['type'] = 'T_PLUS';
Chris@17 1963 break;
Chris@17 1964 case '-':
Chris@17 1965 $newToken['type'] = 'T_MINUS';
Chris@17 1966 break;
Chris@17 1967 case '%':
Chris@17 1968 $newToken['type'] = 'T_MODULUS';
Chris@17 1969 break;
Chris@17 1970 case '^':
Chris@17 1971 $newToken['type'] = 'T_BITWISE_XOR';
Chris@17 1972 break;
Chris@17 1973 case '&':
Chris@17 1974 $newToken['type'] = 'T_BITWISE_AND';
Chris@17 1975 break;
Chris@17 1976 case '|':
Chris@17 1977 $newToken['type'] = 'T_BITWISE_OR';
Chris@17 1978 break;
Chris@17 1979 case '~':
Chris@17 1980 $newToken['type'] = 'T_BITWISE_NOT';
Chris@17 1981 break;
Chris@17 1982 case '<':
Chris@17 1983 $newToken['type'] = 'T_LESS_THAN';
Chris@17 1984 break;
Chris@17 1985 case '>':
Chris@17 1986 $newToken['type'] = 'T_GREATER_THAN';
Chris@17 1987 break;
Chris@17 1988 case '!':
Chris@17 1989 $newToken['type'] = 'T_BOOLEAN_NOT';
Chris@17 1990 break;
Chris@17 1991 case ',':
Chris@17 1992 $newToken['type'] = 'T_COMMA';
Chris@17 1993 break;
Chris@17 1994 case '@':
Chris@17 1995 $newToken['type'] = 'T_ASPERAND';
Chris@17 1996 break;
Chris@17 1997 case '$':
Chris@17 1998 $newToken['type'] = 'T_DOLLAR';
Chris@17 1999 break;
Chris@17 2000 case '`':
Chris@17 2001 $newToken['type'] = 'T_BACKTICK';
Chris@17 2002 break;
Chris@17 2003 default:
Chris@17 2004 $newToken['type'] = 'T_NONE';
Chris@17 2005 break;
Chris@17 2006 }//end switch
Chris@17 2007
Chris@17 2008 $newToken['code'] = constant($newToken['type']);
Chris@17 2009 $newToken['content'] = $token;
Chris@17 2010
Chris@17 2011 self::$resolveTokenCache[$token] = $newToken;
Chris@17 2012 return $newToken;
Chris@17 2013
Chris@17 2014 }//end resolveSimpleToken()
Chris@17 2015
Chris@17 2016
Chris@17 2017 }//end class