annotate vendor/nikic/php-parser/grammar/rebuildParsers.php @ 19:fa3358dc1485 tip

Add ndrum files
author Chris Cannam
date Wed, 28 Aug 2019 13:14:47 +0100
parents 129ea1e6d783
children
rev   line source
Chris@0 1 <?php
Chris@0 2
Chris@0 3 $grammarFileToName = [
Chris@0 4 __DIR__ . '/php5.y' => 'Php5',
Chris@0 5 __DIR__ . '/php7.y' => 'Php7',
Chris@0 6 ];
Chris@0 7
Chris@0 8 $tokensFile = __DIR__ . '/tokens.y';
Chris@0 9 $tokensTemplate = __DIR__ . '/tokens.template';
Chris@0 10 $skeletonFile = __DIR__ . '/parser.template';
Chris@0 11 $tmpGrammarFile = __DIR__ . '/tmp_parser.phpy';
Chris@0 12 $tmpResultFile = __DIR__ . '/tmp_parser.php';
Chris@0 13 $resultDir = __DIR__ . '/../lib/PhpParser/Parser';
Chris@0 14 $tokensResultsFile = $resultDir . '/Tokens.php';
Chris@0 15
Chris@0 16 // check for kmyacc.exe binary in this directory, otherwise fall back to global name
Chris@0 17 $kmyacc = __DIR__ . '/kmyacc.exe';
Chris@0 18 if (!file_exists($kmyacc)) {
Chris@0 19 $kmyacc = 'kmyacc';
Chris@0 20 }
Chris@0 21
Chris@0 22 $options = array_flip($argv);
Chris@0 23 $optionDebug = isset($options['--debug']);
Chris@0 24 $optionKeepTmpGrammar = isset($options['--keep-tmp-grammar']);
Chris@0 25
Chris@0 26 ///////////////////////////////
Chris@0 27 /// Utility regex constants ///
Chris@0 28 ///////////////////////////////
Chris@0 29
Chris@0 30 const LIB = '(?(DEFINE)
Chris@0 31 (?<singleQuotedString>\'[^\\\\\']*+(?:\\\\.[^\\\\\']*+)*+\')
Chris@0 32 (?<doubleQuotedString>"[^\\\\"]*+(?:\\\\.[^\\\\"]*+)*+")
Chris@0 33 (?<string>(?&singleQuotedString)|(?&doubleQuotedString))
Chris@0 34 (?<comment>/\*[^*]*+(?:\*(?!/)[^*]*+)*+\*/)
Chris@0 35 (?<code>\{[^\'"/{}]*+(?:(?:(?&string)|(?&comment)|(?&code)|/)[^\'"/{}]*+)*+})
Chris@0 36 )';
Chris@0 37
Chris@0 38 const PARAMS = '\[(?<params>[^[\]]*+(?:\[(?&params)\][^[\]]*+)*+)\]';
Chris@0 39 const ARGS = '\((?<args>[^()]*+(?:\((?&args)\)[^()]*+)*+)\)';
Chris@0 40
Chris@0 41 ///////////////////
Chris@0 42 /// Main script ///
Chris@0 43 ///////////////////
Chris@0 44
Chris@0 45 $tokens = file_get_contents($tokensFile);
Chris@0 46
Chris@0 47 foreach ($grammarFileToName as $grammarFile => $name) {
Chris@0 48 echo "Building temporary $name grammar file.\n";
Chris@0 49
Chris@0 50 $grammarCode = file_get_contents($grammarFile);
Chris@0 51 $grammarCode = str_replace('%tokens', $tokens, $grammarCode);
Chris@0 52
Chris@0 53 $grammarCode = resolveNodes($grammarCode);
Chris@0 54 $grammarCode = resolveMacros($grammarCode);
Chris@0 55 $grammarCode = resolveStackAccess($grammarCode);
Chris@0 56
Chris@0 57 file_put_contents($tmpGrammarFile, $grammarCode);
Chris@0 58
Chris@0 59 $additionalArgs = $optionDebug ? '-t -v' : '';
Chris@0 60
Chris@0 61 echo "Building $name parser.\n";
Chris@0 62 $output = trim(shell_exec("$kmyacc $additionalArgs -l -m $skeletonFile -p $name $tmpGrammarFile 2>&1"));
Chris@0 63 echo "Output: \"$output\"\n";
Chris@0 64
Chris@0 65 $resultCode = file_get_contents($tmpResultFile);
Chris@0 66 $resultCode = removeTrailingWhitespace($resultCode);
Chris@0 67
Chris@0 68 ensureDirExists($resultDir);
Chris@0 69 file_put_contents("$resultDir/$name.php", $resultCode);
Chris@0 70 unlink($tmpResultFile);
Chris@0 71
Chris@0 72 echo "Building token definition.\n";
Chris@0 73 $output = trim(shell_exec("$kmyacc -l -m $tokensTemplate $tmpGrammarFile 2>&1"));
Chris@0 74 assert($output === '');
Chris@0 75 rename($tmpResultFile, $tokensResultsFile);
Chris@0 76
Chris@0 77 if (!$optionKeepTmpGrammar) {
Chris@0 78 unlink($tmpGrammarFile);
Chris@0 79 }
Chris@0 80 }
Chris@0 81
Chris@0 82 ///////////////////////////////
Chris@0 83 /// Preprocessing functions ///
Chris@0 84 ///////////////////////////////
Chris@0 85
Chris@0 86 function resolveNodes($code) {
Chris@0 87 return preg_replace_callback(
Chris@0 88 '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
Chris@0 89 function($matches) {
Chris@0 90 // recurse
Chris@0 91 $matches['params'] = resolveNodes($matches['params']);
Chris@0 92
Chris@0 93 $params = magicSplit(
Chris@0 94 '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
Chris@0 95 $matches['params']
Chris@0 96 );
Chris@0 97
Chris@0 98 $paramCode = '';
Chris@0 99 foreach ($params as $param) {
Chris@0 100 $paramCode .= $param . ', ';
Chris@0 101 }
Chris@0 102
Chris@0 103 return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
Chris@0 104 },
Chris@0 105 $code
Chris@0 106 );
Chris@0 107 }
Chris@0 108
Chris@0 109 function resolveMacros($code) {
Chris@0 110 return preg_replace_callback(
Chris@0 111 '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
Chris@0 112 function($matches) {
Chris@0 113 // recurse
Chris@0 114 $matches['args'] = resolveMacros($matches['args']);
Chris@0 115
Chris@0 116 $name = $matches['name'];
Chris@0 117 $args = magicSplit(
Chris@0 118 '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
Chris@0 119 $matches['args']
Chris@0 120 );
Chris@0 121
Chris@0 122 if ('attributes' == $name) {
Chris@0 123 assertArgs(0, $args, $name);
Chris@0 124 return '$this->startAttributeStack[#1] + $this->endAttributes';
Chris@0 125 }
Chris@0 126
Chris@0 127 if ('stackAttributes' == $name) {
Chris@0 128 assertArgs(1, $args, $name);
Chris@0 129 return '$this->startAttributeStack[' . $args[0] . ']'
Chris@0 130 . ' + $this->endAttributeStack[' . $args[0] . ']';
Chris@0 131 }
Chris@0 132
Chris@0 133 if ('init' == $name) {
Chris@0 134 return '$$ = array(' . implode(', ', $args) . ')';
Chris@0 135 }
Chris@0 136
Chris@0 137 if ('push' == $name) {
Chris@0 138 assertArgs(2, $args, $name);
Chris@0 139
Chris@0 140 return $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0];
Chris@0 141 }
Chris@0 142
Chris@0 143 if ('pushNormalizing' == $name) {
Chris@0 144 assertArgs(2, $args, $name);
Chris@0 145
Chris@0 146 return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
Chris@0 147 . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
Chris@0 148 }
Chris@0 149
Chris@0 150 if ('toArray' == $name) {
Chris@0 151 assertArgs(1, $args, $name);
Chris@0 152
Chris@0 153 return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
Chris@0 154 }
Chris@0 155
Chris@0 156 if ('parseVar' == $name) {
Chris@0 157 assertArgs(1, $args, $name);
Chris@0 158
Chris@0 159 return 'substr(' . $args[0] . ', 1)';
Chris@0 160 }
Chris@0 161
Chris@0 162 if ('parseEncapsed' == $name) {
Chris@0 163 assertArgs(3, $args, $name);
Chris@0 164
Chris@0 165 return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
Chris@0 166 . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
Chris@0 167 }
Chris@0 168
Chris@0 169 if ('makeNop' == $name) {
Chris@13 170 assertArgs(3, $args, $name);
Chris@0 171
Chris@0 172 return '$startAttributes = ' . $args[1] . ';'
Chris@0 173 . ' if (isset($startAttributes[\'comments\']))'
Chris@13 174 . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
Chris@0 175 . ' else { ' . $args[0] . ' = null; }';
Chris@0 176 }
Chris@0 177
Chris@0 178 if ('strKind' == $name) {
Chris@0 179 assertArgs(1, $args, $name);
Chris@0 180
Chris@0 181 return '(' . $args[0] . '[0] === "\'" || (' . $args[0] . '[1] === "\'" && '
Chris@0 182 . '(' . $args[0] . '[0] === \'b\' || ' . $args[0] . '[0] === \'B\')) '
Chris@0 183 . '? Scalar\String_::KIND_SINGLE_QUOTED : Scalar\String_::KIND_DOUBLE_QUOTED)';
Chris@0 184 }
Chris@0 185
Chris@0 186 if ('prependLeadingComments' == $name) {
Chris@0 187 assertArgs(1, $args, $name);
Chris@0 188
Chris@0 189 return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
Chris@0 190 . 'if (!empty($attrs[\'comments\'])) {'
Chris@0 191 . '$stmts[0]->setAttribute(\'comments\', '
Chris@0 192 . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
Chris@0 193 }
Chris@0 194
Chris@0 195 return $matches[0];
Chris@0 196 },
Chris@0 197 $code
Chris@0 198 );
Chris@0 199 }
Chris@0 200
Chris@0 201 function assertArgs($num, $args, $name) {
Chris@0 202 if ($num != count($args)) {
Chris@0 203 die('Wrong argument count for ' . $name . '().');
Chris@0 204 }
Chris@0 205 }
Chris@0 206
Chris@0 207 function resolveStackAccess($code) {
Chris@0 208 $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
Chris@0 209 $code = preg_replace('/#(\d+)/', '$$1', $code);
Chris@0 210 return $code;
Chris@0 211 }
Chris@0 212
Chris@0 213 function removeTrailingWhitespace($code) {
Chris@0 214 $lines = explode("\n", $code);
Chris@0 215 $lines = array_map('rtrim', $lines);
Chris@0 216 return implode("\n", $lines);
Chris@0 217 }
Chris@0 218
Chris@0 219 function ensureDirExists($dir) {
Chris@0 220 if (!is_dir($dir)) {
Chris@0 221 mkdir($dir, 0777, true);
Chris@0 222 }
Chris@0 223 }
Chris@0 224
Chris@0 225 //////////////////////////////
Chris@0 226 /// Regex helper functions ///
Chris@0 227 //////////////////////////////
Chris@0 228
Chris@0 229 function regex($regex) {
Chris@0 230 return '~' . LIB . '(?:' . str_replace('~', '\~', $regex) . ')~';
Chris@0 231 }
Chris@0 232
Chris@0 233 function magicSplit($regex, $string) {
Chris@0 234 $pieces = preg_split(regex('(?:(?&string)|(?&comment)|(?&code))(*SKIP)(*FAIL)|' . $regex), $string);
Chris@0 235
Chris@0 236 foreach ($pieces as &$piece) {
Chris@0 237 $piece = trim($piece);
Chris@0 238 }
Chris@0 239
Chris@0 240 if ($pieces === ['']) {
Chris@0 241 return [];
Chris@0 242 }
Chris@0 243
Chris@0 244 return $pieces;
Chris@0 245 }