Chris@14: <?php declare(strict_types = 1);
Chris@14: namespace TheSeer\Tokenizer;
Chris@14: 
Chris@14: class Tokenizer {
Chris@14: 
Chris@14:     /**
Chris@14:      * Token Map for "non-tokens"
Chris@14:      *
Chris@14:      * @var array
Chris@14:      */
Chris@14:     private $map = [
Chris@14:         '(' => 'T_OPEN_BRACKET',
Chris@14:         ')' => 'T_CLOSE_BRACKET',
Chris@14:         '[' => 'T_OPEN_SQUARE',
Chris@14:         ']' => 'T_CLOSE_SQUARE',
Chris@14:         '{' => 'T_OPEN_CURLY',
Chris@14:         '}' => 'T_CLOSE_CURLY',
Chris@14:         ';' => 'T_SEMICOLON',
Chris@14:         '.' => 'T_DOT',
Chris@14:         ',' => 'T_COMMA',
Chris@14:         '=' => 'T_EQUAL',
Chris@14:         '<' => 'T_LT',
Chris@14:         '>' => 'T_GT',
Chris@14:         '+' => 'T_PLUS',
Chris@14:         '-' => 'T_MINUS',
Chris@14:         '*' => 'T_MULT',
Chris@14:         '/' => 'T_DIV',
Chris@14:         '?' => 'T_QUESTION_MARK',
Chris@14:         '!' => 'T_EXCLAMATION_MARK',
Chris@14:         ':' => 'T_COLON',
Chris@14:         '"' => 'T_DOUBLE_QUOTES',
Chris@14:         '@' => 'T_AT',
Chris@14:         '&' => 'T_AMPERSAND',
Chris@14:         '%' => 'T_PERCENT',
Chris@14:         '|' => 'T_PIPE',
Chris@14:         '$' => 'T_DOLLAR',
Chris@14:         '^' => 'T_CARET',
Chris@14:         '~' => 'T_TILDE',
Chris@14:         '`' => 'T_BACKTICK'
Chris@14:     ];
Chris@14: 
Chris@14:     public function parse(string $source): TokenCollection {
Chris@14:         $result = new TokenCollection();
Chris@18: 
Chris@18:         if ($source === '') {
Chris@18:             return $result;
Chris@18:         }
Chris@18: 
Chris@14:         $tokens = token_get_all($source);
Chris@14: 
Chris@14:         $lastToken = new Token(
Chris@14:             $tokens[0][2],
Chris@14:             'Placeholder',
Chris@14:             ''
Chris@14:         );
Chris@14: 
Chris@14:         foreach ($tokens as $pos => $tok) {
Chris@14:             if (is_string($tok)) {
Chris@14:                 $token = new Token(
Chris@14:                     $lastToken->getLine(),
Chris@14:                     $this->map[$tok],
Chris@14:                     $tok
Chris@14:                 );
Chris@14:                 $result->addToken($token);
Chris@14:                 $lastToken = $token;
Chris@14:                 continue;
Chris@14:             }
Chris@14: 
Chris@14:             $line   = $tok[2];
Chris@14:             $values = preg_split('/\R+/Uu', $tok[1]);
Chris@14: 
Chris@14:             foreach ($values as $v) {
Chris@14:                 $token = new Token(
Chris@14:                     $line,
Chris@14:                     token_name($tok[0]),
Chris@14:                     $v
Chris@14:                 );
Chris@14:                 $result->addToken($token);
Chris@14:                 $line++;
Chris@14:                 $lastToken = $token;
Chris@14:             }
Chris@14:         }
Chris@14: 
Chris@14:         return $result;
Chris@14:     }
Chris@14: 
Chris@14: }