Chris@0: errors = UTF8Utils::checkForIllegalCodepoints($data); Chris@17: Chris@17: $data = $this->replaceLinefeeds($data); Chris@17: Chris@17: $this->data = $data; Chris@17: $this->char = 0; Chris@17: $this->EOF = strlen($data); Chris@17: } Chris@17: Chris@17: /** Chris@17: * Check if upcomming chars match the given sequence. Chris@17: * Chris@17: * This will read the stream for the $sequence. If it's Chris@17: * found, this will return true. If not, return false. Chris@17: * Since this unconsumes any chars it reads, the caller Chris@17: * will still need to read the next sequence, even if Chris@17: * this returns true. Chris@17: * Chris@17: * Example: $this->scanner->sequenceMatches('') will Chris@17: * see if the input stream is at the start of a Chris@17: * '' string. Chris@17: * Chris@17: * @param string $sequence Chris@17: * @param bool $caseSensitive Chris@17: * Chris@17: * @return bool Chris@17: */ Chris@17: public function sequenceMatches($sequence, $caseSensitive = true) Chris@17: { Chris@17: $portion = substr($this->data, $this->char, strlen($sequence)); Chris@17: Chris@17: return $caseSensitive ? $portion === $sequence : 0 === strcasecmp($portion, $sequence); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get the current position. Chris@0: * Chris@0: * @return int The current intiger byte position. Chris@0: */ Chris@0: public function position() Chris@0: { Chris@17: return $this->char; Chris@0: } Chris@0: Chris@0: /** Chris@0: * Take a peek at the next character in the data. Chris@0: * Chris@0: * @return string The next character. Chris@0: */ Chris@0: public function peek() Chris@0: { Chris@17: if (($this->char + 1) <= $this->EOF) { Chris@17: return $this->data[$this->char + 1]; Chris@17: } Chris@17: Chris@17: return false; Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get the next character. Chris@0: * Note: This advances the pointer. Chris@0: * Chris@0: * @return string The next character. Chris@0: */ Chris@0: public function next() Chris@0: { Chris@17: ++$this->char; Chris@17: Chris@17: if ($this->char < $this->EOF) { Chris@17: return $this->data[$this->char]; Chris@0: } Chris@0: Chris@0: return false; Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get the current character. Chris@0: * Note, this does not advance the pointer. Chris@0: * Chris@0: * @return string The current character. Chris@0: */ Chris@0: public function current() Chris@0: { Chris@17: if ($this->char < $this->EOF) { Chris@17: return $this->data[$this->char]; Chris@0: } Chris@0: Chris@0: return false; Chris@0: } Chris@0: Chris@0: /** Chris@0: * Silently consume N chars. Chris@17: * Chris@17: * @param int $count Chris@0: */ Chris@0: public function consume($count = 1) Chris@0: { Chris@17: $this->char += $count; Chris@0: } Chris@0: Chris@0: /** Chris@0: * Unconsume some of the data. Chris@0: * This moves the data pointer backwards. Chris@0: * Chris@17: * @param int $howMany The number of characters to move the pointer back. Chris@0: */ Chris@0: public function unconsume($howMany = 1) Chris@0: { Chris@17: if (($this->char - $howMany) >= 0) { Chris@17: $this->char -= $howMany; Chris@17: } Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get the next group of that contains hex characters. Chris@0: * Note, along with getting the characters the pointer in the data will be Chris@0: * moved as well. Chris@0: * Chris@0: * @return string The next group that is hex characters. Chris@0: */ Chris@0: public function getHex() Chris@0: { Chris@17: return $this->doCharsWhile(static::CHARS_HEX); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get the next group of characters that are ASCII Alpha characters. Chris@0: * Note, along with getting the characters the pointer in the data will be Chris@0: * moved as well. Chris@0: * Chris@0: * @return string The next group of ASCII alpha characters. Chris@0: */ Chris@0: public function getAsciiAlpha() Chris@0: { Chris@17: return $this->doCharsWhile(static::CHARS_ALPHA); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get the next group of characters that are ASCII Alpha characters and numbers. Chris@0: * Note, along with getting the characters the pointer in the data will be Chris@0: * moved as well. Chris@0: * Chris@0: * @return string The next group of ASCII alpha characters and numbers. Chris@0: */ Chris@0: public function getAsciiAlphaNum() Chris@0: { Chris@17: return $this->doCharsWhile(static::CHARS_ALNUM); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get the next group of numbers. Chris@0: * Note, along with getting the characters the pointer in the data will be Chris@0: * moved as well. Chris@0: * Chris@0: * @return string The next group of numbers. Chris@0: */ Chris@0: public function getNumeric() Chris@0: { Chris@17: return $this->doCharsWhile('0123456789'); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Consume whitespace. Chris@17: * Whitespace in HTML5 is: formfeed, tab, newline, space. Chris@0: * Chris@17: * @return int The length of the matched whitespaces. Chris@0: */ Chris@0: public function whitespace() Chris@0: { Chris@17: if ($this->char >= $this->EOF) { Chris@17: return false; Chris@17: } Chris@17: Chris@17: $len = strspn($this->data, "\n\t\f ", $this->char); Chris@17: Chris@17: $this->char += $len; Chris@17: Chris@17: return $len; Chris@0: } Chris@0: Chris@0: /** Chris@0: * Returns the current line that is being consumed. Chris@0: * Chris@0: * @return int The current line number. Chris@0: */ Chris@0: public function currentLine() Chris@0: { Chris@17: if (empty($this->EOF) || 0 === $this->char) { Chris@17: return 1; Chris@17: } Chris@17: Chris@17: // Add one to $this->char because we want the number for the next Chris@17: // byte to be processed. Chris@17: return substr_count($this->data, "\n", 0, min($this->char, $this->EOF)) + 1; Chris@0: } Chris@0: Chris@0: /** Chris@0: * Read chars until something in the mask is encountered. Chris@17: * Chris@17: * @param string $mask Chris@17: * Chris@17: * @return mixed Chris@0: */ Chris@0: public function charsUntil($mask) Chris@0: { Chris@17: return $this->doCharsUntil($mask); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Read chars as long as the mask matches. Chris@17: * Chris@17: * @param string $mask Chris@17: * Chris@17: * @return int Chris@0: */ Chris@0: public function charsWhile($mask) Chris@0: { Chris@17: return $this->doCharsWhile($mask); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Returns the current column of the current line that the tokenizer is at. Chris@0: * Chris@0: * Newlines are column 0. The first char after a newline is column 1. Chris@0: * Chris@0: * @return int The column number. Chris@0: */ Chris@0: public function columnOffset() Chris@0: { Chris@17: // Short circuit for the first char. Chris@17: if (0 === $this->char) { Chris@17: return 0; Chris@17: } Chris@17: Chris@17: // strrpos is weird, and the offset needs to be negative for what we Chris@17: // want (i.e., the last \n before $this->char). This needs to not have Chris@17: // one (to make it point to the next character, the one we want the Chris@17: // position of) added to it because strrpos's behaviour includes the Chris@17: // final offset byte. Chris@17: $backwardFrom = $this->char - 1 - strlen($this->data); Chris@17: $lastLine = strrpos($this->data, "\n", $backwardFrom); Chris@17: Chris@17: // However, for here we want the length up until the next byte to be Chris@17: // processed, so add one to the current byte ($this->char). Chris@17: if (false !== $lastLine) { Chris@17: $findLengthOf = substr($this->data, $lastLine + 1, $this->char - 1 - $lastLine); Chris@17: } else { Chris@17: // After a newline. Chris@17: $findLengthOf = substr($this->data, 0, $this->char); Chris@17: } Chris@17: Chris@17: return UTF8Utils::countChars($findLengthOf); Chris@0: } Chris@0: Chris@0: /** Chris@0: * Get all characters until EOF. Chris@0: * Chris@0: * This consumes characters until the EOF. Chris@0: * Chris@0: * @return int The number of characters remaining. Chris@0: */ Chris@0: public function remainingChars() Chris@0: { Chris@17: if ($this->char < $this->EOF) { Chris@17: $data = substr($this->data, $this->char); Chris@17: $this->char = $this->EOF; Chris@17: Chris@17: return $data; Chris@17: } Chris@17: Chris@17: return ''; // false; Chris@17: } Chris@17: Chris@17: /** Chris@17: * Replace linefeed characters according to the spec. Chris@17: * Chris@17: * @param $data Chris@17: * Chris@17: * @return string Chris@17: */ Chris@17: private function replaceLinefeeds($data) Chris@17: { Chris@17: /* Chris@17: * U+000D CARRIAGE RETURN (CR) characters and U+000A LINE FEED (LF) characters are treated specially. Chris@17: * Any CR characters that are followed by LF characters must be removed, and any CR characters not Chris@17: * followed by LF characters must be converted to LF characters. Thus, newlines in HTML DOMs are Chris@17: * represented by LF characters, and there are never any CR characters in the input to the tokenization Chris@17: * stage. Chris@17: */ Chris@17: $crlfTable = array( Chris@17: "\0" => "\xEF\xBF\xBD", Chris@17: "\r\n" => "\n", Chris@17: "\r" => "\n", Chris@17: ); Chris@17: Chris@17: return strtr($data, $crlfTable); Chris@17: } Chris@17: Chris@17: /** Chris@17: * Read to a particular match (or until $max bytes are consumed). Chris@17: * Chris@17: * This operates on byte sequences, not characters. Chris@17: * Chris@17: * Matches as far as possible until we reach a certain set of bytes Chris@17: * and returns the matched substring. Chris@17: * Chris@17: * @param string $bytes Bytes to match. Chris@17: * @param int $max Maximum number of bytes to scan. Chris@17: * Chris@17: * @return mixed Index or false if no match is found. You should use strong Chris@17: * equality when checking the result, since index could be 0. Chris@17: */ Chris@17: private function doCharsUntil($bytes, $max = null) Chris@17: { Chris@17: if ($this->char >= $this->EOF) { Chris@17: return false; Chris@17: } Chris@17: Chris@17: if (0 === $max || $max) { Chris@17: $len = strcspn($this->data, $bytes, $this->char, $max); Chris@17: } else { Chris@17: $len = strcspn($this->data, $bytes, $this->char); Chris@17: } Chris@17: Chris@17: $string = (string) substr($this->data, $this->char, $len); Chris@17: $this->char += $len; Chris@17: Chris@17: return $string; Chris@17: } Chris@17: Chris@17: /** Chris@17: * Returns the string so long as $bytes matches. Chris@17: * Chris@17: * Matches as far as possible with a certain set of bytes Chris@17: * and returns the matched substring. Chris@17: * Chris@17: * @param string $bytes A mask of bytes to match. If ANY byte in this mask matches the Chris@17: * current char, the pointer advances and the char is part of the Chris@17: * substring. Chris@17: * @param int $max The max number of chars to read. Chris@17: * Chris@17: * @return string Chris@17: */ Chris@17: private function doCharsWhile($bytes, $max = null) Chris@17: { Chris@17: if ($this->char >= $this->EOF) { Chris@17: return false; Chris@17: } Chris@17: Chris@17: if (0 === $max || $max) { Chris@17: $len = strspn($this->data, $bytes, $this->char, $max); Chris@17: } else { Chris@17: $len = strspn($this->data, $bytes, $this->char); Chris@17: } Chris@17: Chris@17: $string = (string) substr($this->data, $this->char, $len); Chris@17: $this->char += $len; Chris@17: Chris@17: return $string; Chris@0: } Chris@0: }