annotate vendor/masterminds/html5/src/HTML5/Parser/UTF8Utils.php @ 19:fa3358dc1485 tip

Add ndrum files
author Chris Cannam
date Wed, 28 Aug 2019 13:14:47 +0100
parents af1871eacc83
children
rev   line source
Chris@0 1 <?php
Chris@17 2
Chris@0 3 namespace Masterminds\HTML5\Parser;
Chris@17 4
Chris@0 5 /*
Chris@18 6 Portions based on code from html5lib files with the following copyright:
Chris@0 7
Chris@0 8 Copyright 2009 Geoffrey Sneddon <http://gsnedders.com/>
Chris@0 9
Chris@0 10 Permission is hereby granted, free of charge, to any person obtaining a
Chris@0 11 copy of this software and associated documentation files (the
Chris@0 12 "Software"), to deal in the Software without restriction, including
Chris@0 13 without limitation the rights to use, copy, modify, merge, publish,
Chris@0 14 distribute, sublicense, and/or sell copies of the Software, and to
Chris@0 15 permit persons to whom the Software is furnished to do so, subject to
Chris@0 16 the following conditions:
Chris@0 17
Chris@0 18 The above copyright notice and this permission notice shall be included
Chris@0 19 in all copies or substantial portions of the Software.
Chris@0 20
Chris@0 21 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
Chris@0 22 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
Chris@0 23 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
Chris@0 24 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
Chris@0 25 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
Chris@0 26 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
Chris@0 27 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Chris@0 28 */
Chris@17 29
Chris@17 30 use Masterminds\HTML5\Exception;
Chris@17 31
Chris@0 32 class UTF8Utils
Chris@0 33 {
Chris@0 34 /**
Chris@18 35 * The Unicode replacement character.
Chris@0 36 */
Chris@0 37 const FFFD = "\xEF\xBF\xBD";
Chris@0 38
Chris@0 39 /**
Chris@0 40 * Count the number of characters in a string.
Chris@18 41 * UTF-8 aware. This will try (in order) iconv, MB, libxml, and finally a custom counter.
Chris@17 42 *
Chris@17 43 * @param string $string
Chris@17 44 *
Chris@17 45 * @return int
Chris@0 46 */
Chris@0 47 public static function countChars($string)
Chris@0 48 {
Chris@0 49 // Get the length for the string we need.
Chris@0 50 if (function_exists('mb_strlen')) {
Chris@0 51 return mb_strlen($string, 'utf-8');
Chris@18 52 }
Chris@18 53
Chris@18 54 if (function_exists('iconv_strlen')) {
Chris@0 55 return iconv_strlen($string, 'utf-8');
Chris@18 56 }
Chris@18 57
Chris@18 58 if (function_exists('utf8_decode')) {
Chris@0 59 // MPB: Will this work? Won't certain decodes lead to two chars
Chris@0 60 // extrapolated out of 2-byte chars?
Chris@0 61 return strlen(utf8_decode($string));
Chris@0 62 }
Chris@18 63
Chris@0 64 $count = count_chars($string);
Chris@18 65
Chris@0 66 // 0x80 = 0x7F - 0 + 1 (one added to get inclusive range)
Chris@0 67 // 0x33 = 0xF4 - 0x2C + 1 (one added to get inclusive range)
Chris@0 68 return array_sum(array_slice($count, 0, 0x80)) + array_sum(array_slice($count, 0xC2, 0x33));
Chris@0 69 }
Chris@0 70
Chris@0 71 /**
Chris@0 72 * Convert data from the given encoding to UTF-8.
Chris@0 73 *
Chris@0 74 * This has not yet been tested with charactersets other than UTF-8.
Chris@0 75 * It should work with ISO-8859-1/-13 and standard Latin Win charsets.
Chris@0 76 *
Chris@17 77 * @param string $data The data to convert
Chris@17 78 * @param string $encoding A valid encoding. Examples: http://www.php.net/manual/en/mbstring.supported-encodings.php
Chris@17 79 *
Chris@17 80 * @return string
Chris@0 81 */
Chris@0 82 public static function convertToUTF8($data, $encoding = 'UTF-8')
Chris@0 83 {
Chris@0 84 /*
Chris@18 85 * From the HTML5 spec: Given an encoding, the bytes in the input stream must be converted
Chris@18 86 * to Unicode characters for the tokeniser, as described by the rules for that encoding,
Chris@18 87 * except that the leading U+FEFF BYTE ORDER MARK character, if any, must not be stripped
Chris@18 88 * by the encoding layer (it is stripped by the rule below). Bytes or sequences of bytes
Chris@18 89 * in the original byte stream that could not be converted to Unicode characters must be
Chris@18 90 * converted to U+FFFD REPLACEMENT CHARACTER code points.
Chris@0 91 */
Chris@0 92
Chris@0 93 // mb_convert_encoding is chosen over iconv because of a bug. The best
Chris@0 94 // details for the bug are on http://us1.php.net/manual/en/function.iconv.php#108643
Chris@0 95 // which contains links to the actual but reports as well as work around
Chris@0 96 // details.
Chris@0 97 if (function_exists('mb_convert_encoding')) {
Chris@0 98 // mb library has the following behaviors:
Chris@0 99 // - UTF-16 surrogates result in false.
Chris@0 100 // - Overlongs and outside Plane 16 result in empty strings.
Chris@0 101
Chris@0 102 // Before we run mb_convert_encoding we need to tell it what to do with
Chris@0 103 // characters it does not know. This could be different than the parent
Chris@0 104 // application executing this library so we store the value, change it
Chris@0 105 // to our needs, and then change it back when we are done. This feels
Chris@0 106 // a little excessive and it would be great if there was a better way.
Chris@0 107 $save = mb_substitute_character();
Chris@0 108 mb_substitute_character('none');
Chris@0 109 $data = mb_convert_encoding($data, 'UTF-8', $encoding);
Chris@0 110 mb_substitute_character($save);
Chris@18 111 }
Chris@18 112 // @todo Get iconv running in at least some environments if that is possible.
Chris@17 113 elseif (function_exists('iconv') && 'auto' !== $encoding) {
Chris@0 114 // fprintf(STDOUT, "iconv found\n");
Chris@0 115 // iconv has the following behaviors:
Chris@0 116 // - Overlong representations are ignored.
Chris@0 117 // - Beyond Plane 16 is replaced with a lower char.
Chris@0 118 // - Incomplete sequences generate a warning.
Chris@0 119 $data = @iconv($encoding, 'UTF-8//IGNORE', $data);
Chris@0 120 } else {
Chris@0 121 throw new Exception('Not implemented, please install mbstring or iconv');
Chris@0 122 }
Chris@0 123
Chris@0 124 /*
Chris@0 125 * One leading U+FEFF BYTE ORDER MARK character must be ignored if any are present.
Chris@0 126 */
Chris@17 127 if ("\xEF\xBB\xBF" === substr($data, 0, 3)) {
Chris@0 128 $data = substr($data, 3);
Chris@0 129 }
Chris@0 130
Chris@0 131 return $data;
Chris@0 132 }
Chris@0 133
Chris@0 134 /**
Chris@0 135 * Checks for Unicode code points that are not valid in a document.
Chris@0 136 *
Chris@17 137 * @param string $data A string to analyze
Chris@17 138 *
Chris@17 139 * @return array An array of (string) error messages produced by the scanning
Chris@0 140 */
Chris@0 141 public static function checkForIllegalCodepoints($data)
Chris@0 142 {
Chris@0 143 // Vestigal error handling.
Chris@0 144 $errors = array();
Chris@0 145
Chris@0 146 /*
Chris@18 147 * All U+0000 null characters in the input must be replaced by U+FFFD REPLACEMENT CHARACTERs.
Chris@18 148 * Any occurrences of such characters is a parse error.
Chris@0 149 */
Chris@17 150 for ($i = 0, $count = substr_count($data, "\0"); $i < $count; ++$i) {
Chris@0 151 $errors[] = 'null-character';
Chris@0 152 }
Chris@0 153
Chris@0 154 /*
Chris@18 155 * Any occurrences of any characters in the ranges U+0001 to U+0008, U+000B, U+000E to U+001F, U+007F
Chris@18 156 * to U+009F, U+D800 to U+DFFF , U+FDD0 to U+FDEF, and characters U+FFFE, U+FFFF, U+1FFFE, U+1FFFF,
Chris@18 157 * U+2FFFE, U+2FFFF, U+3FFFE, U+3FFFF, U+4FFFE, U+4FFFF, U+5FFFE, U+5FFFF, U+6FFFE, U+6FFFF, U+7FFFE,
Chris@18 158 * U+7FFFF, U+8FFFE, U+8FFFF, U+9FFFE, U+9FFFF, U+AFFFE, U+AFFFF, U+BFFFE, U+BFFFF, U+CFFFE, U+CFFFF,
Chris@18 159 * U+DFFFE, U+DFFFF, U+EFFFE, U+EFFFF, U+FFFFE, U+FFFFF, U+10FFFE, and U+10FFFF are parse errors.
Chris@18 160 * (These are all control characters or permanently undefined Unicode characters.)
Chris@0 161 */
Chris@0 162 // Check PCRE is loaded.
Chris@0 163 $count = preg_match_all(
Chris@0 164 '/(?:
Chris@0 165 [\x01-\x08\x0B\x0E-\x1F\x7F] # U+0001 to U+0008, U+000B, U+000E to U+001F and U+007F
Chris@0 166 |
Chris@0 167 \xC2[\x80-\x9F] # U+0080 to U+009F
Chris@0 168 |
Chris@0 169 \xED(?:\xA0[\x80-\xFF]|[\xA1-\xBE][\x00-\xFF]|\xBF[\x00-\xBF]) # U+D800 to U+DFFFF
Chris@0 170 |
Chris@0 171 \xEF\xB7[\x90-\xAF] # U+FDD0 to U+FDEF
Chris@0 172 |
Chris@0 173 \xEF\xBF[\xBE\xBF] # U+FFFE and U+FFFF
Chris@0 174 |
Chris@0 175 [\xF0-\xF4][\x8F-\xBF]\xBF[\xBE\xBF] # U+nFFFE and U+nFFFF (1 <= n <= 10_{16})
Chris@0 176 )/x', $data, $matches);
Chris@17 177 for ($i = 0; $i < $count; ++$i) {
Chris@0 178 $errors[] = 'invalid-codepoint';
Chris@0 179 }
Chris@0 180
Chris@0 181 return $errors;
Chris@0 182 }
Chris@0 183 }