Chris@0: <?php
Chris@0: 
Chris@0: /**
Chris@0:  * @file
Chris@0:  * Unifies formats of transliteration data from various sources.
Chris@0:  *
Chris@0:  * A few notes about this script:
Chris@0:  * - The functions in this file are NOT SECURE, because they use PHP functions
Chris@0:  *   like eval(). Absolutely do not run this script unless you trust the data
Chris@0:  *   files used for input.
Chris@0:  * - You will need to change the name of this file to remove the .txt extension
Chris@0:  *   before running it (it has been given this name so that you cannot run it
Chris@0:  *   by mistake). When you do that, move it out of your web root as well so
Chris@0:  *   that it cannot be run via a URL, and run the script via the PHP command
Chris@0:  *   at a command prompt.
Chris@0:  * - This script, depending on which portions of it you run, depends on having
Chris@0:  *   input data from various sources in sub-directories below where this file
Chris@0:  *   is located. The data inputs are as follows:
Chris@0:  *   - Existing Drupal Core transliteration data: Sub-directory 'data'; comes
Chris@0:  *     from core/lib/Drupal/Component/Transliteration/data
Chris@0:  *   - Midgardmvc data: Sub-directory 'utf8_to_ascii_db'; download from
Chris@0:  *     https://github.com/bergie/midgardmvc_helper_urlize/downloads
Chris@0:  *   - CPAN Text-Unidecode data: Sub-directory 'Unidecode'; download from
Chris@0:  *     http://search.cpan.org/~sburke/Text-Unidecode-0.04/lib/Text/Unidecode.pm
Chris@0:  *   - Node.js project: Sub-directory 'unidecoder_data'; download from
Chris@0:  *     https://github.com/bitwalker/stringex/downloads
Chris@0:  *   - JUnidecode project: Sub-directory 'junidecode'; download source from
Chris@0:  *     http://www.ippatsuman.com/projects/junidecode/index.html
Chris@0:  * - You will also need to make directory 'outdata' to hold output.
Chris@0:  * - If you plan to use the 'intl' data, you will also need to have the PECL
Chris@0:  *   packages 'yaml' and 'intl' installed.  See
Chris@0:  *   http://php.net/manual/install.pecl.downloads.php for generic PECL
Chris@0:  *   package installation instructions. The following commands on Ubuntu Linux
Chris@0:  *   will install yaml and intl packages:
Chris@0:  *   @code
Chris@0:  *   sudo apt-get install libyaml-dev
Chris@0:  *   sudo pecl install yaml
Chris@0:  *   sudo apt-get install php5-intl
Chris@0:  *   sudo apt-get install libicu-dev
Chris@0:  *   sudo pecl install intl
Chris@0:  *   @endcode
Chris@0:  *   After running these commands, you will need to make sure
Chris@0:  *   'extension=intl.so' and 'extension=yaml.so' are added to the php.ini file
Chris@0:  *   that is in use for the PHP command-line command.
Chris@0:  * - When you have collected all of the data and installed the required
Chris@0:  *   packages, you will need to find the specific commands below that you want
Chris@0:  *   to use and un-comment them. The preferred data source for Drupal Core is
Chris@0:  *   the PECL 'intl' package, and the line that needs to be un-commented in
Chris@0:  *   order to make a Drupal Core patch is:
Chris@0:  *   @code
Chris@0:  *   patch_drupal('outdata');
Chris@0:  *   @endcode
Chris@0:  * - The functions are documented in more detail in their headers where they
Chris@0:  *   are defined. Many have parameters that you can use to change the output.
Chris@0:  */
Chris@0: 
Chris@0: // Commands to read various data sources:
Chris@0: // $data = read_drupal_data();
Chris@0: // $data = read_midgard_data();
Chris@0: // $data = read_cpan_data();
Chris@0: // $data = read_nodejs_data();
Chris@0: // $data = read_intl_data();
Chris@0: // $data = read_junidecode_data();
Chris@0: 
Chris@0: // After running a read_*_data() function, you can print out the data
Chris@0: // (it will make a LOT of output):
Chris@0: // print_r($data);
Chris@0: 
Chris@0: // Command to read in all of data sources and output in CSV format, explaining
Chris@0: // the differences:
Chris@0: // read_all_to_csv();
Chris@0: 
Chris@0: // Command to patch Drupal Core data, using the intl data set, and put the
Chris@0: // resulting changed data files in the 'outdata' directory:
Chris@0: patch_drupal('outdata');
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in all transliteration data and outputs differences in CSV format.
Chris@0:  *
Chris@0:  * Each data set is compared to the Drupal Core reference data set, and the
Chris@0:  * differences are noted. The data must be in the locations noted in the
Chris@0:  * file header above. The CSV output has several columns. The first one is the
Chris@0:  * Unicode character code. The next columns contain the transliteration of
Chris@0:  * that character in each of the data sets. The last column, tells what the
Chris@0:  * differences are between the Drupal Core reference set and the other data
Chris@0:  * sets:
Chris@0:  * - missing: The target set is missing data that the Drupal set has.
Chris@0:  * - provided: The target set has provided data that Drupal does not have.
Chris@0:  * - case: The target and Drupal set output differ only in upper/lower case.
Chris@0:  * - different: The target and Drupal set output differ in more than just case.
Chris@0:  *
Chris@0:  * @param bool $print_all
Chris@0:  *   TRUE to print all data; FALSE (default) to print just data where there
Chris@0:  *   are differences between the Drupal set and other data sources.
Chris@0:  * @param bool $print_missing
Chris@0:  *   TRUE to print cases where one of the non-Drupal sets is missing information
Chris@0:  *   and that is the only difference; FALSE (default) to include these rows.
Chris@0:  */
Chris@0: function read_all_to_csv($print_all = FALSE, $print_missing = FALSE) {
Chris@0:   $data = array();
Chris@0:   $types = array('drupal', 'midgard', 'cpan', 'nodejs', 'junidecode', 'intl');
Chris@0: 
Chris@0:   // Alternatively, if you just want to compare a couple of data sets, you can
Chris@0:   // uncomment and edit the following line:
Chris@0:   // $types = array('drupal', 'intl');
Chris@0: 
Chris@0:   // Read in all the data.
Chris@0:   foreach ($types as $type) {
Chris@0:     $data[$type] = call_user_func('read_' . $type . '_data');
Chris@0:   }
Chris@0: 
Chris@0:   // Print CSV header row.
Chris@0:   print "character,";
Chris@0:   print implode(',', $types);
Chris@0:   print ",why\n";
Chris@0: 
Chris@0:   // Go through all the banks of character data.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0: 
Chris@0:     // Go through characters in bank; skip pure ASCII characters.
Chris@0:     $start = ($bank == 0) ? 0x80 : 0;
Chris@0:     for ($chr = $start; $chr < 256; $chr++) {
Chris@0: 
Chris@0:       // Gather the data together for this character.
Chris@0:       $row = array();
Chris@0:       foreach ($types as $type) {
Chris@0:         $row[$type] = (isset($data[$type][$bank][$chr]) && is_string($data[$type][$bank][$chr])) ? $data[$type][$bank][$chr] : '';
Chris@0:       }
Chris@0: 
Chris@0:       // Only print if there are differences or we are printing all data.
Chris@0:       $print = $print_all;
Chris@0:       $ref = $row['drupal'];
Chris@0:       $why = array();
Chris@0:       foreach ($types as $type) {
Chris@0:         // Try to characterize what the differences are.
Chris@0:         if ($row[$type] != $ref) {
Chris@0:           if ($row[$type] == '') {
Chris@0:             $why['missing'] = 'missing';
Chris@0:             if ($print_missing) {
Chris@0:               $print = TRUE;
Chris@0:             }
Chris@0:           }
Chris@0:           elseif ($ref == '') {
Chris@0:             $why['provided'] = 'provided';
Chris@0:             $print = TRUE;
Chris@0:           }
Chris@0:           elseif ($row[$type] == strtolower($ref) || $row[$type] == strtoupper($ref)) {
Chris@0:             $why['case'] = 'case';
Chris@0:             $print = TRUE;
Chris@0:           }
Chris@0:           else {
Chris@0:             $why['different'] = 'different';
Chris@0:             $print = TRUE;
Chris@0:           }
Chris@0:         }
Chris@0:       }
Chris@0: 
Chris@0:       // Print the data line.
Chris@0:       if ($print) {
Chris@0:         print '0x' . sprintf('%04x', 256 * $bank + $chr) . ',';
Chris@0:         foreach ($row as $out) {
Chris@0:           print '"' . addcslashes($out, '"') . '", ';
Chris@0:         }
Chris@0:         print implode(':', $why);
Chris@0:         print "\n";
Chris@0:       }
Chris@0:     }
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in 'intl' transliteration data and writes out changed Drupal files.
Chris@0:  *
Chris@0:  * Writes out the Drupal data files that would have to change to make our data
Chris@0:  * match the intl data set.
Chris@0:  *
Chris@0:  * @param string $outdir
Chris@0:  *   Directory to put the patched data files in (under where the script is
Chris@0:  *   being run).
Chris@0:  */
Chris@0: function patch_drupal($outdir) {
Chris@0:   $data = array();
Chris@0: 
Chris@0:   // Note that this is hard-wired below. Changing this line will have no
Chris@0:   // effect except to break this function.
Chris@0:   $types = array('drupal', 'intl');
Chris@0: 
Chris@0:   // Read in all the data.
Chris@0:   foreach ($types as $type) {
Chris@0:     $data[$type] = call_user_func('read_' . $type . '_data');
Chris@0:   }
Chris@0: 
Chris@0:   // Go through all the banks of character data.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0:     $print_bank = FALSE;
Chris@0: 
Chris@0:     // Go through characters in bank; skip pure ASCII characters.
Chris@0:     $start = ($bank == 0) ? 0x80 : 0;
Chris@0:     $newdata = array();
Chris@0:     for ($chr = 0; $chr < 256; $chr++) {
Chris@0:       // Fill up the start of the ASCII range.
Chris@0:       if ($chr < $start) {
Chris@0:         $newdata[$chr] = chr($chr);
Chris@0:         continue;
Chris@0:       }
Chris@0: 
Chris@0:       // Figure out what characters we actually have.
Chris@0:       $drupal = isset($data['drupal'][$bank][$chr]) ? $data['drupal'][$bank][$chr] : NULL;
Chris@0:       // Note that for intl, we only want to keep the transliteration if it
Chris@0:       // has something other than '' in it.
Chris@0:       $intl = isset($data['intl'][$bank][$chr]) && $data['intl'][$bank][$chr] != '' ? $data['intl'][$bank][$chr] : NULL;
Chris@0:       // Make sure we have something in the Drupal data set, in case we need
Chris@0:       // to print.
Chris@0:       $newdata[$chr] = $drupal;
Chris@0: 
Chris@0:       if (!isset($intl)) {
Chris@0:         continue;
Chris@0:       }
Chris@0:       if (!isset($drupal) || $drupal != $intl) {
Chris@0:         $print_bank = TRUE;
Chris@0:         $newdata[$chr] = $intl;
Chris@0:       }
Chris@0:     }
Chris@0: 
Chris@0:     // If we found a difference, output a data file.
Chris@0:     if ($print_bank) {
Chris@0:       write_data_file($newdata, $bank, $outdir);
Chris@0:     }
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in the Drupal Core generic transliteration data set.
Chris@0:  *
Chris@0:  * The data is expected to be in files xNN.php in directory 'data' under
Chris@0:  * this file's directory.
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Nested array of transliteration data. Outer keys are the first two
Chris@0:  *   bytes of Unicode characters (or 0 for base ASCII characters). The next
Chris@0:  *   level is the other two bytes, and the values are the transliterations.
Chris@0:  *
Chris@0:  * @see PhpTransliteration::readGenericData()
Chris@0:  */
Chris@0: function read_drupal_data() {
Chris@0:   $dir = __DIR__ . '/data';
Chris@0:   $out = array();
Chris@0: 
Chris@0:   // Read data files.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0:     $base = array();
Chris@0:     $file = $dir . '/x' . sprintf('%02x', $bank) . '.php';
Chris@0:     if (is_file($file)) {
Chris@0:       include($file);
Chris@0:     }
Chris@0:     $out[$bank] = $base;
Chris@0:   }
Chris@0: 
Chris@0:   return $out;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in the MidgardMVC transliteration data.
Chris@0:  *
Chris@0:  * The data is expected to be in files xNN.php in directory utf8_to_ascii_db
Chris@0:  * under the directory where this file resides. It can be downloaded from
Chris@0:  * https://github.com/bergie/midgardmvc_helper_urlize/downloads.
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Nested array of transliteration data. Outer keys are the first two
Chris@0:  *   bytes of Unicode characters (or 0 for base ASCII characters). The next
Chris@0:  *   level is the other two bytes, and the values are the transliterations.
Chris@0:  */
Chris@0: function read_midgard_data() {
Chris@0:   $dir = __DIR__ . '/utf8_to_ascii_db';
Chris@0:   $out = array();
Chris@0: 
Chris@0:   // Read data files.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0:     $UTF8_TO_ASCII = array($bank => array());
Chris@0:     $file = $dir . '/x' . sprintf('%02x', $bank) . '.php';
Chris@0:     if (is_file($file)) {
Chris@0:       include($file);
Chris@0:     }
Chris@0:     $base = $UTF8_TO_ASCII[$bank];
Chris@0: 
Chris@0:     // For unknown characters, these files have '[?]' in them. Replace with
Chris@0:     // NULL for compatibility with our data.
Chris@0:     $base = array_map('_replace_question_with_null', $base);
Chris@0:     $out[$bank] = $base;
Chris@0:   }
Chris@0: 
Chris@0:   return $out;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in the CPAN Text::Unidecode data set.
Chris@0:  *
Chris@0:  * The data is expected to be in files xNN.pm in directory 'Unidecode' under
Chris@0:  * this file's directory. It can be downloaded from
Chris@0:  * http://search.cpan.org/~sburke/Text-Unidecode-0.04/lib/Text/Unidecode.pm.
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Nested array of transliteration data. Outer keys are the first two
Chris@0:  *   bytes of Unicode characters (or 0 for base ASCII characters). The next
Chris@0:  *   level is the other two bytes, and the values are the transliterations.
Chris@0:  */
Chris@0: function read_cpan_data() {
Chris@0:   $dir = __DIR__ . '/Unidecode';
Chris@0:   $out = array();
Chris@0: 
Chris@0:   // Read data files.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0:     $base = array();
Chris@0:     $file = $dir . '/x' . sprintf('%02x', $bank) . '.pm';
Chris@0:     if (is_file($file)) {
Chris@0:       $base = _cpan_read_file($file);
Chris@0:     }
Chris@0:     $out[$bank] = $base;
Chris@0:   }
Chris@0: 
Chris@0:   return $out;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in the data in a single file from the Text::Unidecode CPAN project.
Chris@0:  *
Chris@0:  * @param string $file
Chris@0:  *   File to read from.
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Data read from the file.
Chris@0:  *
Chris@0:  * @see read_cpan_data()
Chris@0:  */
Chris@0: function _cpan_read_file($file) {
Chris@0: 
Chris@0:   $contents = file($file);
Chris@0:   $save = '';
Chris@0:   foreach ($contents as $line) {
Chris@0:     // Discard lines starting with # or $. The first line seems to have a
Chris@0:     // comment starting with #, the second has a Perl line like
Chris@0:     // $Text::Unidecode::Char[0x04] = [, -- and we do not want either.
Chris@0:     if (preg_match('|^\s*[#\$]|', $line)) {
Chris@0:       continue;
Chris@0:     }
Chris@0: 
Chris@0:     // Discard lines ending with semi-colons, which we also don't want
Chris@0:     // (there seem to be two of these lines at the end of the files).
Chris@0:     if (preg_match('|;\s*$|', $line)) {
Chris@0:       continue;
Chris@0:     }
Chris@0: 
Chris@0:     // Replace '[?]' with nothing (that means "don't know how to
Chris@0:     // transliterate"). In some files, this is encoded as qq{[?]} or
Chris@0:     // qq{[?] } instead.
Chris@0:     $line = str_replace('qq{[?]}', 'NULL', $line);
Chris@0:     $line = str_replace('qq{[?] }', 'NULL', $line);
Chris@0:     $line = str_replace("'[?]'", 'NULL', $line);
Chris@0: 
Chris@0:     // Replace qq{} with either "" or '' or nothing, depending on what is
Chris@0:     // inside it.
Chris@0:     $line = str_replace('qq{\{}', "'{'", $line);
Chris@0:     $line = str_replace('qq{\}}', "'}'", $line);
Chris@0:     $line = str_replace('qq{\} }', "'} '", $line);
Chris@0:     $line = str_replace("qq{\\\\}", '"\\\\"', $line);
Chris@0:     $line = str_replace("qq{\\", "qq{'", $line);
Chris@0:     $line = str_replace("qq{\"'}", "\"\\\"'\"", $line);
Chris@0:     $line = preg_replace('|qq\{([^\'\}]+)\}|', "'$1'", $line);
Chris@0:     $line = preg_replace('|qq\{([^\}]+)\}|', '"$1"', $line);
Chris@0: 
Chris@0:     $save .= $line;
Chris@0:   }
Chris@0: 
Chris@0:   // Now we should have a string that looks like:
Chris@0:   // 'a', 'b', ...
Chris@0:   // Evaluate as an array.
Chris@0:   $save = 'return array(' . $save . ');';
Chris@0: 
Chris@0:   $data = @eval($save);
Chris@0:   if (isset($data) && is_array($data)) {
Chris@0:     $data = array_map('_replace_hex_with_character', $data);
Chris@0:   }
Chris@0:   else {
Chris@0:     // There was a problem, so throw an error and exit.
Chris@0:     print "Problem in evaluating $file\n";
Chris@0:     print $save;
Chris@0:     eval($save);
Chris@0:     exit();
Chris@0:   }
Chris@0: 
Chris@0:   // For unknown characters, these files may still have '[?]' in them. Replace
Chris@0:   // with NULL for compatibility with our data.
Chris@0:   $data = array_map('_replace_question_with_null', $data);
Chris@0: 
Chris@0:   return $data;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in the Node.js transliteration data.
Chris@0:  *
Chris@0:  * The data is expected to be in files xNN.yml in directory unidecoder_data
Chris@0:  * under the directory where this file resides. It can be downloaded from
Chris@0:  * https://github.com/bitwalker/stringex/downloads. You also need the PECL
Chris@0:  * 'yaml' extension installed for this function to work.
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Nested array of transliteration data. Outer keys are the first two
Chris@0:  *   bytes of Unicode characters (or 0 for base ASCII characters). The next
Chris@0:  *   level is the other two bytes, and the values are the transliterations.
Chris@0:  */
Chris@0: function read_nodejs_data() {
Chris@0:   $dir = __DIR__ . '/unidecoder_data';
Chris@0:   $out = array();
Chris@0: 
Chris@0:   // Read data files.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0:     $base = array();
Chris@0:     $file = $dir . '/x' . sprintf('%02x', $bank) . '.yml';
Chris@0:     if (is_file($file)) {
Chris@0:       $base = yaml_parse_file($file);
Chris@0:       // For unknown characters, these files have '[?]' in them. Replace with
Chris@0:       // NULL for compatibility with our data.
Chris@0:       $base = array_map('_replace_question_with_null', $base);
Chris@0:     }
Chris@0:     $out[$bank] = $base;
Chris@0:   }
Chris@0: 
Chris@0:   return $out;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Loads the PECL 'intl' Transliterator class's transliteration data.
Chris@0:  *
Chris@0:  * You need to have the PECL 'intl' package installed for this to work.
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Nested array of transliteration data. Outer keys are the first two
Chris@0:  *   bytes of Unicode characters (or 0 for base ASCII characters). The next
Chris@0:  *   level is the other two bytes, and the values are the transliterations.
Chris@0:  */
Chris@0: function read_intl_data() {
Chris@0:   // In order to transliterate, you first have to create a transliterator
Chris@0:   // object. This needs a list of transliteration operations. You can get a
Chris@0:   // list of available operations with:
Chris@0:   //   print_r(Transliterator::listIDs()); exit();
Chris@0:   // And a few of these are documented on
Chris@0:   // http://userguide.icu-project.org/transforms/general and
Chris@0:   // http://www.unicode.org/reports/tr15/ (for normalizations).
Chris@0:   // There are also maps to the Unicode characters at:
Chris@0:   //  http://www.unicode.org/roadmaps/bmp/
Chris@0:   //  http://www.unicode.org/charts/nameslist/
Chris@0:   $ops = '';
Chris@0: 
Chris@0:   // The first step in any transform: separate out accents and remove them.
Chris@0:   $ops .= 'NFD; [:Nonspacing Mark:] Remove; NFC;';
Chris@0: 
Chris@0:   // Then you need to do a bunch of language-specific or script-specific
Chris@0:   // transliterations. Here is hopefully a representative set. There are
Chris@0:   // quite a few scripts that don't appear to have rules currently, such
Chris@0:   // as Etheopian.
Chris@0:   $ops .= 'Greek-Latin; ';
Chris@0:   $ops .= 'Cyrillic-Latin; ';
Chris@0:   $ops .= 'Armenian-Latin; ';
Chris@0:   $ops .= 'Hebrew-Latin; ';
Chris@0:   $ops .= 'Arabic-Latin; ';
Chris@0:   $ops .= 'Syriac-Latin; ';
Chris@0:   $ops .= 'Thaana-Latin; ';
Chris@0:   $ops .= 'Devanagari-Latin; ';
Chris@0:   $ops .= 'Bengali-Latin; ';
Chris@0:   $ops .= 'Gurmukhi-Latin; ';
Chris@0:   $ops .= 'Gujarati-Latin; ';
Chris@0:   $ops .= 'Oriya-Latin; ';
Chris@0:   $ops .= 'Tamil-Latin; ';
Chris@0:   $ops .= 'Telugu-Latin; ';
Chris@0:   $ops .= 'Kannada-Latin; ';
Chris@0:   $ops .= 'Malayalam-Latin; ';
Chris@0:   $ops .= 'Thai-Latin; ';
Chris@0:   $ops .= 'Georgian-Latin; ';
Chris@0:   $ops .= 'Hangul-Latin; ';
Chris@0:   $ops .= 'Mongolian-Latin/BGN; ';
Chris@0:   $ops .= 'Jamo-Latin; ';
Chris@0:   $ops .= 'Katakana-Latin; ';
Chris@0:   $ops .= 'Any-Latin; ';
Chris@0: 
Chris@0:   // Finally, after transforming to Latin, transform to ASCII.
Chris@0:   $ops .= 'Latin-ASCII; ';
Chris@0: 
Chris@0:   // Remove any remaining accents and recompose.
Chris@0:   $ops .= 'NFD; [:Nonspacing Mark:] Remove; NFC;';
Chris@0: 
Chris@0:   $trans = Transliterator::create($ops);
Chris@0:   $out = array();
Chris@0: 
Chris@0:   // Transliterate all possible characters.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0:     $data = array();
Chris@0:     for ($chr = 0; $chr < 256; $chr++) {
Chris@0:       // Skip the UTF-16 and "private use" ranges completely.
Chris@0:       $OK = ($bank <= 0xd8 || $bank > 0xf8);
Chris@0: 
Chris@0:       $result = $OK ? $trans->transliterate(mb_convert_encoding(pack('n', 256 * $bank + $chr), 'UTF-8', 'UTF-16BE')) : '';
Chris@0: 
Chris@0:       // See if we have managed to transliterate this to ASCII or not. If not,
Chris@0:       // return NULL instead of this character.
Chris@0:       $max = chr(127);
Chris@0:       foreach (preg_split('//u', $result, 0, PREG_SPLIT_NO_EMPTY) as $character) {
Chris@0:         if ($character > $max) {
Chris@0:           $OK = $OK && FALSE;
Chris@0:           break;
Chris@0:         }
Chris@0:       }
Chris@0:       $data[$chr] = ($OK) ? $result : NULL;
Chris@0:     }
Chris@0:     $out[$bank] = $data;
Chris@0:   }
Chris@0: 
Chris@0:   return $out;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in the JUnidecode data set.
Chris@0:  *
Chris@0:  * The data is expected to be in files XNN.java in directory 'junidecode' under
Chris@0:  * this file's directory. It can be downloaded from
Chris@0:  * http://www.ippatsuman.com/projects/junidecode/index.html
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Nested array of transliteration data. Outer keys are the first two
Chris@0:  *   bytes of Unicode characters (or 0 for base ASCII characters). The next
Chris@0:  *   level is the other two bytes, and the values are the transliterations.
Chris@0:  */
Chris@0: function read_junidecode_data() {
Chris@0:   $dir = __DIR__ . '/junidecode';
Chris@0:   $out = array();
Chris@0: 
Chris@0:   // Read data files.
Chris@0:   for ($bank = 0; $bank < 256; $bank++) {
Chris@0:     $base = array();
Chris@0:     $file = $dir . '/X' . sprintf('%02x', $bank) . '.java';
Chris@0:     if (is_file($file)) {
Chris@0:       $base = _junidecode_read_file($file);
Chris@0:     }
Chris@0:     $out[$bank] = $base;
Chris@0:   }
Chris@0: 
Chris@0:   return $out;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Reads in the data in a single file from the JUnidecode project.
Chris@0:  *
Chris@0:  * @param string $file
Chris@0:  *   File to read from.
Chris@0:  *
Chris@0:  * @return array
Chris@0:  *   Data read from the file.
Chris@0:  *
Chris@0:  * @see read_junidecode_data()
Chris@0:  */
Chris@0: function _junidecode_read_file($file) {
Chris@0:   $contents = file($file);
Chris@0:   $save = '';
Chris@0:   foreach ($contents as $line) {
Chris@0:     // Discard lines starting with * or / or package or class or public or },
Chris@0:     // to get rid of comments and Java code.
Chris@0:     if (preg_match('|^\s*[\*/\}]|', $line)) {
Chris@0:       continue;
Chris@0:     }
Chris@0:     if (preg_match('/^\s*package|public|class/', $line)) {
Chris@0:       continue;
Chris@0:     }
Chris@0: 
Chris@0:     // Some of the lines look like this:
Chris@0:     //      new String("" + (char) 0x00), // 0x00
Chris@0:     // Transform to be '0x00,'
Chris@0:     $line = preg_replace('|^\s*new\s+String\s*\(\s*""\s*\+\s*\(char\)\s+0x([0-9]+).*$|', '0x$1,', $line);
Chris@0: 
Chris@0:     // Strings are in double quotes, yet many have \' in them.
Chris@0:     $line = str_replace("\'", "'", $line);
Chris@0: 
Chris@0:     // Everything else should probably be OK -- the lines are like:
Chris@0:     //  "Ie", // 0x00
Chris@0:     $save .= $line;
Chris@0:   }
Chris@0: 
Chris@0:   // Evaluate as an array.
Chris@0:   $save = 'return array(' . $save . ');';
Chris@0: 
Chris@0:   $data = @eval($save);
Chris@0:   if (isset($data) && is_array($data)) {
Chris@0:     $data = array_map('_replace_hex_with_character', $data);
Chris@0:     $data = array_map('_replace_question_with_null', $data);
Chris@0:   }
Chris@0:   else {
Chris@0:     // There was a problem, so throw an error and exit.
Chris@0:     print "Problem in evaluating $file\n";
Chris@0:     print $save;
Chris@0:     eval($save);
Chris@0:     exit();
Chris@0:   }
Chris@0: 
Chris@0:   return $data;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Callback for array_map(): Returns $data, with '[?]' replaced with NULL.
Chris@0:  */
Chris@0: function _replace_question_with_null($data) {
Chris@0:   return ($data == '[?]' || $data == '[?] ') ? NULL : $data;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Callback for array_map(): Replaces '\xNN' with the actual character.
Chris@0:  */
Chris@0: function _replace_hex_with_character($item) {
Chris@0:   if (strpos($item, '\x') === 0) {
Chris@0:     $item = eval($item);
Chris@0:   }
Chris@0:   return $item;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Writes a data file out in the standard Drupal Core data format.
Chris@0:  *
Chris@0:  * @param array $data
Chris@0:  *   Array of data to write out.
Chris@0:  * @param string $bank
Chris@0:  *   Bank of characters it belongs to.
Chris@0:  * @param string $dir
Chris@0:  *   Output directory.
Chris@0:  */
Chris@0: function write_data_file($data, $bank, $outdir) {
Chris@0:   $dir = __DIR__ . '/' . $outdir;
Chris@0:   $file = $dir . '/x' . sprintf('%02x', $bank) . '.php';
Chris@0: 
Chris@0:   $out = '';
Chris@0:   $out .= "<?php\n\n/**\n * @file\n * Generic transliteration data for the PhpTransliteration class.\n */\n\n\$base = array(\n";
Chris@0: 
Chris@0:   // The 00 file skips the ASCII range
Chris@0:   $start = 0;
Chris@0:   if ($bank == 0) {
Chris@0:     $start = 0x80;
Chris@0:     $out .= "  // Note: to save memory plain ASCII mappings have been left out.\n";
Chris@0:   }
Chris@0: 
Chris@0:   for ($line = $start; $line <= 0xf0; $line += 0x10) {
Chris@0:     $out .= '  0x' . sprintf('%02X', $line) . ' =>';
Chris@0:     $elems = array_values(array_slice($data, $line, 16));
Chris@0:     for ($i = 0; $i < 16; $i++ ) {
Chris@0:       if (isset($elems[$i])) {
Chris@0:         $out .= " '" . addcslashes($elems[$i], "'\\") . "',";
Chris@0:       }
Chris@0:       else {
Chris@0:         $out .= ' NULL,';
Chris@0:       }
Chris@0:     }
Chris@0:     $out .= "\n";
Chris@0:   }
Chris@0: 
Chris@0:   $out .= ");\n";
Chris@0: 
Chris@0:   file_put_contents($file, $out);
Chris@0: }