danielebarchiesi@4: <?php
danielebarchiesi@4: /**
danielebarchiesi@4:  * ARC2 RDF Store LOAD Query Handler
danielebarchiesi@4:  *
danielebarchiesi@4:  * @author Benjamin Nowack
danielebarchiesi@4:  * @license <http://arc.semsol.org/license>
danielebarchiesi@4:  * @homepage <http://arc.semsol.org/>
danielebarchiesi@4:  * @package ARC2
danielebarchiesi@4: */
danielebarchiesi@4: 
danielebarchiesi@4: ARC2::inc('StoreQueryHandler');
danielebarchiesi@4: 
danielebarchiesi@4: class ARC2_StoreLoadQueryHandler extends ARC2_StoreQueryHandler {
danielebarchiesi@4: 
danielebarchiesi@4:   function __construct($a, &$caller) {/* caller has to be a store */
danielebarchiesi@4:     parent::__construct($a, $caller);
danielebarchiesi@4:   }
danielebarchiesi@4:   
danielebarchiesi@4:   function __init() {/* db_con, store_log_inserts */
danielebarchiesi@4:     parent::__init();
danielebarchiesi@4:     $this->store = $this->caller;
danielebarchiesi@4:     $this->write_buffer_size = $this->v('store_write_buffer', 2500, $this->a);
danielebarchiesi@4:     $this->split_threshold = $this->v('store_split_threshold', 0, $this->a);
danielebarchiesi@4:     $this->strip_mb_comp_str = $this->v('store_strip_mb_comp_str', 0, $this->a);
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4:   /*  */
danielebarchiesi@4:   
danielebarchiesi@4:   function runQuery($infos, $data = '', $keep_bnode_ids = 0) {
danielebarchiesi@4:     $url = $infos['query']['url'];
danielebarchiesi@4:     $graph = $infos['query']['target_graph'];
danielebarchiesi@4:     $this->target_graph = $graph ? $this->calcURI($graph) : $this->calcURI($url);
danielebarchiesi@4:     $this->fixed_target_graph = $graph ? $this->target_graph : '';
danielebarchiesi@4:     $this->keep_bnode_ids = $keep_bnode_ids;
danielebarchiesi@4:     /* reader */
danielebarchiesi@4:     ARC2::inc('Reader');
danielebarchiesi@4:     $reader = new ARC2_Reader($this->a, $this);
danielebarchiesi@4:     $reader->activate($url, $data);
danielebarchiesi@4:     /* format detection */
danielebarchiesi@4:     $mappings = array(
danielebarchiesi@4:       'rdfxml' => 'RDFXML', 
danielebarchiesi@4:       'sparqlxml' => 'SPOG', 
danielebarchiesi@4:       'turtle' => 'Turtle', 
danielebarchiesi@4:       'ntriples' => 'Turtle', 
danielebarchiesi@4:       'rss' => 'RSS',
danielebarchiesi@4:       'atom' => 'Atom',
danielebarchiesi@4:       'n3' => 'Turtle', 
danielebarchiesi@4:       'html' => 'SemHTML',
danielebarchiesi@4:       'sgajson' => 'SGAJSON',
danielebarchiesi@4:       'cbjson' => 'CBJSON'
danielebarchiesi@4:     );
danielebarchiesi@4:     $format = $reader->getFormat();
danielebarchiesi@4:     if (!$format || !isset($mappings[$format])) {
danielebarchiesi@4:       return $this->addError('No loader available for "' .$url. '": ' . $format);
danielebarchiesi@4:     }
danielebarchiesi@4:     /* format loader */
danielebarchiesi@4:     $suffix = 'Store' . $mappings[$format] . 'Loader';
danielebarchiesi@4:     ARC2::inc($suffix);
danielebarchiesi@4:     $cls = 'ARC2_' . $suffix;
danielebarchiesi@4:     $loader = new $cls($this->a, $this);
danielebarchiesi@4:     $loader->setReader($reader);
danielebarchiesi@4:     /* lock */
danielebarchiesi@4:     if (!$this->store->getLock()) {
danielebarchiesi@4:       $l_name = $this->a['db_name'] . '.' . $this->store->getTablePrefix() . '.write_lock';
danielebarchiesi@4:       return $this->addError('Could not get lock in "runQuery" (' . $l_name . ')');
danielebarchiesi@4:     }
danielebarchiesi@4:     $this->has_lock = 1;
danielebarchiesi@4:     /* logging */
danielebarchiesi@4:     $this->t_count = 0;
danielebarchiesi@4:     $this->t_start = ARC2::mtime();
danielebarchiesi@4:     $this->log_inserts = $this->v('store_log_inserts', 0, $this->a);
danielebarchiesi@4:     if ($this->log_inserts) {
danielebarchiesi@4:       @unlink("arc_insert_log.txt");
danielebarchiesi@4:       $this->inserts = array();
danielebarchiesi@4:       $this->insert_times = array();
danielebarchiesi@4:       $this->t_prev = $this->t_start;
danielebarchiesi@4:       $this->t_count_prev = 0 ;
danielebarchiesi@4:     }
danielebarchiesi@4:     /* load and parse */
danielebarchiesi@4:     $this->max_term_id = $this->getMaxTermID();
danielebarchiesi@4:     $this->max_triple_id = $this->getMaxTripleID();
danielebarchiesi@4:     $this->column_type = $this->store->getColumnType();
danielebarchiesi@4:     //$this->createMergeTable();
danielebarchiesi@4:     $this->term_ids = array();
danielebarchiesi@4:     $this->triple_ids = array();
danielebarchiesi@4:     $this->sql_buffers = array();
danielebarchiesi@4:     $r = $loader->parse($url, $data);
danielebarchiesi@4:     /* done */
danielebarchiesi@4:     $this->checkSQLBuffers(1);
danielebarchiesi@4:     if ($this->log_inserts) {
danielebarchiesi@4:       $this->logInserts();
danielebarchiesi@4:     }
danielebarchiesi@4:     $this->store->releaseLock();
danielebarchiesi@4:     //$this->dropMergeTable();
danielebarchiesi@4:     if ((rand(1, 100) == 1)) $this->store->optimizeTables();
danielebarchiesi@4:     $t2 = ARC2::mtime();
danielebarchiesi@4:     $dur = round($t2 - $this->t_start, 4);
danielebarchiesi@4:     $r = array(
danielebarchiesi@4:       't_count' => $this->t_count,
danielebarchiesi@4:       'load_time' => $dur,
danielebarchiesi@4:     );
danielebarchiesi@4:     if ($this->log_inserts) {
danielebarchiesi@4:       $r['inserts'] = $this->inserts;
danielebarchiesi@4:       $r['insert_times'] = $this->insert_times;
danielebarchiesi@4:     }
danielebarchiesi@4:     return $r;
danielebarchiesi@4:   }
danielebarchiesi@4:   
danielebarchiesi@4:   /*  */
danielebarchiesi@4: 
danielebarchiesi@4:   function addT($s, $p, $o, $s_type, $o_type, $o_dt = '', $o_lang = '') {
danielebarchiesi@4:     if (!$this->has_lock) return 0;
danielebarchiesi@4:     $type_ids = array ('uri' => '0', 'bnode' => '1' , 'literal' => '2');
danielebarchiesi@4:     $g = $this->getStoredTermID($this->target_graph, '0', 'id');
danielebarchiesi@4:     $s = (($s_type == 'bnode') && !$this->keep_bnode_ids) ? '_:b' . abs(crc32($g . $s)) . '_' . (strlen($s) > 12 ? substr(substr($s, 2) , -10) : substr($s, 2)) : $s;
danielebarchiesi@4:     $o = (($o_type == 'bnode') && !$this->keep_bnode_ids) ? '_:b' . abs(crc32($g . $o)) . '_' . (strlen($o) > 12 ? substr(substr($o, 2), -10) : substr($o, 2)) : $o;
danielebarchiesi@4:     /* triple */
danielebarchiesi@4:     $t = array(
danielebarchiesi@4:       's' => $this->getStoredTermID($s, $type_ids[$s_type], 's'),
danielebarchiesi@4:       'p' => $this->getStoredTermID($p, '0', 'id'),
danielebarchiesi@4:       'o' => $this->getStoredTermID($o, $type_ids[$o_type], 'o'),
danielebarchiesi@4:       'o_lang_dt' => $this->getStoredTermID($o_dt . $o_lang, $o_dt ? '0' : '2', 'id'),
danielebarchiesi@4:       'o_comp' => $this->getOComp($o),
danielebarchiesi@4:       's_type' => $type_ids[$s_type], 
danielebarchiesi@4:       'o_type' => $type_ids[$o_type],
danielebarchiesi@4:     );
danielebarchiesi@4:     $t['t'] = $this->getTripleID($t);
danielebarchiesi@4:     if (is_array($t['t'])) {/* t exists already */
danielebarchiesi@4:       $t['t'] = $t['t'][0];
danielebarchiesi@4:     }
danielebarchiesi@4:     else {
danielebarchiesi@4:       $this->bufferTripleSQL($t);
danielebarchiesi@4:     }
danielebarchiesi@4:     /* g2t */
danielebarchiesi@4:     $g2t = array('g' => $g, 't' => $t['t']);
danielebarchiesi@4:     $this->bufferGraphSQL($g2t);
danielebarchiesi@4:     $this->t_count++;
danielebarchiesi@4:     /* check buffers */
danielebarchiesi@4:     if (($this->t_count % $this->write_buffer_size) == 0) {
danielebarchiesi@4:       $force_write = 1;
danielebarchiesi@4:       $reset_buffers = (($this->t_count % ($this->write_buffer_size * 2)) == 0);
danielebarchiesi@4:       $refresh_lock = (($this->t_count % 25000) == 0);
danielebarchiesi@4:       $split_tables = (($this->t_count % ($this->write_buffer_size * 10)) == 0);
danielebarchiesi@4:       if ($this->log_inserts) $this->logInserts();
danielebarchiesi@4:       $this->checkSQLBuffers($force_write, $reset_buffers, $refresh_lock, $split_tables);
danielebarchiesi@4:     }
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4:   /*  */
danielebarchiesi@4:   
danielebarchiesi@4:   function getMaxTermID() {
danielebarchiesi@4:     $con = $this->store->getDBCon();
danielebarchiesi@4:     $sql = '';
danielebarchiesi@4:     foreach (array('id2val', 's2val', 'o2val') as $tbl) {
danielebarchiesi@4:       $sql .= $sql ? ' UNION ' : '';
danielebarchiesi@4:       $sql .= "(SELECT MAX(id) as `id` FROM " . $this->store->getTablePrefix() . $tbl . ')';
danielebarchiesi@4:     }
danielebarchiesi@4:     $r = 0;
danielebarchiesi@4:     if (($rs = $this->queryDB($sql, $con)) && mysql_num_rows($rs)) {
danielebarchiesi@4:       while ($row = mysql_fetch_array($rs)) {
danielebarchiesi@4:         $r = ($r < $row['id']) ? $row['id'] : $r;
danielebarchiesi@4:       }
danielebarchiesi@4:     }
danielebarchiesi@4:     return $r + 1;
danielebarchiesi@4:   }
danielebarchiesi@4:   
danielebarchiesi@4:   function getMaxTripleID() {
danielebarchiesi@4:     $con = $this->store->getDBCon();
danielebarchiesi@4:     $sql = "SELECT MAX(t) AS `id` FROM " . $this->store->getTablePrefix() . "triple";
danielebarchiesi@4:     if (($rs = $this->queryDB($sql, $con)) && mysql_num_rows($rs) && ($row = mysql_fetch_array($rs))) {
danielebarchiesi@4:       return $row['id'] + 1;
danielebarchiesi@4:     }
danielebarchiesi@4:     return 1;
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4:   function getStoredTermID($val, $type_id, $tbl) {
danielebarchiesi@4:     $con = $this->store->getDBCon();
danielebarchiesi@4:     /* buffered */
danielebarchiesi@4:     if (isset($this->term_ids[$val])) {
danielebarchiesi@4:       if (!isset($this->term_ids[$val][$tbl])) {
danielebarchiesi@4:         foreach (array('id', 's', 'o') as $other_tbl) {
danielebarchiesi@4:           if (isset($this->term_ids[$val][$other_tbl])) {
danielebarchiesi@4:             $this->term_ids[$val][$tbl] = $this->term_ids[$val][$other_tbl];
danielebarchiesi@4:             $this->bufferIDSQL($tbl, $this->term_ids[$val][$tbl], $val, $type_id);
danielebarchiesi@4:             break;
danielebarchiesi@4:           }
danielebarchiesi@4:         }
danielebarchiesi@4:       }
danielebarchiesi@4:       return $this->term_ids[$val][$tbl];
danielebarchiesi@4:     }
danielebarchiesi@4:     /* db */
danielebarchiesi@4:     $tbl_prefix = $this->store->getTablePrefix();
danielebarchiesi@4:     $sub_tbls = ($tbl == 'id') ? array('id2val', 's2val', 'o2val') : ($tbl == 's' ? array('s2val', 'id2val', 'o2val') : array('o2val', 'id2val', 's2val'));
danielebarchiesi@4:     foreach ($sub_tbls as $sub_tbl) {
danielebarchiesi@4:       $id = 0;
danielebarchiesi@4:       //$sql = "SELECT id AS `id`, '" . $sub_tbl . "' AS `tbl` FROM " . $tbl_prefix . $sub_tbl . " WHERE val = BINARY '" . mysql_real_escape_string($val, $con) . "'";
danielebarchiesi@4:       /* via hash */
danielebarchiesi@4:       if (preg_match('/^(s2val|o2val)$/', $sub_tbl) && $this->hasHashColumn($sub_tbl)) {
danielebarchiesi@4:         $sql = "SELECT id AS `id`, val AS `val` FROM " . $tbl_prefix . $sub_tbl . " WHERE val_hash = BINARY '" . $this->getValueHash($val) . "'";
danielebarchiesi@4:         if (($rs = $this->queryDB($sql, $con)) && mysql_num_rows($rs)) {
danielebarchiesi@4:           while ($row = mysql_fetch_array($rs)) {
danielebarchiesi@4:             if ($row['val'] == $val) {
danielebarchiesi@4:               $id = $row['id'];
danielebarchiesi@4:               break;
danielebarchiesi@4:             }
danielebarchiesi@4:           }
danielebarchiesi@4:         }
danielebarchiesi@4:       }
danielebarchiesi@4:       else {
danielebarchiesi@4:         $sql = "SELECT id AS `id` FROM " . $tbl_prefix . $sub_tbl . " WHERE val = BINARY '" . mysql_real_escape_string($val, $con) . "'";
danielebarchiesi@4:         if (($rs = $this->queryDB($sql . ' LIMIT 1', $con)) && mysql_num_rows($rs)) {
danielebarchiesi@4:           $row = mysql_fetch_array($rs);
danielebarchiesi@4:           $id = $row['id'];
danielebarchiesi@4:         }
danielebarchiesi@4:       }
danielebarchiesi@4:       if ($id) {
danielebarchiesi@4:         $this->term_ids[$val] = array($tbl => $id);
danielebarchiesi@4:         if ($sub_tbl != $tbl . '2val') {
danielebarchiesi@4:           $this->bufferIDSQL($tbl, $id, $val, $type_id);
danielebarchiesi@4:         }
danielebarchiesi@4:         break;
danielebarchiesi@4:       }
danielebarchiesi@4:     }
danielebarchiesi@4:     /* new */
danielebarchiesi@4:     if (!isset($this->term_ids[$val])) {
danielebarchiesi@4:       $this->term_ids[$val] = array($tbl => $this->max_term_id);
danielebarchiesi@4:       $this->bufferIDSQL($tbl, $this->max_term_id, $val, $type_id);
danielebarchiesi@4:       $this->max_term_id++;
danielebarchiesi@4:       /* upgrade tables ? */
danielebarchiesi@4:       if (($this->column_type == 'mediumint') && ($this->max_term_id >= 16750000)) {
danielebarchiesi@4:         $this->store->extendColumns();
danielebarchiesi@4:         $this->column_type = 'int';
danielebarchiesi@4:       }
danielebarchiesi@4:     }
danielebarchiesi@4:     return $this->term_ids[$val][$tbl];
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4:   function getTripleID($t) {
danielebarchiesi@4:     $con = $this->store->getDBCon();
danielebarchiesi@4:     $val = serialize($t);
danielebarchiesi@4:     /* buffered */
danielebarchiesi@4:     if (isset($this->triple_ids[$val])) {
danielebarchiesi@4:       return array($this->triple_ids[$val]);/* hack for "don't insert this triple" */
danielebarchiesi@4:     }
danielebarchiesi@4:     /* db */
danielebarchiesi@4:     $sql = "SELECT t FROM " . $this->store->getTablePrefix() . "triple WHERE 
danielebarchiesi@4:       s = " . $t['s'] . " AND p = " . $t['p'] . " AND o = " . $t['o'] . " AND o_lang_dt = " . $t['o_lang_dt'] . " AND s_type = " . $t['s_type'] . " AND o_type = " . $t['o_type'] . "
danielebarchiesi@4:       LIMIT 1
danielebarchiesi@4:     ";
danielebarchiesi@4:     if (($rs = $this->queryDB($sql, $con)) && mysql_num_rows($rs) && ($row = mysql_fetch_array($rs))) {
danielebarchiesi@4:       $this->triple_ids[$val] = $row['t'];/* hack for "don't insert this triple" */
danielebarchiesi@4:       return array($row['t']);/* hack for "don't insert this triple" */
danielebarchiesi@4:     }
danielebarchiesi@4:     /* new */
danielebarchiesi@4:     else {
danielebarchiesi@4:       $this->triple_ids[$val] = $this->max_triple_id;
danielebarchiesi@4:       $this->max_triple_id++;
danielebarchiesi@4:       /* split tables ? */
danielebarchiesi@4:       if (0 && $this->split_threshold && !($this->max_triple_id % $this->split_threshold)) {
danielebarchiesi@4:         $this->store->splitTables();
danielebarchiesi@4:         $this->dropMergeTable();
danielebarchiesi@4:         $this->createMergeTable();
danielebarchiesi@4:       }
danielebarchiesi@4:       /* upgrade tables ? // Thanks to patch by Mark Fichtner (https://github.com/Knurg) */
danielebarchiesi@4:       if (($this->column_type == 'mediumint') && ($this->max_triple_id >= 16750000)) {
danielebarchiesi@4:         $this->store->extendColumns();
danielebarchiesi@4:         $this->column_type = 'int';
danielebarchiesi@4:       }
danielebarchiesi@4:       return $this->triple_ids[$val];
danielebarchiesi@4:     }
danielebarchiesi@4:   }
danielebarchiesi@4:   
danielebarchiesi@4:   function getOComp($val) {
danielebarchiesi@4:     /* try date (e.g. 21 August 2007) */
danielebarchiesi@4:     if (preg_match('/^[0-9]{1,2}\s+[a-z]+\s+[0-9]{4}/i', $val) && ($uts = strtotime($val)) && ($uts !== -1)) {
danielebarchiesi@4:       return date("Y-m-d\TH:i:s", $uts);
danielebarchiesi@4:     }
danielebarchiesi@4:     /* xsd date (e.g. 2009-05-28T18:03:38+09:00 2009-05-28T18:03:38GMT) */
danielebarchiesi@4:     if (preg_match('/^([0-9]{4}\-[0-9]{2}\-[0-9]{2}\T)([0-9\:]+)?([0-9\+\-\:\Z]+)?(\s*[a-z]{2,3})?$/si', $val, $m)) {
danielebarchiesi@4:       /* yyyy-mm-dd */
danielebarchiesi@4:       $val = $m[1];
danielebarchiesi@4:       /* hh:ss */
danielebarchiesi@4:       if ($m[2]) {
danielebarchiesi@4:         $val .= $m[2];
danielebarchiesi@4:         /* timezone offset */
danielebarchiesi@4:         if (isset($m[3]) && ($m[3] != 'Z')) {
danielebarchiesi@4:           $uts = strtotime(str_replace('T', ' ', $val));
danielebarchiesi@4:           if (preg_match('/([\+\-])([0-9]{2})\:?([0-9]{2})$/', $m[3], $sub_m)) {
danielebarchiesi@4:             $diff_mins = (3600 * ltrim($sub_m[2], '0')) + ltrim($sub_m[3], '0');
danielebarchiesi@4:             $uts = ($sub_m[1] == '-') ? $uts + $diff_mins : $uts - $diff_mins;
danielebarchiesi@4:             $val = date('Y-m-d\TH:i:s\Z', $uts);
danielebarchiesi@4:           }
danielebarchiesi@4:         }
danielebarchiesi@4:         else {
danielebarchiesi@4:           $val .= 'Z';
danielebarchiesi@4:         }
danielebarchiesi@4:       }
danielebarchiesi@4:       return $val;
danielebarchiesi@4:     }
danielebarchiesi@4:     /* fallback & backup w/o UTC calculation, to be removed in later revision */
danielebarchiesi@4:     if (preg_match('/^[0-9]{4}[0-9\-\:\T\Z\+]+([a-z]{2,3})?$/i', $val)) {
danielebarchiesi@4:       return $val;
danielebarchiesi@4:     }
danielebarchiesi@4:     if (is_numeric($val)) {
danielebarchiesi@4:       $val = sprintf("%f", $val);
danielebarchiesi@4:       if (preg_match("/([\-\+])([0-9]*)\.([0-9]*)/", $val, $m)) {
danielebarchiesi@4:         return $m[1] . sprintf("%018s", $m[2]) . "." . sprintf("%-015s", $m[3]);
danielebarchiesi@4:       }
danielebarchiesi@4:       if (preg_match("/([0-9]*)\.([0-9]*)/", $val, $m)) {
danielebarchiesi@4:         return "+" . sprintf("%018s", $m[1]) . "." . sprintf("%-015s", $m[2]);
danielebarchiesi@4:       }
danielebarchiesi@4:       return $val;
danielebarchiesi@4:     }
danielebarchiesi@4:     /* any other string: remove tags, linebreaks etc., but keep MB-chars  */
danielebarchiesi@4:     //$val = substr(trim(preg_replace('/[\W\s]+/is', '-', strip_tags($val))), 0, 35);
danielebarchiesi@4:     // [\PL\s]+ ( = non-Letters) kills digits
danielebarchiesi@4:     $re = $this->has_pcre_unicode ? '/[\PL\s]+/isu' : '/[\s\'\"\´\`]+/is';
danielebarchiesi@4:     $re = '/[\s\'\"\´\`]+/is';
danielebarchiesi@4:     $val = trim(preg_replace($re, '-', strip_tags($val)));
danielebarchiesi@4:     if (strlen($val) > 35) {
danielebarchiesi@4:       $fnc = function_exists("mb_substr") ? 'mb_substr' : 'substr';
danielebarchiesi@4:       $val = $fnc($val, 0, 17) . '-' . $fnc($val, -17);
danielebarchiesi@4:     }
danielebarchiesi@4:     if ($this->strip_mb_comp_str) {
danielebarchiesi@4:       $val = urldecode(preg_replace('/\%[0-9A-F]{2}/', '', urlencode($val)));
danielebarchiesi@4:     }
danielebarchiesi@4:     return $this->toUTF8($val);
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4:   /*  */
danielebarchiesi@4:   
danielebarchiesi@4:   function bufferTripleSQL($t) {
danielebarchiesi@4:     $con = $this->store->getDBCon();
danielebarchiesi@4:     $tbl = 'triple';
danielebarchiesi@4:     $sql = ", ";
danielebarchiesi@4:     if (!isset($this->sql_buffers[$tbl])) {
danielebarchiesi@4:       $this->sql_buffers[$tbl] = "INSERT IGNORE INTO " . $this->store->getTablePrefix() . $tbl . " (t, s, p, o, o_lang_dt, o_comp, s_type, o_type) VALUES";
danielebarchiesi@4:       $sql = " ";
danielebarchiesi@4:     }
danielebarchiesi@4:     $this->sql_buffers[$tbl] .= $sql . "(" . $t['t'] . ", " . $t['s'] . ", " . $t['p'] . ", " . $t['o'] . ", " . $t['o_lang_dt'] . ", '" . mysql_real_escape_string($t['o_comp'], $con) . "', " . $t['s_type'] . ", " . $t['o_type'] . ")";
danielebarchiesi@4:   }
danielebarchiesi@4:   
danielebarchiesi@4:   function bufferGraphSQL($g2t) {
danielebarchiesi@4:     $tbl = 'g2t';
danielebarchiesi@4:     $sql = ", ";
danielebarchiesi@4:     if (!isset($this->sql_buffers[$tbl])) {
danielebarchiesi@4:       $this->sql_buffers[$tbl] = "INSERT IGNORE INTO " . $this->store->getTablePrefix() . $tbl . " (g, t) VALUES";
danielebarchiesi@4:       $sql = " ";
danielebarchiesi@4:     }
danielebarchiesi@4:     $this->sql_buffers[$tbl] .= $sql . "(" . $g2t['g'] . ", " . $g2t['t'] . ")";
danielebarchiesi@4:   }
danielebarchiesi@4:   
danielebarchiesi@4:   function bufferIDSQL($tbl, $id, $val, $val_type) {
danielebarchiesi@4:     $con = $this->store->getDBCon();
danielebarchiesi@4:     $tbl = $tbl . '2val';
danielebarchiesi@4:     if ($tbl == 'id2val') {
danielebarchiesi@4:       $cols = "id, val, val_type";
danielebarchiesi@4:       $vals = "(" . $id . ", '" . mysql_real_escape_string($val, $con) . "', " . $val_type . ")";
danielebarchiesi@4:     }
danielebarchiesi@4:     elseif (preg_match('/^(s2val|o2val)$/', $tbl) && $this->hasHashColumn($tbl)) {
danielebarchiesi@4:       $cols = "id, val_hash, val";
danielebarchiesi@4:       $vals = "(" . $id . ", '" . $this->getValueHash($val). "', '" . mysql_real_escape_string($val, $con) . "')";
danielebarchiesi@4:     }
danielebarchiesi@4:     else {
danielebarchiesi@4:       $cols = "id, val";
danielebarchiesi@4:       $vals = "(" . $id . ", '" . mysql_real_escape_string($val, $con) . "')";
danielebarchiesi@4:     }
danielebarchiesi@4:     if (!isset($this->sql_buffers[$tbl])) {
danielebarchiesi@4:       $this->sql_buffers[$tbl] = '';
danielebarchiesi@4:       $sql = "INSERT IGNORE INTO " . $this->store->getTablePrefix() . $tbl . "(" . $cols . ") VALUES ";
danielebarchiesi@4:     }
danielebarchiesi@4:     else {
danielebarchiesi@4:       $sql = ", ";
danielebarchiesi@4:     }
danielebarchiesi@4:     $sql .= $vals;
danielebarchiesi@4:     $this->sql_buffers[$tbl] .= $sql;
danielebarchiesi@4:   }
danielebarchiesi@4:   
danielebarchiesi@4:   /*  */
danielebarchiesi@4: 
danielebarchiesi@4:   function checkSQLBuffers($force_write = 0, $reset_id_buffers = 0, $refresh_lock = 0, $split_tables = 0) {
danielebarchiesi@4:     $con = $this->store->getDBCon();
danielebarchiesi@4:     if (!$this->keep_time_limit) @set_time_limit($this->v('time_limit', 60, $this->a));
danielebarchiesi@4:     foreach (array('triple', 'g2t', 'id2val', 's2val', 'o2val') as $tbl) {
danielebarchiesi@4:       $buffer_size = isset($this->sql_buffers[$tbl]) ? 1 : 0;
danielebarchiesi@4:       if ($buffer_size && $force_write) {
danielebarchiesi@4:         $t1 = ARC2::mtime();
danielebarchiesi@4:         $this->queryDB($this->sql_buffers[$tbl], $con);
danielebarchiesi@4:         /* table error */
danielebarchiesi@4:         if ($er = mysql_error($con)) {
danielebarchiesi@4:           $this->autoRepairTable($er, $con, $this->sql_buffers[$tbl]);
danielebarchiesi@4:         }
danielebarchiesi@4:         unset($this->sql_buffers[$tbl]);
danielebarchiesi@4:         if ($this->log_inserts) {
danielebarchiesi@4:           $t2 = ARC2::mtime();
danielebarchiesi@4:           $this->inserts[$tbl] = $this->v($tbl, 0, $this->inserts) + max(0, mysql_affected_rows($con));
danielebarchiesi@4:           $dur = round($t2 - $t1, 4);
danielebarchiesi@4:           $this->insert_times[$tbl] = isset($this->insert_times[$tbl]) ? $this->insert_times[$tbl] : array('min' => $dur, 'max' => $dur, 'sum' => $dur);
danielebarchiesi@4:           $this->insert_times[$tbl] = array('min' => min($dur, $this->insert_times[$tbl]['min']), 'max' => max($dur, $this->insert_times[$tbl]['max']), 'sum' => $dur + $this->insert_times[$tbl]['sum']);
danielebarchiesi@4:         }
danielebarchiesi@4:         /* reset term id buffers */
danielebarchiesi@4:         if ($reset_id_buffers) {
danielebarchiesi@4:           $this->term_ids = array();
danielebarchiesi@4:           $this->triple_ids = array();
danielebarchiesi@4:         }
danielebarchiesi@4:         /* refresh lock */
danielebarchiesi@4:         if ($refresh_lock) {
danielebarchiesi@4:           $this->store->releaseLock();
danielebarchiesi@4:           $this->has_lock = 0;
danielebarchiesi@4:           sleep(1);
danielebarchiesi@4:           if (!$this->store->getLock(5)) return $this->addError('Could not re-obtain lock in "checkSQLBuffers"');
danielebarchiesi@4:           $this->has_lock = 1;
danielebarchiesi@4:         }
danielebarchiesi@4:       }
danielebarchiesi@4:     }
danielebarchiesi@4:     return 1;
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4:   function autoRepairTable($er, $con, $sql = '') {
danielebarchiesi@4:     $this->addError('MySQL error: ' . $er . ' (' . $sql . ')');
danielebarchiesi@4:     if (preg_match('/Table \'[^\']+\/([a-z0-9\_\-]+)\' .*(crashed|repair)/i', $er, $m)) {
danielebarchiesi@4:       $rs = $this->queryDB('REPAIR TABLE ' . rawurlencode($m[1]), $con);
danielebarchiesi@4:       $msg = $rs ? mysql_fetch_array($rs) : array();
danielebarchiesi@4:       if ($this->v('Msg_type', 'error', $msg) == 'error') {
danielebarchiesi@4:         /* auto-reset */
danielebarchiesi@4:         if ($this->v('store_reset_on_table_crash', 0, $this->a)) {
danielebarchiesi@4:           $this->store->drop();
danielebarchiesi@4:           $this->store->setUp();
danielebarchiesi@4:         }
danielebarchiesi@4:         else {
danielebarchiesi@4:           $er = $this->v('Msg_text', 'unknown error', $msg);
danielebarchiesi@4:           $this->addError('Auto-repair failed on ' . rawurlencode($m[1]) . ': ' . $er);
danielebarchiesi@4:         }
danielebarchiesi@4:         //die("Fatal errors: \n" . print_r($this->getErrors(), 1));
danielebarchiesi@4:       }
danielebarchiesi@4:     }
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4:   /* speed log */
danielebarchiesi@4:   
danielebarchiesi@4:   function logInserts() {
danielebarchiesi@4:     $t_start = $this->t_start;
danielebarchiesi@4:     $t_prev = $this->t_prev;
danielebarchiesi@4:     $t_now = ARC2::mtime();
danielebarchiesi@4:     $tc_prev = $this->t_count_prev;
danielebarchiesi@4:     $tc_now = $this->t_count;
danielebarchiesi@4:     $tc_diff = $tc_now - $tc_prev;
danielebarchiesi@4:     
danielebarchiesi@4:     $dur_full = $t_now - $t_start;
danielebarchiesi@4:     $dur_diff = $t_now - $t_prev;
danielebarchiesi@4: 
danielebarchiesi@4:     $speed_full = round($tc_now / $dur_full);
danielebarchiesi@4:     $speed_now = round($tc_diff / $dur_diff);
danielebarchiesi@4: 
danielebarchiesi@4:     $r = $tc_diff . ' in ' . round($dur_diff, 5) . ' = ' . $speed_now . ' t/s  (' .$tc_now. ' in ' . round($dur_full, 5). ' = ' . $speed_full . ' t/s )'; 
danielebarchiesi@4:     $fp = @fopen("arc_insert_log.txt", "a");
danielebarchiesi@4:     @fwrite($fp, $r . "\r\n");
danielebarchiesi@4:     @fclose($fp);
danielebarchiesi@4:     
danielebarchiesi@4:     $this->t_prev = $t_now;
danielebarchiesi@4:     $this->t_count_prev = $tc_now;
danielebarchiesi@4:   }
danielebarchiesi@4: 
danielebarchiesi@4: }