Mercurial > hg > vamp-website

annotate forum/Sources/ManageSearchEngines.php @ 85:6d7b61434be7 website

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add a copy of this here, just in case!
author Chris Cannam
date Mon, 20 Jan 2014 11:02:36 +0000
parents e3e11437ecea
children
rev   line source
Chris@76 1 <?php
Chris@76 2
Chris@76 3 /**
Chris@76 4 * Simple Machines Forum (SMF)
Chris@76 5 *
Chris@76 6 * @package SMF
Chris@76 7 * @author Simple Machines http://www.simplemachines.org
Chris@76 8 * @copyright 2011 Simple Machines
Chris@76 9 * @license http://www.simplemachines.org/about/smf/license.php BSD
Chris@76 10 *
Chris@76 11 * @version 2.0
Chris@76 12 */
Chris@76 13
Chris@76 14 if (!defined('SMF'))
Chris@76 15 die('Hacking attempt...');
Chris@76 16
Chris@76 17 /* This file contains all the screens that relate to search engines.
Chris@76 18
Chris@76 19 // !!!
Chris@76 20 */
Chris@76 21
Chris@76 22 // Entry point for this section.
Chris@76 23 function SearchEngines()
Chris@76 24 {
Chris@76 25 global $context, $txt, $scripturl;
Chris@76 26
Chris@76 27 isAllowedTo('admin_forum');
Chris@76 28
Chris@76 29 loadLanguage('Search');
Chris@76 30 loadTemplate('ManageSearch');
Chris@76 31
Chris@76 32 $subActions = array(
Chris@76 33 'editspiders' => 'EditSpider',
Chris@76 34 'logs' => 'SpiderLogs',
Chris@76 35 'settings' => 'ManageSearchEngineSettings',
Chris@76 36 'spiders' => 'ViewSpiders',
Chris@76 37 'stats' => 'SpiderStats',
Chris@76 38 );
Chris@76 39
Chris@76 40 // Ensure we have a valid subaction.
Chris@76 41 $context['sub_action'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : 'stats';
Chris@76 42
Chris@76 43 $context['page_title'] = $txt['search_engines'];
Chris@76 44
Chris@76 45 // Some more tab data.
Chris@76 46 $context[$context['admin_menu_name']]['tab_data'] = array(
Chris@76 47 'title' => $txt['search_engines'],
Chris@76 48 'description' => $txt['search_engines_description'],
Chris@76 49 );
Chris@76 50
Chris@76 51 // Call the function!
Chris@76 52 $subActions[$context['sub_action']]();
Chris@76 53 }
Chris@76 54
Chris@76 55 // This is really just the settings page.
Chris@76 56 function ManageSearchEngineSettings($return_config = false)
Chris@76 57 {
Chris@76 58 global $context, $txt, $modSettings, $scripturl, $sourcedir, $smcFunc;
Chris@76 59
Chris@76 60 $config_vars = array(
Chris@76 61 // How much detail?
Chris@76 62 array('select', 'spider_mode', array($txt['spider_mode_off'], $txt['spider_mode_standard'], $txt['spider_mode_high'], $txt['spider_mode_vhigh']), 'onchange' => 'disableFields();'),
Chris@76 63 'spider_group' => array('select', 'spider_group', array($txt['spider_group_none'], $txt['membergroups_members'])),
Chris@76 64 array('select', 'show_spider_online', array($txt['show_spider_online_no'], $txt['show_spider_online_summary'], $txt['show_spider_online_detail'], $txt['show_spider_online_detail_admin'])),
Chris@76 65 );
Chris@76 66
Chris@76 67 // Set up a message.
Chris@76 68 $context['settings_message'] = '<span class="smalltext">' . sprintf($txt['spider_settings_desc'], $scripturl . '?action=admin;area=logs;sa=pruning;' . $context['session_var'] . '=' . $context['session_id']) . '</span>';
Chris@76 69
Chris@76 70 // Do some javascript.
Chris@76 71 $javascript_function = '
Chris@76 72 function disableFields()
Chris@76 73 {
Chris@76 74 disabledState = document.getElementById(\'spider_mode\').value == 0;';
Chris@76 75
Chris@76 76 foreach ($config_vars as $variable)
Chris@76 77 if ($variable[1] != 'spider_mode')
Chris@76 78 $javascript_function .= '
Chris@76 79 if (document.getElementById(\'' . $variable[1] . '\'))
Chris@76 80 document.getElementById(\'' . $variable[1] . '\').disabled = disabledState;';
Chris@76 81
Chris@76 82 $javascript_function .= '
Chris@76 83 }
Chris@76 84 disableFields();';
Chris@76 85
Chris@76 86 if ($return_config)
Chris@76 87 return $config_vars;
Chris@76 88
Chris@76 89 // We need to load the groups for the spider group thingy.
Chris@76 90 $request = $smcFunc['db_query']('', '
Chris@76 91 SELECT id_group, group_name
Chris@76 92 FROM {db_prefix}membergroups
Chris@76 93 WHERE id_group != {int:admin_group}
Chris@76 94 AND id_group != {int:moderator_group}',
Chris@76 95 array(
Chris@76 96 'admin_group' => 1,
Chris@76 97 'moderator_group' => 3,
Chris@76 98 )
Chris@76 99 );
Chris@76 100 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 101 $config_vars['spider_group'][2][$row['id_group']] = $row['group_name'];
Chris@76 102 $smcFunc['db_free_result']($request);
Chris@76 103
Chris@76 104 // Make sure it's valid - note that regular members are given id_group = 1 which is reversed in Load.php - no admins here!
Chris@76 105 if (isset($_POST['spider_group']) && !isset($config_vars['spider_group'][2][$_POST['spider_group']]))
Chris@76 106 $_POST['spider_group'] = 0;
Chris@76 107
Chris@76 108 // We'll want this for our easy save.
Chris@76 109 require_once($sourcedir . '/ManageServer.php');
Chris@76 110
Chris@76 111 // Setup the template.
Chris@76 112 $context['page_title'] = $txt['settings'];
Chris@76 113 $context['sub_template'] = 'show_settings';
Chris@76 114
Chris@76 115 // Are we saving them - are we??
Chris@76 116 if (isset($_GET['save']))
Chris@76 117 {
Chris@76 118 checkSession();
Chris@76 119
Chris@76 120 saveDBSettings($config_vars);
Chris@76 121 recacheSpiderNames();
Chris@76 122 redirectexit('action=admin;area=sengines;sa=settings');
Chris@76 123 }
Chris@76 124
Chris@76 125 // Final settings...
Chris@76 126 $context['post_url'] = $scripturl . '?action=admin;area=sengines;save;sa=settings';
Chris@76 127 $context['settings_title'] = $txt['settings'];
Chris@76 128 $context['settings_insert_below'] = '
Chris@76 129 <script type="text/javascript"><!-- // --><![CDATA[
Chris@76 130 ' . $javascript_function . '
Chris@76 131 // ]]></script>';
Chris@76 132
Chris@76 133 // Prepare the settings...
Chris@76 134 prepareDBSettingContext($config_vars);
Chris@76 135 }
Chris@76 136
Chris@76 137 // View a list of all the spiders we know about.
Chris@76 138 function ViewSpiders()
Chris@76 139 {
Chris@76 140 global $context, $txt, $sourcedir, $scripturl, $smcFunc;
Chris@76 141
Chris@76 142 if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
Chris@76 143 {
Chris@76 144 consolidateSpiderStats();
Chris@76 145 $_SESSION['spider_stat'] = time();
Chris@76 146 }
Chris@76 147
Chris@76 148 // Are we adding a new one?
Chris@76 149 if (!empty($_POST['addSpider']))
Chris@76 150 return EditSpider();
Chris@76 151 // User pressed the 'remove selection button'.
Chris@76 152 elseif (!empty($_POST['removeSpiders']) && !empty($_POST['remove']) && is_array($_POST['remove']))
Chris@76 153 {
Chris@76 154 checkSession();
Chris@76 155
Chris@76 156 // Make sure every entry is a proper integer.
Chris@76 157 foreach ($_POST['remove'] as $index => $spider_id)
Chris@76 158 $_POST['remove'][(int) $index] = (int) $spider_id;
Chris@76 159
Chris@76 160 // Delete them all!
Chris@76 161 $smcFunc['db_query']('', '
Chris@76 162 DELETE FROM {db_prefix}spiders
Chris@76 163 WHERE id_spider IN ({array_int:remove_list})',
Chris@76 164 array(
Chris@76 165 'remove_list' => $_POST['remove'],
Chris@76 166 )
Chris@76 167 );
Chris@76 168 $smcFunc['db_query']('', '
Chris@76 169 DELETE FROM {db_prefix}log_spider_hits
Chris@76 170 WHERE id_spider IN ({array_int:remove_list})',
Chris@76 171 array(
Chris@76 172 'remove_list' => $_POST['remove'],
Chris@76 173 )
Chris@76 174 );
Chris@76 175 $smcFunc['db_query']('', '
Chris@76 176 DELETE FROM {db_prefix}log_spider_stats
Chris@76 177 WHERE id_spider IN ({array_int:remove_list})',
Chris@76 178 array(
Chris@76 179 'remove_list' => $_POST['remove'],
Chris@76 180 )
Chris@76 181 );
Chris@76 182
Chris@76 183 cache_put_data('spider_search', null, 300);
Chris@76 184 recacheSpiderNames();
Chris@76 185 }
Chris@76 186
Chris@76 187 // Get the last seens.
Chris@76 188 $request = $smcFunc['db_query']('', '
Chris@76 189 SELECT id_spider, MAX(last_seen) AS last_seen_time
Chris@76 190 FROM {db_prefix}log_spider_stats
Chris@76 191 GROUP BY id_spider',
Chris@76 192 array(
Chris@76 193 )
Chris@76 194 );
Chris@76 195
Chris@76 196 $context['spider_last_seen'] = array();
Chris@76 197 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 198 $context['spider_last_seen'][$row['id_spider']] = $row['last_seen_time'];
Chris@76 199 $smcFunc['db_free_result']($request);
Chris@76 200
Chris@76 201 $listOptions = array(
Chris@76 202 'id' => 'spider_list',
Chris@76 203 'items_per_page' => 20,
Chris@76 204 'base_href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
Chris@76 205 'default_sort_col' => 'name',
Chris@76 206 'get_items' => array(
Chris@76 207 'function' => 'list_getSpiders',
Chris@76 208 ),
Chris@76 209 'get_count' => array(
Chris@76 210 'function' => 'list_getNumSpiders',
Chris@76 211 ),
Chris@76 212 'no_items_label' => $txt['spiders_no_entries'],
Chris@76 213 'columns' => array(
Chris@76 214 'name' => array(
Chris@76 215 'header' => array(
Chris@76 216 'value' => $txt['spider_name'],
Chris@76 217 ),
Chris@76 218 'data' => array(
Chris@76 219 'function' => create_function('$rowData', '
Chris@76 220 global $scripturl;
Chris@76 221
Chris@76 222 return sprintf(\'<a href="%1$s?action=admin;area=sengines;sa=editspiders;sid=%2$d">%3$s</a>\', $scripturl, $rowData[\'id_spider\'], htmlspecialchars($rowData[\'spider_name\']));
Chris@76 223 '),
Chris@76 224 ),
Chris@76 225 'sort' => array(
Chris@76 226 'default' => 'spider_name',
Chris@76 227 'reverse' => 'spider_name DESC',
Chris@76 228 ),
Chris@76 229 ),
Chris@76 230 'last_seen' => array(
Chris@76 231 'header' => array(
Chris@76 232 'value' => $txt['spider_last_seen'],
Chris@76 233 ),
Chris@76 234 'data' => array(
Chris@76 235 'function' => create_function('$rowData', '
Chris@76 236 global $context, $txt;
Chris@76 237
Chris@76 238 return isset($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) ? timeformat($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) : $txt[\'spider_last_never\'];
Chris@76 239 '),
Chris@76 240 ),
Chris@76 241 ),
Chris@76 242 'user_agent' => array(
Chris@76 243 'header' => array(
Chris@76 244 'value' => $txt['spider_agent'],
Chris@76 245 ),
Chris@76 246 'data' => array(
Chris@76 247 'db_htmlsafe' => 'user_agent',
Chris@76 248 ),
Chris@76 249 'sort' => array(
Chris@76 250 'default' => 'user_agent',
Chris@76 251 'reverse' => 'user_agent DESC',
Chris@76 252 ),
Chris@76 253 ),
Chris@76 254 'ip_info' => array(
Chris@76 255 'header' => array(
Chris@76 256 'value' => $txt['spider_ip_info'],
Chris@76 257 ),
Chris@76 258 'data' => array(
Chris@76 259 'db_htmlsafe' => 'ip_info',
Chris@76 260 'class' => 'smalltext',
Chris@76 261 ),
Chris@76 262 'sort' => array(
Chris@76 263 'default' => 'ip_info',
Chris@76 264 'reverse' => 'ip_info DESC',
Chris@76 265 ),
Chris@76 266 ),
Chris@76 267 'check' => array(
Chris@76 268 'header' => array(
Chris@76 269 'value' => '<input type="checkbox" onclick="invertAll(this, this.form);" class="input_check" />',
Chris@76 270 ),
Chris@76 271 'data' => array(
Chris@76 272 'sprintf' => array(
Chris@76 273 'format' => '<input type="checkbox" name="remove[]" value="%1$d" class="input_check" />',
Chris@76 274 'params' => array(
Chris@76 275 'id_spider' => false,
Chris@76 276 ),
Chris@76 277 ),
Chris@76 278 'style' => 'text-align: center',
Chris@76 279 ),
Chris@76 280 ),
Chris@76 281 ),
Chris@76 282 'form' => array(
Chris@76 283 'href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
Chris@76 284 ),
Chris@76 285 'additional_rows' => array(
Chris@76 286 array(
Chris@76 287 'position' => 'below_table_data',
Chris@76 288 'value' => '
Chris@76 289 <input type="submit" name="addSpider" value="' . $txt['spiders_add'] . '" class="button_submit" />
Chris@76 290 <input type="submit" name="removeSpiders" value="' . $txt['spiders_remove_selected'] . '" onclick="return confirm(\'' . $txt['spider_remove_selected_confirm'] . '\');" class="button_submit" />
Chris@76 291 ',
Chris@76 292 'style' => 'text-align: right;',
Chris@76 293 ),
Chris@76 294 ),
Chris@76 295 );
Chris@76 296
Chris@76 297 require_once($sourcedir . '/Subs-List.php');
Chris@76 298 createList($listOptions);
Chris@76 299
Chris@76 300 $context['sub_template'] = 'show_list';
Chris@76 301 $context['default_list'] = 'spider_list';
Chris@76 302 }
Chris@76 303
Chris@76 304 function list_getSpiders($start, $items_per_page, $sort)
Chris@76 305 {
Chris@76 306 global $smcFunc;
Chris@76 307
Chris@76 308 $request = $smcFunc['db_query']('', '
Chris@76 309 SELECT id_spider, spider_name, user_agent, ip_info
Chris@76 310 FROM {db_prefix}spiders
Chris@76 311 ORDER BY ' . $sort . '
Chris@76 312 LIMIT ' . $start . ', ' . $items_per_page,
Chris@76 313 array(
Chris@76 314 )
Chris@76 315 );
Chris@76 316 $spiders = array();
Chris@76 317 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 318 $spiders[$row['id_spider']] = $row;
Chris@76 319 $smcFunc['db_free_result']($request);
Chris@76 320
Chris@76 321 return $spiders;
Chris@76 322 }
Chris@76 323
Chris@76 324 function list_getNumSpiders()
Chris@76 325 {
Chris@76 326 global $smcFunc;
Chris@76 327
Chris@76 328 $request = $smcFunc['db_query']('', '
Chris@76 329 SELECT COUNT(*) AS num_spiders
Chris@76 330 FROM {db_prefix}spiders',
Chris@76 331 array(
Chris@76 332 )
Chris@76 333 );
Chris@76 334 list ($numSpiders) = $smcFunc['db_fetch_row']($request);
Chris@76 335 $smcFunc['db_free_result']($request);
Chris@76 336
Chris@76 337 return $numSpiders;
Chris@76 338 }
Chris@76 339
Chris@76 340 // Here we can add, and edit, spider info!
Chris@76 341 function EditSpider()
Chris@76 342 {
Chris@76 343 global $context, $smcFunc, $txt;
Chris@76 344
Chris@76 345 // Some standard stuff.
Chris@76 346 $context['id_spider'] = !empty($_GET['sid']) ? (int) $_GET['sid'] : 0;
Chris@76 347 $context['page_title'] = $context['id_spider'] ? $txt['spiders_edit'] : $txt['spiders_add'];
Chris@76 348 $context['sub_template'] = 'spider_edit';
Chris@76 349
Chris@76 350 // Are we saving?
Chris@76 351 if (!empty($_POST['save']))
Chris@76 352 {
Chris@76 353 checkSession();
Chris@76 354
Chris@76 355 $ips = array();
Chris@76 356 // Check the IP range is valid.
Chris@76 357 $ip_sets = explode(',', $_POST['spider_ip']);
Chris@76 358 foreach ($ip_sets as $set)
Chris@76 359 {
Chris@76 360 $test = ip2range(trim($set));
Chris@76 361 if (!empty($test))
Chris@76 362 $ips[] = $set;
Chris@76 363 }
Chris@76 364 $ips = implode(',', $ips);
Chris@76 365
Chris@76 366 // Goes in as it is...
Chris@76 367 if ($context['id_spider'])
Chris@76 368 $smcFunc['db_query']('', '
Chris@76 369 UPDATE {db_prefix}spiders
Chris@76 370 SET spider_name = {string:spider_name}, user_agent = {string:spider_agent},
Chris@76 371 ip_info = {string:ip_info}
Chris@76 372 WHERE id_spider = {int:current_spider}',
Chris@76 373 array(
Chris@76 374 'current_spider' => $context['id_spider'],
Chris@76 375 'spider_name' => $_POST['spider_name'],
Chris@76 376 'spider_agent' => $_POST['spider_agent'],
Chris@76 377 'ip_info' => $ips,
Chris@76 378 )
Chris@76 379 );
Chris@76 380 else
Chris@76 381 $smcFunc['db_insert']('insert',
Chris@76 382 '{db_prefix}spiders',
Chris@76 383 array(
Chris@76 384 'spider_name' => 'string', 'user_agent' => 'string', 'ip_info' => 'string',
Chris@76 385 ),
Chris@76 386 array(
Chris@76 387 $_POST['spider_name'], $_POST['spider_agent'], $ips,
Chris@76 388 ),
Chris@76 389 array('id_spider')
Chris@76 390 );
Chris@76 391
Chris@76 392 // Order by user agent length.
Chris@76 393 sortSpiderTable();
Chris@76 394
Chris@76 395 cache_put_data('spider_search', null, 300);
Chris@76 396 recacheSpiderNames();
Chris@76 397
Chris@76 398 redirectexit('action=admin;area=sengines;sa=spiders');
Chris@76 399 }
Chris@76 400
Chris@76 401 // The default is new.
Chris@76 402 $context['spider'] = array(
Chris@76 403 'id' => 0,
Chris@76 404 'name' => '',
Chris@76 405 'agent' => '',
Chris@76 406 'ip_info' => '',
Chris@76 407 );
Chris@76 408
Chris@76 409 // An edit?
Chris@76 410 if ($context['id_spider'])
Chris@76 411 {
Chris@76 412 $request = $smcFunc['db_query']('', '
Chris@76 413 SELECT id_spider, spider_name, user_agent, ip_info
Chris@76 414 FROM {db_prefix}spiders
Chris@76 415 WHERE id_spider = {int:current_spider}',
Chris@76 416 array(
Chris@76 417 'current_spider' => $context['id_spider'],
Chris@76 418 )
Chris@76 419 );
Chris@76 420 if ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 421 $context['spider'] = array(
Chris@76 422 'id' => $row['id_spider'],
Chris@76 423 'name' => $row['spider_name'],
Chris@76 424 'agent' => $row['user_agent'],
Chris@76 425 'ip_info' => $row['ip_info'],
Chris@76 426 );
Chris@76 427 $smcFunc['db_free_result']($request);
Chris@76 428 }
Chris@76 429
Chris@76 430 }
Chris@76 431
Chris@76 432 //!!! Should this not be... you know... in a different file?
Chris@76 433 // Do we think the current user is a spider?
Chris@76 434 function SpiderCheck()
Chris@76 435 {
Chris@76 436 global $modSettings, $smcFunc;
Chris@76 437
Chris@76 438 if (isset($_SESSION['id_robot']))
Chris@76 439 unset($_SESSION['id_robot']);
Chris@76 440 $_SESSION['robot_check'] = time();
Chris@76 441
Chris@76 442 // We cache the spider data for five minutes if we can.
Chris@76 443 if (!empty($modSettings['cache_enable']))
Chris@76 444 $spider_data = cache_get_data('spider_search', 300);
Chris@76 445
Chris@76 446 if (!isset($spider_data) || $spider_data === NULL)
Chris@76 447 {
Chris@76 448 $request = $smcFunc['db_query']('spider_check', '
Chris@76 449 SELECT id_spider, user_agent, ip_info
Chris@76 450 FROM {db_prefix}spiders',
Chris@76 451 array(
Chris@76 452 )
Chris@76 453 );
Chris@76 454 $spider_data = array();
Chris@76 455 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 456 $spider_data[] = $row;
Chris@76 457 $smcFunc['db_free_result']($request);
Chris@76 458
Chris@76 459 if (!empty($modSettings['cache_enable']))
Chris@76 460 cache_put_data('spider_search', $spider_data, 300);
Chris@76 461 }
Chris@76 462
Chris@76 463 if (empty($spider_data))
Chris@76 464 return false;
Chris@76 465
Chris@76 466 // Only do these bits once.
Chris@76 467 $ci_user_agent = strtolower($_SERVER['HTTP_USER_AGENT']);
Chris@76 468 preg_match('/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $_SERVER['REMOTE_ADDR'], $ip_parts);
Chris@76 469
Chris@76 470 foreach ($spider_data as $spider)
Chris@76 471 {
Chris@76 472 // User agent is easy.
Chris@76 473 if (!empty($spider['user_agent']) && strpos($ci_user_agent, strtolower($spider['user_agent'])) !== false)
Chris@76 474 $_SESSION['id_robot'] = $spider['id_spider'];
Chris@76 475 // IP stuff is harder.
Chris@76 476 elseif (!empty($ip_parts))
Chris@76 477 {
Chris@76 478 $ips = explode(',', $spider['ip_info']);
Chris@76 479 foreach ($ips as $ip)
Chris@76 480 {
Chris@76 481 $ip = ip2range($ip);
Chris@76 482 if (!empty($ip))
Chris@76 483 {
Chris@76 484 foreach ($ip as $key => $value)
Chris@76 485 {
Chris@76 486 if ($value['low'] > $ip_parts[$key + 1] || $value['high'] < $ip_parts[$key + 1])
Chris@76 487 break;
Chris@76 488 elseif ($key == 3)
Chris@76 489 $_SESSION['id_robot'] = $spider['id_spider'];
Chris@76 490 }
Chris@76 491 }
Chris@76 492 }
Chris@76 493 }
Chris@76 494
Chris@76 495 if (isset($_SESSION['id_robot']))
Chris@76 496 break;
Chris@76 497 }
Chris@76 498
Chris@76 499 // If this is low server tracking then log the spider here as oppossed to the main logging function.
Chris@76 500 if (!empty($modSettings['spider_mode']) && $modSettings['spider_mode'] == 1 && !empty($_SESSION['id_robot']))
Chris@76 501 logSpider();
Chris@76 502
Chris@76 503 return !empty($_SESSION['id_robot']) ? $_SESSION['id_robot'] : 0;
Chris@76 504 }
Chris@76 505
Chris@76 506 // Log the spider presence online.
Chris@76 507 //!!! Different file?
Chris@76 508 function logSpider()
Chris@76 509 {
Chris@76 510 global $smcFunc, $modSettings, $context;
Chris@76 511
Chris@76 512 if (empty($modSettings['spider_mode']) || empty($_SESSION['id_robot']))
Chris@76 513 return;
Chris@76 514
Chris@76 515 // Attempt to update today's entry.
Chris@76 516 if ($modSettings['spider_mode'] == 1)
Chris@76 517 {
Chris@76 518 $date = strftime('%Y-%m-%d', forum_time(false));
Chris@76 519 $smcFunc['db_query']('', '
Chris@76 520 UPDATE {db_prefix}log_spider_stats
Chris@76 521 SET last_seen = {int:current_time}, page_hits = page_hits + 1
Chris@76 522 WHERE id_spider = {int:current_spider}
Chris@76 523 AND stat_date = {date:current_date}',
Chris@76 524 array(
Chris@76 525 'current_date' => $date,
Chris@76 526 'current_time' => time(),
Chris@76 527 'current_spider' => $_SESSION['id_robot'],
Chris@76 528 )
Chris@76 529 );
Chris@76 530
Chris@76 531 // Nothing updated?
Chris@76 532 if ($smcFunc['db_affected_rows']() == 0)
Chris@76 533 {
Chris@76 534 $smcFunc['db_insert']('ignore',
Chris@76 535 '{db_prefix}log_spider_stats',
Chris@76 536 array(
Chris@76 537 'id_spider' => 'int', 'last_seen' => 'int', 'stat_date' => 'date', 'page_hits' => 'int',
Chris@76 538 ),
Chris@76 539 array(
Chris@76 540 $_SESSION['id_robot'], time(), $date, 1,
Chris@76 541 ),
Chris@76 542 array('id_spider', 'stat_date')
Chris@76 543 );
Chris@76 544 }
Chris@76 545 }
Chris@76 546 // If we're tracking better stats than track, better stats - we sort out the today thing later.
Chris@76 547 else
Chris@76 548 {
Chris@76 549 if ($modSettings['spider_mode'] > 2)
Chris@76 550 {
Chris@76 551 $url = $_GET + array('USER_AGENT' => $_SERVER['HTTP_USER_AGENT']);
Chris@76 552 unset($url['sesc'], $url[$context['session_var']]);
Chris@76 553 $url = serialize($url);
Chris@76 554 }
Chris@76 555 else
Chris@76 556 $url = '';
Chris@76 557
Chris@76 558 $smcFunc['db_insert']('insert',
Chris@76 559 '{db_prefix}log_spider_hits',
Chris@76 560 array('id_spider' => 'int', 'log_time' => 'int', 'url' => 'string'),
Chris@76 561 array($_SESSION['id_robot'], time(), $url),
Chris@76 562 array()
Chris@76 563 );
Chris@76 564 }
Chris@76 565 }
Chris@76 566
Chris@76 567 // This function takes any unprocessed hits and turns them into stats.
Chris@76 568 function consolidateSpiderStats()
Chris@76 569 {
Chris@76 570 global $smcFunc;
Chris@76 571
Chris@76 572 $request = $smcFunc['db_query']('consolidate_spider_stats', '
Chris@76 573 SELECT id_spider, MAX(log_time) AS last_seen, COUNT(*) AS num_hits
Chris@76 574 FROM {db_prefix}log_spider_hits
Chris@76 575 WHERE processed = {int:not_processed}
Chris@76 576 GROUP BY id_spider, MONTH(log_time), DAYOFMONTH(log_time)',
Chris@76 577 array(
Chris@76 578 'not_processed' => 0,
Chris@76 579 )
Chris@76 580 );
Chris@76 581 $spider_hits = array();
Chris@76 582 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 583 $spider_hits[] = $row;
Chris@76 584 $smcFunc['db_free_result']($request);
Chris@76 585
Chris@76 586 if (empty($spider_hits))
Chris@76 587 return;
Chris@76 588
Chris@76 589 // Attempt to update the master data.
Chris@76 590 $stat_inserts = array();
Chris@76 591 foreach ($spider_hits as $stat)
Chris@76 592 {
Chris@76 593 // We assume the max date is within the right day.
Chris@76 594 $date = strftime('%Y-%m-%d', $stat['last_seen']);
Chris@76 595 $smcFunc['db_query']('', '
Chris@76 596 UPDATE {db_prefix}log_spider_stats
Chris@76 597 SET page_hits = page_hits + ' . $stat['num_hits'] . ',
Chris@76 598 last_seen = CASE WHEN last_seen > {int:last_seen} THEN last_seen ELSE {int:last_seen} END
Chris@76 599 WHERE id_spider = {int:current_spider}
Chris@76 600 AND stat_date = {date:last_seen_date}',
Chris@76 601 array(
Chris@76 602 'last_seen_date' => $date,
Chris@76 603 'last_seen' => $stat['last_seen'],
Chris@76 604 'current_spider' => $stat['id_spider'],
Chris@76 605 )
Chris@76 606 );
Chris@76 607 if ($smcFunc['db_affected_rows']() == 0)
Chris@76 608 $stat_inserts[] = array($date, $stat['id_spider'], $stat['num_hits'], $stat['last_seen']);
Chris@76 609 }
Chris@76 610
Chris@76 611 // New stats?
Chris@76 612 if (!empty($stat_inserts))
Chris@76 613 $smcFunc['db_insert']('ignore',
Chris@76 614 '{db_prefix}log_spider_stats',
Chris@76 615 array('stat_date' => 'date', 'id_spider' => 'int', 'page_hits' => 'int', 'last_seen' => 'int'),
Chris@76 616 $stat_inserts,
Chris@76 617 array('stat_date', 'id_spider')
Chris@76 618 );
Chris@76 619
Chris@76 620 // All processed.
Chris@76 621 $smcFunc['db_query']('', '
Chris@76 622 UPDATE {db_prefix}log_spider_hits
Chris@76 623 SET processed = {int:is_processed}
Chris@76 624 WHERE processed = {int:not_processed}',
Chris@76 625 array(
Chris@76 626 'is_processed' => 1,
Chris@76 627 'not_processed' => 0,
Chris@76 628 )
Chris@76 629 );
Chris@76 630 }
Chris@76 631
Chris@76 632 // See what spiders have been up to.
Chris@76 633 function SpiderLogs()
Chris@76 634 {
Chris@76 635 global $context, $txt, $sourcedir, $scripturl, $smcFunc, $modSettings;
Chris@76 636
Chris@76 637 // Load the template and language just incase.
Chris@76 638 loadLanguage('Search');
Chris@76 639 loadTemplate('ManageSearch');
Chris@76 640
Chris@76 641 // Did they want to delete some entries?
Chris@76 642 if (!empty($_POST['delete_entries']) && isset($_POST['older']))
Chris@76 643 {
Chris@76 644 checkSession();
Chris@76 645
Chris@76 646 $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
Chris@76 647
Chris@76 648 // Delete the entires.
Chris@76 649 $smcFunc['db_query']('', '
Chris@76 650 DELETE FROM {db_prefix}log_spider_hits
Chris@76 651 WHERE log_time < {int:delete_period}',
Chris@76 652 array(
Chris@76 653 'delete_period' => $deleteTime,
Chris@76 654 )
Chris@76 655 );
Chris@76 656 }
Chris@76 657
Chris@76 658 $listOptions = array(
Chris@76 659 'id' => 'spider_logs',
Chris@76 660 'items_per_page' => 20,
Chris@76 661 'title' => $txt['spider_logs'],
Chris@76 662 'no_items_label' => $txt['spider_logs_empty'],
Chris@76 663 'base_href' => $context['admin_area'] == 'sengines' ? $scripturl . '?action=admin;area=sengines;sa=logs' : $scripturl . '?action=admin;area=logs;sa=spiderlog',
Chris@76 664 'default_sort_col' => 'log_time',
Chris@76 665 'get_items' => array(
Chris@76 666 'function' => 'list_getSpiderLogs',
Chris@76 667 ),
Chris@76 668 'get_count' => array(
Chris@76 669 'function' => 'list_getNumSpiderLogs',
Chris@76 670 ),
Chris@76 671 'columns' => array(
Chris@76 672 'name' => array(
Chris@76 673 'header' => array(
Chris@76 674 'value' => $txt['spider'],
Chris@76 675 ),
Chris@76 676 'data' => array(
Chris@76 677 'db' => 'spider_name',
Chris@76 678 ),
Chris@76 679 'sort' => array(
Chris@76 680 'default' => 's.spider_name',
Chris@76 681 'reverse' => 's.spider_name DESC',
Chris@76 682 ),
Chris@76 683 ),
Chris@76 684 'log_time' => array(
Chris@76 685 'header' => array(
Chris@76 686 'value' => $txt['spider_time'],
Chris@76 687 ),
Chris@76 688 'data' => array(
Chris@76 689 'function' => create_function('$rowData', '
Chris@76 690 return timeformat($rowData[\'log_time\']);
Chris@76 691 '),
Chris@76 692 ),
Chris@76 693 'sort' => array(
Chris@76 694 'default' => 'sl.id_hit DESC',
Chris@76 695 'reverse' => 'sl.id_hit',
Chris@76 696 ),
Chris@76 697 ),
Chris@76 698 'viewing' => array(
Chris@76 699 'header' => array(
Chris@76 700 'value' => $txt['spider_viewing'],
Chris@76 701 ),
Chris@76 702 'data' => array(
Chris@76 703 'db' => 'url',
Chris@76 704 ),
Chris@76 705 ),
Chris@76 706 ),
Chris@76 707 'additional_rows' => array(
Chris@76 708 array(
Chris@76 709 'position' => 'after_title',
Chris@76 710 'value' => $txt['spider_logs_info'],
Chris@76 711 'class' => 'smalltext',
Chris@76 712 ),
Chris@76 713 ),
Chris@76 714 );
Chris@76 715
Chris@76 716 require_once($sourcedir . '/Subs-List.php');
Chris@76 717 createList($listOptions);
Chris@76 718
Chris@76 719 // Now determine the actions of the URLs.
Chris@76 720 if (!empty($context['spider_logs']['rows']))
Chris@76 721 {
Chris@76 722 $urls = array();
Chris@76 723 // Grab the current /url.
Chris@76 724 foreach ($context['spider_logs']['rows'] as $k => $row)
Chris@76 725 {
Chris@76 726 // Feature disabled?
Chris@76 727 if (empty($row['viewing']['value']) && isset($modSettings['spider_mode']) && $modSettings['spider_mode'] < 3)
Chris@76 728 $context['spider_logs']['rows'][$k]['viewing']['value'] = '<em>' . $txt['spider_disabled'] . '</em>';
Chris@76 729 else
Chris@76 730 $urls[$k] = array($row['viewing']['value'], -1);
Chris@76 731 }
Chris@76 732
Chris@76 733 // Now stick in the new URLs.
Chris@76 734 require_once($sourcedir . '/Who.php');
Chris@76 735 $urls = determineActions($urls, 'whospider_');
Chris@76 736 foreach ($urls as $k => $new_url)
Chris@76 737 {
Chris@76 738 $context['spider_logs']['rows'][$k]['viewing']['value'] = $new_url;
Chris@76 739 }
Chris@76 740 }
Chris@76 741
Chris@76 742 $context['page_title'] = $txt['spider_logs'];
Chris@76 743 $context['sub_template'] = 'show_spider_logs';
Chris@76 744 $context['default_list'] = 'spider_logs';
Chris@76 745 }
Chris@76 746
Chris@76 747 function list_getSpiderLogs($start, $items_per_page, $sort)
Chris@76 748 {
Chris@76 749 global $smcFunc;
Chris@76 750
Chris@76 751 $request = $smcFunc['db_query']('', '
Chris@76 752 SELECT sl.id_spider, sl.url, sl.log_time, s.spider_name
Chris@76 753 FROM {db_prefix}log_spider_hits AS sl
Chris@76 754 INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = sl.id_spider)
Chris@76 755 ORDER BY ' . $sort . '
Chris@76 756 LIMIT ' . $start . ', ' . $items_per_page,
Chris@76 757 array(
Chris@76 758 )
Chris@76 759 );
Chris@76 760 $spider_logs = array();
Chris@76 761 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 762 $spider_logs[] = $row;
Chris@76 763 $smcFunc['db_free_result']($request);
Chris@76 764
Chris@76 765 return $spider_logs;
Chris@76 766 }
Chris@76 767
Chris@76 768 function list_getNumSpiderLogs()
Chris@76 769 {
Chris@76 770 global $smcFunc;
Chris@76 771
Chris@76 772 $request = $smcFunc['db_query']('', '
Chris@76 773 SELECT COUNT(*) AS num_logs
Chris@76 774 FROM {db_prefix}log_spider_hits',
Chris@76 775 array(
Chris@76 776 )
Chris@76 777 );
Chris@76 778 list ($numLogs) = $smcFunc['db_fetch_row']($request);
Chris@76 779 $smcFunc['db_free_result']($request);
Chris@76 780
Chris@76 781 return $numLogs;
Chris@76 782 }
Chris@76 783
Chris@76 784 // Show the spider statistics.
Chris@76 785 function SpiderStats()
Chris@76 786 {
Chris@76 787 global $context, $txt, $sourcedir, $scripturl, $smcFunc;
Chris@76 788
Chris@76 789 // Force an update of the stats every 60 seconds.
Chris@76 790 if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
Chris@76 791 {
Chris@76 792 consolidateSpiderStats();
Chris@76 793 $_SESSION['spider_stat'] = time();
Chris@76 794 }
Chris@76 795
Chris@76 796 // Get the earliest and latest dates.
Chris@76 797 $request = $smcFunc['db_query']('', '
Chris@76 798 SELECT MIN(stat_date) AS first_date, MAX(stat_date) AS last_date
Chris@76 799 FROM {db_prefix}log_spider_stats',
Chris@76 800 array(
Chris@76 801 )
Chris@76 802 );
Chris@76 803
Chris@76 804 list ($min_date, $max_date) = $smcFunc['db_fetch_row']($request);
Chris@76 805 $smcFunc['db_free_result']($request);
Chris@76 806
Chris@76 807 $min_year = (int) substr($min_date, 0, 4);
Chris@76 808 $max_year = (int) substr($max_date, 0, 4);
Chris@76 809 $min_month = (int) substr($min_date, 5, 2);
Chris@76 810 $max_month = (int) substr($max_date, 5, 2);
Chris@76 811
Chris@76 812 // Prepare the dates for the drop down.
Chris@76 813 $date_choices = array();
Chris@76 814 for ($y = $min_year; $y <= $max_year; $y++)
Chris@76 815 for ($m = 1; $m <= 12; $m++)
Chris@76 816 {
Chris@76 817 // This doesn't count?
Chris@76 818 if ($y == $min_year && $m < $min_month)
Chris@76 819 continue;
Chris@76 820 if ($y == $max_year && $m > $max_month)
Chris@76 821 break;
Chris@76 822
Chris@76 823 $date_choices[$y . $m] = $txt['months_short'][$m] . ' ' . $y;
Chris@76 824 }
Chris@76 825
Chris@76 826 // What are we currently viewing?
Chris@76 827 $current_date = isset($_REQUEST['new_date']) && isset($date_choices[$_REQUEST['new_date']]) ? $_REQUEST['new_date'] : $max_date;
Chris@76 828
Chris@76 829 // Prepare the HTML.
Chris@76 830 $date_select = '
Chris@76 831 ' . $txt['spider_stats_select_month'] . ':
Chris@76 832 <select name="new_date" onchange="document.spider_stat_list.submit();">';
Chris@76 833
Chris@76 834 if (empty($date_choices))
Chris@76 835 $date_select .= '
Chris@76 836 <option></option>';
Chris@76 837 else
Chris@76 838 foreach ($date_choices as $id => $text)
Chris@76 839 $date_select .= '
Chris@76 840 <option value="' . $id . '"' . ($current_date == $id ? ' selected="selected"' : '') . '>' . $text . '</option>';
Chris@76 841
Chris@76 842 $date_select .= '
Chris@76 843 </select>
Chris@76 844 <noscript>
Chris@76 845 <input type="submit" name="go" value="' . $txt['go'] . '" class="button_submit" />
Chris@76 846 </noscript>';
Chris@76 847
Chris@76 848 // If we manually jumped to a date work out the offset.
Chris@76 849 if (isset($_REQUEST['new_date']))
Chris@76 850 {
Chris@76 851 $date_query = sprintf('%04d-%02d-01', substr($current_date, 0, 4), substr($current_date, 4));
Chris@76 852
Chris@76 853 $request = $smcFunc['db_query']('', '
Chris@76 854 SELECT COUNT(*) AS offset
Chris@76 855 FROM {db_prefix}log_spider_stats
Chris@76 856 WHERE stat_date < {date:date_being_viewed}',
Chris@76 857 array(
Chris@76 858 'date_being_viewed' => $date_query,
Chris@76 859 )
Chris@76 860 );
Chris@76 861 list ($_REQUEST['start']) = $smcFunc['db_fetch_row']($request);
Chris@76 862 $smcFunc['db_free_result']($request);
Chris@76 863 }
Chris@76 864
Chris@76 865 $listOptions = array(
Chris@76 866 'id' => 'spider_stat_list',
Chris@76 867 'items_per_page' => 20,
Chris@76 868 'base_href' => $scripturl . '?action=admin;area=sengines;sa=stats',
Chris@76 869 'default_sort_col' => 'stat_date',
Chris@76 870 'get_items' => array(
Chris@76 871 'function' => 'list_getSpiderStats',
Chris@76 872 ),
Chris@76 873 'get_count' => array(
Chris@76 874 'function' => 'list_getNumSpiderStats',
Chris@76 875 ),
Chris@76 876 'no_items_label' => $txt['spider_stats_no_entries'],
Chris@76 877 'columns' => array(
Chris@76 878 'stat_date' => array(
Chris@76 879 'header' => array(
Chris@76 880 'value' => $txt['date'],
Chris@76 881 ),
Chris@76 882 'data' => array(
Chris@76 883 'db' => 'stat_date',
Chris@76 884 ),
Chris@76 885 'sort' => array(
Chris@76 886 'default' => 'stat_date',
Chris@76 887 'reverse' => 'stat_date DESC',
Chris@76 888 ),
Chris@76 889 ),
Chris@76 890 'name' => array(
Chris@76 891 'header' => array(
Chris@76 892 'value' => $txt['spider_name'],
Chris@76 893 ),
Chris@76 894 'data' => array(
Chris@76 895 'db' => 'spider_name',
Chris@76 896 ),
Chris@76 897 'sort' => array(
Chris@76 898 'default' => 's.spider_name',
Chris@76 899 'reverse' => 's.spider_name DESC',
Chris@76 900 ),
Chris@76 901 ),
Chris@76 902 'page_hits' => array(
Chris@76 903 'header' => array(
Chris@76 904 'value' => $txt['spider_stats_page_hits'],
Chris@76 905 ),
Chris@76 906 'data' => array(
Chris@76 907 'db' => 'page_hits',
Chris@76 908 ),
Chris@76 909 'sort' => array(
Chris@76 910 'default' => 'ss.page_hits',
Chris@76 911 'reverse' => 'ss.page_hits DESC',
Chris@76 912 ),
Chris@76 913 ),
Chris@76 914 ),
Chris@76 915 'form' => array(
Chris@76 916 'href' => $scripturl . '?action=admin;area=sengines;sa=stats',
Chris@76 917 'name' => 'spider_stat_list',
Chris@76 918 ),
Chris@76 919 'additional_rows' => array(
Chris@76 920 array(
Chris@76 921 'position' => 'below_table_data',
Chris@76 922 'value' => $date_select,
Chris@76 923 'style' => 'text-align: right;',
Chris@76 924 ),
Chris@76 925 ),
Chris@76 926 );
Chris@76 927
Chris@76 928 require_once($sourcedir . '/Subs-List.php');
Chris@76 929 createList($listOptions);
Chris@76 930
Chris@76 931 $context['sub_template'] = 'show_list';
Chris@76 932 $context['default_list'] = 'spider_stat_list';
Chris@76 933 }
Chris@76 934
Chris@76 935 function list_getSpiderStats($start, $items_per_page, $sort)
Chris@76 936 {
Chris@76 937 global $smcFunc;
Chris@76 938
Chris@76 939 $request = $smcFunc['db_query']('', '
Chris@76 940 SELECT ss.id_spider, ss.stat_date, ss.page_hits, s.spider_name
Chris@76 941 FROM {db_prefix}log_spider_stats AS ss
Chris@76 942 INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = ss.id_spider)
Chris@76 943 ORDER BY ' . $sort . '
Chris@76 944 LIMIT ' . $start . ', ' . $items_per_page,
Chris@76 945 array(
Chris@76 946 )
Chris@76 947 );
Chris@76 948 $spider_stats = array();
Chris@76 949 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 950 $spider_stats[] = $row;
Chris@76 951 $smcFunc['db_free_result']($request);
Chris@76 952
Chris@76 953 return $spider_stats;
Chris@76 954 }
Chris@76 955
Chris@76 956 function list_getNumSpiderStats()
Chris@76 957 {
Chris@76 958 global $smcFunc;
Chris@76 959
Chris@76 960 $request = $smcFunc['db_query']('', '
Chris@76 961 SELECT COUNT(*) AS num_stats
Chris@76 962 FROM {db_prefix}log_spider_stats',
Chris@76 963 array(
Chris@76 964 )
Chris@76 965 );
Chris@76 966 list ($numStats) = $smcFunc['db_fetch_row']($request);
Chris@76 967 $smcFunc['db_free_result']($request);
Chris@76 968
Chris@76 969 return $numStats;
Chris@76 970 }
Chris@76 971
Chris@76 972 // Recache spider names?
Chris@76 973 function recacheSpiderNames()
Chris@76 974 {
Chris@76 975 global $smcFunc;
Chris@76 976
Chris@76 977 $request = $smcFunc['db_query']('', '
Chris@76 978 SELECT id_spider, spider_name
Chris@76 979 FROM {db_prefix}spiders',
Chris@76 980 array(
Chris@76 981 )
Chris@76 982 );
Chris@76 983 $spiders = array();
Chris@76 984 while ($row = $smcFunc['db_fetch_assoc']($request))
Chris@76 985 $spiders[$row['id_spider']] = $row['spider_name'];
Chris@76 986 $smcFunc['db_free_result']($request);
Chris@76 987
Chris@76 988 updateSettings(array('spider_name_cache' => serialize($spiders)));
Chris@76 989 }
Chris@76 990
Chris@76 991 // Sort the search engine table by user agent name to avoid misidentification of engine.
Chris@76 992 function sortSpiderTable()
Chris@76 993 {
Chris@76 994 global $smcFunc;
Chris@76 995
Chris@76 996 db_extend('packages');
Chris@76 997
Chris@76 998 // Add a sorting column.
Chris@76 999 $smcFunc['db_add_column']('{db_prefix}spiders', array('name' => 'temp_order', 'size' => 8, 'type' => 'mediumint', 'null' => false));
Chris@76 1000
Chris@76 1001 // Set the contents of this column.
Chris@76 1002 $smcFunc['db_query']('set_spider_order', '
Chris@76 1003 UPDATE {db_prefix}spiders
Chris@76 1004 SET temp_order = LENGTH(user_agent)',
Chris@76 1005 array(
Chris@76 1006 )
Chris@76 1007 );
Chris@76 1008
Chris@76 1009 // Order the table by this column.
Chris@76 1010 $smcFunc['db_query']('alter_table_spiders', '
Chris@76 1011 ALTER TABLE {db_prefix}spiders
Chris@76 1012 ORDER BY temp_order DESC',
Chris@76 1013 array(
Chris@76 1014 'db_error_skip' => true,
Chris@76 1015 )
Chris@76 1016 );
Chris@76 1017
Chris@76 1018 // Remove the sorting column.
Chris@76 1019 $smcFunc['db_remove_column']('{db_prefix}spiders', 'temp_order');
Chris@76 1020 }
Chris@76 1021
Chris@76 1022 ?>