includes/search/SearchMySQL.php

   1 <?php
   2 /**
   3  * MySQL search engine
   4  *
   5  * Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Search
  25  */
  26
  27 /**
  28  * Search engine hook for MySQL 4+
  29  * @ingroup Search
  30  */
  31 class SearchMySQL extends SearchEngine {
  32         var $strictMatching = true;
  33         static $mMinSearchLength;
  34
  35         /** @todo document */
  36         function __construct( $db ) {
  37                 $this->db = $db;
  38         }
  39
  40         /**
  41          * Parse the user's query and transform it into an SQL fragment which will
  42          * become part of a WHERE clause
  43          */
  44         function parseQuery( $filteredText, $fulltext ) {
  45                 global $wgContLang;
  46                 $lc = SearchEngine::legalSearchChars(); // Minus format chars
  47                 $searchon = '';
  48                 $this->searchTerms = array();
  49
  50                 # FIXME: This doesn't handle parenthetical expressions.
  51                 $m = array();
  52                 if( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
  53                           $filteredText, $m, PREG_SET_ORDER ) ) {
  54                         foreach( $m as $bits ) {
  55                                 @list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
  56
  57                                 if( $nonQuoted != '' ) {
  58                                         $term = $nonQuoted;
  59                                         $quote = '';
  60                                 } else {
  61                                         $term = str_replace( '"', '', $term );
  62                                         $quote = '"';
  63                                 }
  64
  65                                 if( $searchon !== '' ) $searchon .= ' ';
  66                                 if( $this->strictMatching && ($modifier == '') ) {
  67                                         // If we leave this out, boolean op defaults to OR which is rarely helpful.
  68                                         $modifier = '+';
  69                                 }
  70
  71                                 // Some languages such as Serbian store the input form in the search index,
  72                                 // so we may need to search for matches in multiple writing system variants.
  73                                 $convertedVariants = $wgContLang->autoConvertToAllVariants( $term );
  74                                 if( is_array( $convertedVariants ) ) {
  75                                         $variants = array_unique( array_values( $convertedVariants ) );
  76                                 } else {
  77                                         $variants = array( $term );
  78                                 }
  79
  80                                 // The low-level search index does some processing on input to work
  81                                 // around problems with minimum lengths and encoding in MySQL's
  82                                 // fulltext engine.
  83                                 // For Chinese this also inserts spaces between adjacent Han characters.
  84                                 $strippedVariants = array_map(
  85                                         array( $wgContLang, 'normalizeForSearch' ),
  86                                         $variants );
  87
  88                                 // Some languages such as Chinese force all variants to a canonical
  89                                 // form when stripping to the low-level search index, so to be sure
  90                                 // let's check our variants list for unique items after stripping.
  91                                 $strippedVariants = array_unique( $strippedVariants );
  92
  93                                 $searchon .= $modifier;
  94                                 if( count( $strippedVariants) > 1 )
  95                                         $searchon .= '(';
  96                                 foreach( $strippedVariants as $stripped ) {
  97                                         $stripped = $this->normalizeText( $stripped );
  98                                         if( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
  99                                                 // Hack for Chinese: we need to toss in quotes for
 100                                                 // multiple-character phrases since normalizeForSearch()
 101                                                 // added spaces between them to make word breaks.
 102                                                 $stripped = '"' . trim( $stripped ) . '"';
 103                                         }
 104                                         $searchon .= "$quote$stripped$quote$wildcard ";
 105                                 }
 106                                 if( count( $strippedVariants) > 1 )
 107                                         $searchon .= ')';
 108
 109                                 // Match individual terms or quoted phrase in result highlighting...
 110                                 // Note that variants will be introduced in a later stage for highlighting!
 111                                 $regexp = $this->regexTerm( $term, $wildcard );
 112                                 $this->searchTerms[] = $regexp;
 113                         }
 114                         wfDebug( __METHOD__ . ": Would search with '$searchon'\n" );
 115                         wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/\n" );
 116                 } else {
 117                         wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'\n" );
 118                 }
 119
 120                 $searchon = $this->db->strencode( $searchon );
 121                 $field = $this->getIndexField( $fulltext );
 122                 return " MATCH($field) AGAINST('$searchon' IN BOOLEAN MODE) ";
 123         }
 124
 125         function regexTerm( $string, $wildcard ) {
 126                 global $wgContLang;
 127
 128                 $regex = preg_quote( $string, '/' );
 129                 if( $wgContLang->hasWordBreaks() ) {
 130                         if( $wildcard ) {
 131                                 // Don't cut off the final bit!
 132                                 $regex = "\b$regex";
 133                         } else {
 134                                 $regex = "\b$regex\b";
 135                         }
 136                 } else {
 137                         // For Chinese, words may legitimately abut other words in the text literal.
 138                         // Don't add \b boundary checks... note this could cause false positives
 139                         // for latin chars.
 140                 }
 141                 return $regex;
 142         }
 143
 144         public static function legalSearchChars() {
 145                 return "\"*" . parent::legalSearchChars();
 146         }
 147
 148         /**
 149          * Perform a full text search query and return a result set.
 150          *
 151          * @param $term String: raw search term
 152          * @return MySQLSearchResultSet
 153          */
 154         function searchText( $term ) {
 155                 return $this->searchInternal( $term, true );
 156         }
 157
 158         /**
 159          * Perform a title-only search query and return a result set.
 160          *
 161          * @param $term String: raw search term
 162          * @return MySQLSearchResultSet
 163          */
 164         function searchTitle( $term ) {
 165                 return $this->searchInternal( $term, false );
 166         }
 167
 168         protected function searchInternal( $term, $fulltext ) {
 169                 global $wgCountTotalSearchHits;
 170
 171                 $filteredTerm = $this->filter( $term );
 172                 $resultSet = $this->db->query( $this->getQuery( $filteredTerm, $fulltext ) );
 173
 174                 $total = null;
 175                 if( $wgCountTotalSearchHits ) {
 176                         $totalResult = $this->db->query( $this->getCountQuery( $filteredTerm, $fulltext ) );
 177                         $row = $totalResult->fetchObject();
 178                         if( $row ) {
 179                                 $total = intval( $row->c );
 180                         }
 181                         $totalResult->free();
 182                 }
 183
 184                 return new MySQLSearchResultSet( $resultSet, $this->searchTerms, $total );
 185         }
 186
 187
 188         /**
 189          * Return a partial WHERE clause to exclude redirects, if so set
 190          * @return String
 191          */
 192         function queryRedirect() {
 193                 if( $this->showRedirects ) {
 194                         return '';
 195                 } else {
 196                         return 'AND page_is_redirect=0';
 197                 }
 198         }
 199
 200         /**
 201          * Return a partial WHERE clause to limit the search to the given namespaces
 202          * @return String
 203          */
 204         function queryNamespaces() {
 205                 if( is_null($this->namespaces) )
 206                         return '';  # search all
 207                 if ( !count( $this->namespaces ) ) {
 208                         $namespaces = '0';
 209                 } else {
 210                         $namespaces = $this->db->makeList( $this->namespaces );
 211                 }
 212                 return 'AND page_namespace IN (' . $namespaces . ')';
 213         }
 214
 215         /**
 216          * Return a LIMIT clause to limit results on the query.
 217          * @return String
 218          */
 219         function queryLimit() {
 220                 return $this->db->limitResult( '', $this->limit, $this->offset );
 221         }
 222
 223         /**
 224          * Does not do anything for generic search engine
 225          * subclasses may define this though
 226          * @return String
 227          */
 228         function queryRanking( $filteredTerm, $fulltext ) {
 229                 return '';
 230         }
 231
 232         /**
 233          * Construct the full SQL query to do the search.
 234          * The guts shoulds be constructed in queryMain()
 235          * @param $filteredTerm String
 236          * @param $fulltext Boolean
 237          */
 238         function getQuery( $filteredTerm, $fulltext ) {
 239                 return $this->queryMain( $filteredTerm, $fulltext ) . ' ' .
 240                         $this->queryRedirect() . ' ' .
 241                         $this->queryNamespaces() . ' ' .
 242                         $this->queryRanking( $filteredTerm, $fulltext ) . ' ' .
 243                         $this->queryLimit();
 244         }
 245
 246         /**
 247          * Picks which field to index on, depending on what type of query.
 248          * @param $fulltext Boolean
 249          * @return String
 250          */
 251         function getIndexField( $fulltext ) {
 252                 return $fulltext ? 'si_text' : 'si_title';
 253         }
 254
 255         /**
 256          * Get the base part of the search query.
 257          * The actual match syntax will depend on the server
 258          * version; MySQL 3 and MySQL 4 have different capabilities
 259          * in their fulltext search indexes.
 260          *
 261          * @param $filteredTerm String
 262          * @param $fulltext Boolean
 263          * @return String
 264          */
 265         function queryMain( $filteredTerm, $fulltext ) {
 266                 $match = $this->parseQuery( $filteredTerm, $fulltext );
 267                 $page        = $this->db->tableName( 'page' );
 268                 $searchindex = $this->db->tableName( 'searchindex' );
 269                 return 'SELECT page_id, page_namespace, page_title ' .
 270                         "FROM $page,$searchindex " .
 271                         'WHERE page_id=si_page AND ' . $match;
 272         }
 273
 274         function getCountQuery( $filteredTerm, $fulltext ) {
 275                 $match = $this->parseQuery( $filteredTerm, $fulltext );
 276                 $page        = $this->db->tableName( 'page' );
 277                 $searchindex = $this->db->tableName( 'searchindex' );
 278                 return "SELECT COUNT(*) AS c " .
 279                         "FROM $page,$searchindex " .
 280                         'WHERE page_id=si_page AND ' . $match .
 281                         $this->queryRedirect() . ' ' .
 282                         $this->queryNamespaces();
 283         }
 284
 285         /**
 286          * Create or update the search index record for the given page.
 287          * Title and text should be pre-processed.
 288          *
 289          * @param $id Integer
 290          * @param $title String
 291          * @param $text String
 292          */
 293         function update( $id, $title, $text ) {
 294                 $dbw = wfGetDB( DB_MASTER );
 295                 $dbw->replace( 'searchindex',
 296                         array( 'si_page' ),
 297                         array(
 298                                 'si_page' => $id,
 299                                 'si_title' => $this->normalizeText( $title ),
 300                                 'si_text' => $this->normalizeText( $text )
 301                         ), __METHOD__ );
 302         }
 303
 304         /**
 305          * Update a search index record's title only.
 306          * Title should be pre-processed.
 307          *
 308          * @param $id Integer
 309          * @param $title String
 310          */
 311     function updateTitle( $id, $title ) {
 312                 $dbw = wfGetDB( DB_MASTER );
 313
 314                 $dbw->update( 'searchindex',
 315                         array( 'si_title' => $this->normalizeText( $title ) ),
 316                         array( 'si_page'  => $id ),
 317                         __METHOD__,
 318                         array( $dbw->lowPriorityOption() ) );
 319         }
 320
 321         /**
 322          * Converts some characters for MySQL's indexing to grok it correctly,
 323          * and pads short words to overcome limitations.
 324          */
 325         function normalizeText( $string ) {
 326                 global $wgContLang;
 327
 328                 wfProfileIn( __METHOD__ );
 329
 330                 $out = parent::normalizeText( $string );
 331
 332                 // MySQL fulltext index doesn't grok utf-8, so we
 333                 // need to fold cases and convert to hex
 334                 $out = preg_replace_callback(
 335                         "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
 336                         array( $this, 'stripForSearchCallback' ),
 337                         $wgContLang->lc( $out ) );
 338
 339                 // And to add insult to injury, the default indexing
 340                 // ignores short words... Pad them so we can pass them
 341                 // through without reconfiguring the server...
 342                 $minLength = $this->minSearchLength();
 343                 if( $minLength > 1 ) {
 344                         $n = $minLength - 1;
 345                         $out = preg_replace(
 346                                 "/\b(\w{1,$n})\b/",
 347                                 "$1u800",
 348                                 $out );
 349                 }
 350
 351                 // Periods within things like hostnames and IP addresses
 352                 // are also important -- we want a search for "example.com"
 353                 // or "192.168.1.1" to work sanely.
 354                 //
 355                 // MySQL's search seems to ignore them, so you'd match on
 356                 // "example.wikipedia.com" and "192.168.83.1" as well.
 357                 $out = preg_replace(
 358                         "/(\w)\.(\w|\*)/u",
 359                         "$1u82e$2",
 360                         $out );
 361
 362                 wfProfileOut( __METHOD__ );
 363
 364                 return $out;
 365         }
 366
 367         /**
 368          * Armor a case-folded UTF-8 string to get through MySQL's
 369          * fulltext search without being mucked up by funny charset
 370          * settings or anything else of the sort.
 371          */
 372         protected function stripForSearchCallback( $matches ) {
 373                 return 'u8' . bin2hex( $matches[1] );
 374         }
 375
 376         /**
 377          * Check MySQL server's ft_min_word_len setting so we know
 378          * if we need to pad short words...
 379          *
 380          * @return int
 381          */
 382         protected function minSearchLength() {
 383                 if( is_null( self::$mMinSearchLength ) ) {
 384                         $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
 385
 386                         $dbr = wfGetDB( DB_SLAVE );
 387                         $result = $dbr->query( $sql );
 388                         $row = $result->fetchObject();
 389                         $result->free();
 390
 391                         if( $row && $row->Variable_name == 'ft_min_word_len' ) {
 392                                 self::$mMinSearchLength = intval( $row->Value );
 393                         } else {
 394                                 self::$mMinSearchLength = 0;
 395                         }
 396                 }
 397                 return self::$mMinSearchLength;
 398         }
 399 }
 400
 401 /**
 402  * @ingroup Search
 403  */
 404 class MySQLSearchResultSet extends SqlSearchResultSet {
 405         function __construct( $resultSet, $terms, $totalHits=null ) {
 406                 parent::__construct( $resultSet, $terms );
 407                 $this->mTotalHits = $totalHits;
 408         }
 409
 410         function getTotalHits() {
 411                 return $this->mTotalHits;
 412         }
 413 }