4e01ee71abae6c844444db67b72ca26053aab857
[lhc/web/wiklou.git] / includes / SearchEngine.php
1 <?php
2 # See search.doc
3
4 class SearchEngine {
5 /* private */ var $mRawtext, $mUsertext, $mSearchterms;
6 /* private */ var $mTitlecond, $mTextcond;
7
8 var $doSearchRedirects = true;
9 var $addtoquery = array();
10 var $namespacesToSearch = array();
11 var $alternateTitle;
12 var $all_titles = false;
13
14 function SearchEngine( $text )
15 {
16 # We display the query, so let's strip it for safety
17 #
18 global $wgDBmysql4;
19 $lc = SearchEngine::legalSearchChars() . "()";
20 if( $wgDBmysql4 ) $lc .= "\"~<>*+-";
21 $this->mRawtext = $text;
22 $this->mUsertext = trim( preg_replace( "/[^{$lc}]/", " ", $text ) );
23 $this->mSearchterms = array();
24 $this->mStrictMatching = true; # Google-style, add '+' on all terms
25 }
26
27 function queryNamespaces()
28 {
29 $namespaces = implode( ",", $this->namespacesToSearch );
30 if ($namespaces == "") {
31 $namespaces = "0";
32 }
33 return "AND cur_namespace IN (" . $namespaces . ")";
34 }
35
36 function searchRedirects()
37 {
38 if ( $this->doSearchRedirects ) {
39 return "";
40 } else {
41 return "AND cur_is_redirect=0 ";
42 }
43 }
44
45 /* private */ function initNamespaceCheckbox( $i )
46 {
47 global $wgUser, $wgNamespacesToBeSearchedDefault;
48
49 if ($wgUser->getID()) {
50 // User is logged in so we retrieve his default namespaces
51 return $wgUser->getOption( "searchNs".$i );
52 } else {
53 // User is not logged in so we give him the global default namespaces
54 return !empty($wgNamespacesToBeSearchedDefault[ $i ]);
55 }
56 }
57
58 # Display the "power search" footer. Does not actually perform the search,
59 # that is done by showResults()
60 function powersearch()
61 {
62 global $wgUser, $wgOut, $wgLang, $wgTitle, $wgRequest;
63 $sk =& $wgUser->getSkin();
64
65 $search = $wgRequest->getText( 'search' );
66 $searchx = $wgRequest->getVal( 'searchx' );
67 $listredirs = $wgRequest->getVal( 'redirs' );
68
69 $ret = wfMsg("powersearchtext"); # Text to be returned
70 $tempText = ""; # Temporary text, for substitution into $ret
71
72 if( isset( $_REQUEST["searchx"] ) ) {
73 $this->addtoquery["searchx"] = "1";
74 }
75
76 # Do namespace checkboxes
77 $namespaces = $wgLang->getNamespaces();
78 foreach ( $namespaces as $i => $namespace ) {
79 # Skip virtual namespaces
80 if ( $i < 0 ) {
81 continue;
82 }
83
84 $formVar = "ns$i";
85
86 # Initialise checkboxValues, either from defaults or from
87 # a previous invocation
88 if ( !isset( $searchx ) ) {
89 $checkboxValue = $this->initNamespaceCheckbox( $i );
90 } else {
91 $checkboxValue = $wgRequest->getVal( $formVar );
92 }
93
94 $checked = "";
95 if ( $checkboxValue == 1 ) {
96 $checked = " checked='checked'";
97 $this->addtoquery["ns{$i}"] = 1;
98 array_push( $this->namespacesToSearch, $i );
99 }
100 $name = str_replace( "_", " ", $namespaces[$i] );
101 if ( "" == $name ) {
102 $name = wfMsg( "blanknamespace" );
103 }
104
105 if ( $tempText !== "" ) {
106 $tempText .= " ";
107 }
108 $tempText .= "<input type='checkbox' value=\"1\" name=\"" .
109 "ns{$i}\"{$checked} />{$name}\n";
110 }
111 $ret = str_replace ( "$1", $tempText, $ret );
112
113 # List redirects checkbox
114
115 $checked = "";
116 if ( $listredirs == 1 ) {
117 $this->addtoquery["redirs"] = 1;
118 $checked = " checked='checked'";
119 }
120 $tempText = "<input type='checkbox' value='1' name=\"redirs\"{$checked} />\n";
121 $ret = str_replace( "$2", $tempText, $ret );
122
123 # Search field
124
125 $tempText = "<input type='text' name=\"search\" value=\"" .
126 htmlspecialchars( $search ) ."\" width='80' />\n";
127 $ret = str_replace( "$3", $tempText, $ret );
128
129 # Searchx button
130
131 $tempText = "<input type='submit' name=\"searchx\" value=\"" .
132 wfMsg("powersearch") . "\" />\n";
133 $ret = str_replace( "$9", $tempText, $ret );
134
135 $action = $sk->escapeSearchLink();
136 $ret = "<br /><br />\n<form id=\"powersearch\" method=\"get\" " .
137 "action=\"$action\">\n{$ret}\n</form>\n";
138
139 if ( isset ( $searchx ) ) {
140 if ( ! $listredirs ) {
141 $this->doSearchRedirects = false;
142 }
143 }
144 return $ret;
145 }
146
147 function setupPage() {
148 global $wgOut;
149 $wgOut->setPageTitle( wfMsg( "searchresults" ) );
150 $q = wfMsg( "searchquery", htmlspecialchars( $this->mRawtext ) );
151 $wgOut->setSubtitle( $q );
152 $wgOut->setArticleRelated( false );
153 $wgOut->setRobotpolicy( "noindex,nofollow" );
154 }
155
156 # Perform the search and construct the results page
157 function showResults()
158 {
159 global $wgUser, $wgTitle, $wgOut, $wgLang, $wgRequest;
160 global $wgDisableTextSearch, $wgInputEncoding;
161 $fname = "SearchEngine::showResults";
162
163 $search = $wgRequest->getText( 'search' );
164
165 $powersearch = $this->powersearch(); /* Need side-effects here? */
166
167 $this->setupPage();
168
169 $sk = $wgUser->getSkin();
170 $header = wfMsg( "searchresulttext", $sk->makeKnownLink(
171 wfMsg( "searchhelppage" ), wfMsg( "searchingwikipedia" ) ) );
172 $wgOut->addHTML( $header );
173
174 $dbr =& wfGetDB( DB_SLAVE );
175 $this->parseQuery( $dbr );
176 if ( "" == $this->mTitlecond || "" == $this->mTextcond ) {
177 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
178 "<p>" . wfMsg( "badquerytext" ) . "</p>\n" );
179 return;
180 }
181 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
182
183 $searchnamespaces = $this->queryNamespaces();
184 $redircond = $this->searchRedirects();
185 $searchindex = $dbr->tableName( 'searchindex' );
186 $cur = $dbr->tableName( 'cur' );
187
188 if ( $wgDisableTextSearch ) {
189 $wgOut->addHTML( wfMsg( "searchdisabled" ) );
190 $wgOut->addHTML( wfMsg( "googlesearch", htmlspecialchars( $search ), $GLOBALS['wgInputEncoding'] ) );
191 } else {
192 $sql = "SELECT cur_id,cur_namespace,cur_title," .
193 "cur_text FROM $cur,$searchindex " .
194 "WHERE cur_id=si_page AND {$this->mTitlecond} " .
195 "{$searchnamespaces} {$redircond}" .
196 "LIMIT {$offset}, {$limit}";
197 $res1 = $dbr->query( $sql, $fname );
198 $num = $dbr->numRows($res1);
199
200 $sk = $wgUser->getSkin();
201 $text = "";
202
203 $this->parseQuery( $dbr );
204 if ( "" == $this->mTitlecond || "" == $this->mTextcond ) {
205 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
206 "<p>" . wfMsg( "badquerytext" ) . "</p>\n" );
207 return;
208 }
209 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
210
211 $searchnamespaces = $this->queryNamespaces();
212 $redircond = $this->searchRedirects();
213
214 $sql = "SELECT cur_id,cur_namespace,cur_title," .
215 "cur_text FROM $cur,$searchindex " .
216 "WHERE cur_id=si_page AND {$this->mTitlecond} " .
217 "{$searchnamespaces} {$redircond}" .
218 "LIMIT {$offset}, {$limit}";
219 $res1 = $dbr->query( $sql, $fname );
220 $num = $dbr->numRows($res1);
221
222 $sql = "SELECT cur_id,cur_namespace,cur_title," .
223 "cur_text FROM $cur,$searchindex " .
224 "WHERE cur_id=si_page AND {$this->mTextcond} " .
225 "{$searchnamespaces} {$redircond} " .
226 "LIMIT {$offset}, {$limit}";
227 $res2 = $dbr->query( $sql, $fname );
228 $num = $num + $dbr->numRows($res2);
229
230 if ( $num == $limit ) {
231 $top = wfShowingResults( $offset, $limit);
232 } else {
233 $top = wfShowingResultsNum( $offset, $limit, $num );
234 }
235 $wgOut->addHTML( "<p>{$top}</p>\n" );
236
237 # For powersearch
238
239 $a2l = "" ;
240 $akk = array_keys( $this->addtoquery ) ;
241 foreach ( $akk AS $ak ) {
242 $a2l .= "&{$ak}={$this->addtoquery[$ak]}" ;
243 }
244
245 $sl = wfViewPrevNext( $offset, $limit, "",
246 "search=" . wfUrlencode( $this->mUsertext ) . $a2l );
247 $wgOut->addHTML( "<br />{$sl}\n" );
248
249 $foundsome = false;
250
251 if ( 0 == $dbr->numRows( $res1 ) ) {
252 $wgOut->addHTML( "<h2>" . wfMsg( "notitlematches" ) .
253 "</h2>\n" );
254 } else {
255 $foundsome = true;
256 $off = $offset + 1;
257 $wgOut->addHTML( "<h2>" . wfMsg( "titlematches" ) .
258 "</h2>\n<ol start='{$off}'>" );
259
260 while ( $row = $dbr->fetchObject( $res1 ) ) {
261 $this->showHit( $row );
262 }
263 $dbr->freeResult( $res1 );
264 $wgOut->addHTML( "</ol>\n" );
265 }
266
267 if ( 0 == $dbr->numRows( $res2 ) ) {
268 $wgOut->addHTML( "<h2>" . wfMsg( "notextmatches" ) .
269 "</h2>\n" );
270 } else {
271 $foundsome = true;
272 $off = $offset + 1;
273 $wgOut->addHTML( "<h2>" . wfMsg( "textmatches" ) . "</h2>\n" .
274 "<ol start='{$off}'>" );
275 while ( $row = $dbr->fetchObject( $res2 ) ) {
276 $this->showHit( $row );
277 }
278 $dbr->freeResult( $res2 );
279 $wgOut->addHTML( "</ol>\n" );
280 }
281 if ( ! $foundsome ) {
282 $wgOut->addHTML( "<p>" . wfMsg( "nonefound" ) . "</p>\n" );
283 }
284 $wgOut->addHTML( "<p>{$sl}</p>\n" );
285 $wgOut->addHTML( $powersearch );
286 }
287 }
288
289 function legalSearchChars()
290 {
291 $lc = "A-Za-z_'0-9\\x80-\\xFF\\-";
292 return $lc;
293 }
294
295 function parseQuery( &$db )
296 {
297 global $wgDBminWordLen, $wgLang, $wgDBmysql4;
298
299 if( $wgDBmysql4 ) {
300 # Use cleaner boolean search if available
301 return $this->parseQuery4( $db );
302 }
303 # on non mysql4 database: get list of words we don't want to search for
304 require_once( "FulltextStoplist.php" );
305
306 $lc = SearchEngine::legalSearchChars() . "()";
307 $q = preg_replace( "/([()])/", " \\1 ", $this->mUsertext );
308 $q = preg_replace( "/\\s+/", " ", $q );
309 $w = explode( " ", trim( $q ) );
310
311 $last = $cond = "";
312 foreach ( $w as $word ) {
313 $word = $wgLang->stripForSearch( $word );
314 if ( "and" == $word || "or" == $word || "not" == $word
315 || "(" == $word || ")" == $word ) {
316 $cond .= " " . strtoupper( $word );
317 $last = "";
318 } else if ( strlen( $word ) < $wgDBminWordLen ) {
319 continue;
320 } else if ( FulltextStoplist::inList( $word ) ) {
321 continue;
322 } else {
323 if ( "" != $last ) { $cond .= " AND"; }
324 $cond .= " (MATCH (##field##) AGAINST ('" .
325 $db->strencode( $word ). "'))";
326 $last = $word;
327 array_push( $this->mSearchterms, "\\b" . $word . "\\b" );
328 }
329 }
330 if ( 0 == count( $this->mSearchterms ) ) { return; }
331
332 $this->mTitlecond = "(" . str_replace( "##field##",
333 "si_title", $cond ) . " )";
334
335 $this->mTextcond = "(" . str_replace( "##field##",
336 "si_text", $cond ) . " AND (cur_is_redirect=0) )";
337 }
338
339 function parseQuery4( &$db )
340 {
341 global $wgLang;
342 $lc = SearchEngine::legalSearchChars();
343 $searchon = "";
344 $this->mSearchterms = array();
345
346 # FIXME: This doesn't handle parenthetical expressions.
347 if( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
348 $this->mUsertext, $m, PREG_SET_ORDER ) ) {
349 foreach( $m as $terms ) {
350 if( $searchon !== "" ) $searchon .= " ";
351 if( $this->mStrictMatching && ($terms[1] == "") ) {
352 $terms[1] = "+";
353 }
354 $searchon .= $terms[1] . $wgLang->stripForSearch( $terms[2] );
355 if( $terms[3] ) {
356 $regexp = preg_quote( $terms[3] );
357 if( $terms[4] ) $regexp .= "[0-9A-Za-z_]+";
358 } else {
359 $regexp = preg_quote( str_replace( '"', '', $terms[2] ) );
360 }
361 $this->mSearchterms[] = $regexp;
362 }
363 wfDebug( "Would search with '$searchon'\n" );
364 wfDebug( "Match with /\b" . implode( '\b|\b', $this->mSearchterms ) . "\b/\n" );
365 } else {
366 wfDebug( "Can't understand search query '$this->mUsertext'\n" );
367 }
368
369 $searchon = $db->strencode( $searchon );
370 $this->mTitlecond = " MATCH(si_title) AGAINST('$searchon' IN BOOLEAN MODE)";
371 $this->mTextcond = " (MATCH(si_text) AGAINST('$searchon' IN BOOLEAN MODE) AND cur_is_redirect=0)";
372 }
373
374 function showHit( $row )
375 {
376 global $wgUser, $wgOut, $wgLang;
377
378 $t = Title::makeName( $row->cur_namespace, $row->cur_title );
379 $sk = $wgUser->getSkin();
380
381 $contextlines = $wgUser->getOption( "contextlines" );
382 if ( "" == $contextlines ) { $contextlines = 5; }
383 $contextchars = $wgUser->getOption( "contextchars" );
384 if ( "" == $contextchars ) { $contextchars = 50; }
385
386 $link = $sk->makeKnownLink( $t, "" );
387 $size = wfMsg( "nbytes", strlen( $row->cur_text ) );
388 $wgOut->addHTML( "<li>{$link} ({$size})" );
389
390 $lines = explode( "\n", $row->cur_text );
391 $pat1 = "/(.*)(" . implode( "|", $this->mSearchterms ) . ")(.*)/i";
392 $lineno = 0;
393
394 foreach ( $lines as $line ) {
395 if ( 0 == $contextlines ) {
396 break;
397 }
398 --$contextlines;
399 ++$lineno;
400 if ( ! preg_match( $pat1, $line, $m ) ) {
401 continue;
402 }
403
404 $pre = $wgLang->truncate( $m[1], -$contextchars, "..." );
405
406 if ( count( $m ) < 3 ) {
407 $post = "";
408 } else {
409 $post = $wgLang->truncate( $m[3], $contextchars, "..." );
410 }
411
412 $found = $m[2];
413
414 $line = htmlspecialchars( $pre . $found . $post );
415 $pat2 = "/(" . implode( "|", $this->mSearchterms ) . ")/i";
416 $line = preg_replace( $pat2,
417 "<font color='red'>\\1</font>", $line );
418
419 $wgOut->addHTML( "<br /><small>{$lineno}: {$line}</small>\n" );
420 }
421 $wgOut->addHTML( "</li>\n" );
422 }
423
424 function goResult()
425 {
426 global $wgOut, $wgRequest, $wgGoToEdit;
427 global $wgDisableTextSearch;
428 $fname = "SearchEngine::goResult";
429
430 $search = trim( $wgRequest->getText( "search" ) );
431
432 # Try to go to page as entered.
433 #
434 $t = Title::newFromText( $search );
435
436 # If the string cannot be used to create a title
437 if( false == $t ){
438 $this->showResults();
439 return;
440 }
441
442 # Exact match? No need to look further.
443 if ( $t->getNamespace() == NS_SPECIAL || 0 != $t->getArticleID() ) {
444 $wgOut->redirect( $t->getFullURL() );
445 return;
446 }
447
448 # Now try all lower case (i.e. first letter capitalized)
449 #
450 $t = Title::newFromText( strtolower( $search ) );
451 if ( 0 != $t->getArticleID() ) {
452 $wgOut->redirect( $t->getFullURL() );
453 return;
454 }
455
456 # Now try capitalized string
457 #
458 $t = Title::newFromText( ucwords( strtolower( $search ) ) );
459 if ( 0 != $t->getArticleID() ) {
460 $wgOut->redirect( $t->getFullURL() );
461 return;
462 }
463
464 # Now try all upper case
465 #
466 $t = Title::newFromText( strtoupper( $search ) );
467 if ( 0 != $t->getArticleID() ) {
468 $wgOut->redirect( $t->getFullURL() );
469 return;
470 }
471
472 # Entering an IP address goes to the contributions page
473 if ( preg_match( '/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/', $search ) ) {
474 $title = Title::makeTitle( NS_SPECIAL, "Contributions" );
475 $wgOut->redirect( $title->getFullUrl( "target=$search" ) );
476 return;
477 }
478
479 # No match, generate an edit URL
480 $t = Title::newFromText( $this->mRawtext );
481
482 # If the feature is enabled, go straight to the edit page
483 if ( $wgGoToEdit ) {
484 $wgOut->redirect( $t->getFullURL( "action=edit" ) );
485 return;
486 }
487
488 if( $t ) {
489 $editurl = $t->escapeLocalURL( "action=edit" );
490 } else {
491 $editurl = ""; # ??
492 }
493 $wgOut->addHTML( "<p>" . wfMsg("nogomatch", $editurl ) . "</p>\n" );
494
495 # Try a fuzzy title search
496 $anyhit = false;
497 global $wgDisableFuzzySearch;
498 if(! $wgDisableFuzzySearch ){
499 foreach( array(NS_MAIN, NS_WP, NS_USER, NS_IMAGE, NS_MEDIAWIKI) as $namespace){
500 $anyhit |= SearchEngine::doFuzzyTitleSearch( $search, $namespace );
501 }
502 }
503
504 if( ! $anyhit ){
505 return $this->showResults();
506 }
507 }
508
509 /* static */ function doFuzzyTitleSearch( $search, $namespace ){
510 global $wgLang, $wgOut;
511
512 $this->setupPage();
513
514 $sstr = ucfirst($search);
515 $sstr = str_replace(" ", "_", $sstr);
516 $fuzzymatches = SearchEngine::fuzzyTitles( $sstr, $namespace );
517 $fuzzymatches = array_slice($fuzzymatches, 0, 10);
518 $slen = strlen( $search );
519 $wikitext = "";
520 foreach($fuzzymatches as $res){
521 $t = str_replace("_", " ", $res[1]);
522 $tfull = $wgLang->getNsText( $namespace ) . ":$t|$t";
523 if( $namespace == NS_MAIN )
524 $tfull = "$t";
525 $distance = $res[0];
526 $closeness = (strlen( $search ) - $distance) / strlen( $search );
527 $percent = intval( $closeness * 100 ) . "%";
528 $stars = str_repeat("*", ceil(5 * $closeness) );
529 $wikitext .= "* [[$tfull]] $percent ($stars)\n";
530 }
531 if( $wikitext ){
532 if( $namespace != NS_MAIN )
533 $wikitext = "=== " . $wgLang->getNsText( $namespace ) . " ===\n" . $wikitext;
534 $wgOut->addWikiText( $wikitext );
535 return true;
536 }
537 return false;
538 }
539
540 /* static */ function fuzzyTitles( $sstr, $namespace = NS_MAIN ){
541 $span = 0.10; // weed on title length before doing levenshtein.
542 $tolerance = 0.35; // allowed percentage of erronous characters
543 $slen = strlen($sstr);
544 $tolerance_count = ceil($tolerance * $slen);
545 $spanabs = ceil($slen * (1 + $span)) - $slen;
546 # print "Word: $sstr, len = $slen, range = [$min, $max], tolerance_count = $tolerance_count<BR>\n";
547 $result = array();
548 $cnt = 0;
549 for( $i=0; $i <= $spanabs; $i++ ){
550 $titles = SearchEngine::getTitlesByLength( $slen + $i, $namespace );
551 if( $i != 0) {
552 $titles = array_merge($titles, SearchEngine::getTitlesByLength( $slen - $i, $namespace ) );
553 }
554 foreach($titles as $t){
555 $d = levenshtein($sstr, $t);
556 if($d < $tolerance_count)
557 $result[] = array($d, $t);
558 $cnt++;
559 }
560 }
561 usort($result, "SearchEngine_pcmp");
562 return $result;
563 }
564
565 /* static */ function getTitlesByLength($aLength, $aNamespace = 0){
566 global $wgMemc, $wgDBname;
567 $fname = 'SearchEngin::getTitlesByLength';
568
569 // to avoid multiple costly SELECTs in case of no memcached
570 if( $this->all_titles ){
571 if( isset( $this->all_titles[$aLength][$aNamespace] ) ){
572 return $this->all_titles[$aLength][$aNamespace];
573 } else {
574 return array();
575 }
576 }
577
578 $mkey = "$wgDBname:titlesbylength:$aLength:$aNamespace";
579 $mkeyts = "$wgDBname:titlesbylength:createtime";
580 $ts = $wgMemc->get( $mkeyts );
581 $result = $wgMemc->get( $mkey );
582
583 if( time() - $ts < 3600 ){
584 // note: in case of insufficient memcached space, we return
585 // an empty list instead of starting to hit the DB.
586 return is_array( $result ) ? $result : array();
587 }
588
589 $wgMemc->set( $mkeyts, time() );
590
591 $dbr =& wfGetDB( DB_SLAVE );
592 $res = $dbr->select( 'cur', array( 'cur_title', 'cur_namespace' ), false, $fname );
593 $titles = array(); // length, ns, [titles]
594 while( $obj = $dbr->fetchObject( $res ) ){
595 $title = $obj->cur_title;
596 $ns = $obj->cur_namespace;
597 $len = strlen( $title );
598 $titles[$len][$ns][] = $title;
599 }
600 foreach($titles as $length => $length_arr){
601 foreach($length_arr as $ns => $title_arr){
602 $mkey = "$wgDBname:titlesbylength:$length:$ns";
603 $wgMemc->set( $mkey, $title_arr, 3600 * 24 );
604 }
605 }
606 $this->all_titles = $titles;
607 if( isset( $titles[$aLength][$aNamespace] ) )
608 return $titles[$aLength][$aNamespace];
609 else
610 return array();
611 }
612 }
613
614 /* private static */ function SearchEngine_pcmp($a, $b){ return $a[0] - $b[0]; }
615
616 ?>