includes/deferred/SearchUpdate.php

   1 <?php
   2 /**
   3  * Search index updater
   4  *
   5  * See deferred.txt
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup Search
  24  */
  25
  26 use MediaWiki\MediaWikiServices;
  27
  28 /**
  29  * Database independant search index updater
  30  *
  31  * @ingroup Search
  32  */
  33 class SearchUpdate implements DeferrableUpdate {
  34         /** @var int Page id being updated */
  35         private $id = 0;
  36
  37         /** @var Title Title we're updating */
  38         private $title;
  39
  40         /** @var Content|null Content of the page (not text) */
  41         private $content;
  42
  43         /** @var WikiPage */
  44         private $page;
  45
  46         /**
  47          * @param int $id Page id to update
  48          * @param Title $title Title of page to update
  49          * @param Content|null $c Content of the page to update.
  50          */
  51         public function __construct( $id, $title, $c = null ) {
  52                 if ( is_string( $title ) ) {
  53                         wfDeprecated( __METHOD__ . " with a string for the title", 1.34 );
  54                         $this->title = Title::newFromText( $title );
  55                         if ( $this->title === null ) {
  56                                 throw new InvalidArgumentException( "Cannot construct the title: $title" );
  57                         }
  58                 } else {
  59                         $this->title = $title;
  60                 }
  61
  62                 $this->id = $id;
  63                 // is_string() check is back-compat for ApprovedRevs
  64                 if ( is_string( $c ) ) {
  65                         wfDeprecated( __METHOD__ . " with a string for the content", 1.34 );
  66                         $c = new TextContent( $c );
  67                 } elseif ( is_bool( $c ) ) {
  68                         wfDeprecated( __METHOD__ . " with a boolean for the content", 1.34 );
  69                         $c = null;
  70                 }
  71                 $this->content = $c;
  72         }
  73
  74         /**
  75          * Perform actual update for the entry
  76          */
  77         public function doUpdate() {
  78                 $services = MediaWikiServices::getInstance();
  79                 $config = $services->getSearchEngineConfig();
  80
  81                 if ( $config->getConfig()->get( 'DisableSearchUpdate' ) || !$this->id ) {
  82                         return;
  83                 }
  84
  85                 $seFactory = $services->getSearchEngineFactory();
  86                 foreach ( $config->getSearchTypes() as $type ) {
  87                         $search = $seFactory->create( $type );
  88                         if ( !$search->supports( 'search-update' ) ) {
  89                                 continue;
  90                         }
  91
  92                         $normalTitle = $this->getNormalizedTitle( $search );
  93
  94                         if ( $this->getLatestPage() === null ) {
  95                                 $search->delete( $this->id, $normalTitle );
  96                                 continue;
  97                         } elseif ( $this->content === null ) {
  98                                 $search->updateTitle( $this->id, $normalTitle );
  99                                 continue;
 100                         }
 101
 102                         $text = $this->content !== null ? $this->content->getTextForSearchIndex() : '';
 103                         $text = $this->updateText( $text, $search );
 104
 105                         # Perform the actual update
 106                         $search->update( $this->id, $normalTitle, $search->normalizeText( $text ) );
 107                 }
 108         }
 109
 110         /**
 111          * Clean text for indexing. Only really suitable for indexing in databases.
 112          * If you're using a real search engine, you'll probably want to override
 113          * this behavior and do something nicer with the original wikitext.
 114          * @param string $text
 115          * @param SearchEngine|null $se Search engine
 116          * @return string
 117          */
 118         public function updateText( $text, SearchEngine $se = null ) {
 119                 $services = MediaWikiServices::getInstance();
 120                 $contLang = $services->getContentLanguage();
 121                 # Language-specific strip/conversion
 122                 $text = $contLang->normalizeForSearch( $text );
 123                 $se = $se ?: $services->newSearchEngine();
 124                 $lc = $se->legalSearchChars() . '&#;';
 125
 126                 # Strip HTML markup
 127                 $text = preg_replace( "/<\\/?\\s*[A-Za-z][^>]*?>/",
 128                         ' ', $contLang->lc( " " . $text . " " ) );
 129                 $text = preg_replace( "/(^|\\n)==\\s*([^\\n]+)\\s*==(\\s)/sD",
 130                         "\\1\\2 \\2 \\2\\3", $text ); # Emphasize headings
 131
 132                 # Strip external URLs
 133                 $uc = "A-Za-z0-9_\\/:.,~%\\-+&;#?!=()@\\x80-\\xFF";
 134                 $protos = "http|https|ftp|mailto|news|gopher";
 135                 $pat = "/(^|[^\\[])({$protos}):[{$uc}]+([^{$uc}]|$)/";
 136                 $text = preg_replace( $pat, "\\1 \\3", $text );
 137
 138                 $p1 = "/([^\\[])\\[({$protos}):[{$uc}]+]/";
 139                 $p2 = "/([^\\[])\\[({$protos}):[{$uc}]+\\s+([^\\]]+)]/";
 140                 $text = preg_replace( $p1, "\\1 ", $text );
 141                 $text = preg_replace( $p2, "\\1 \\3 ", $text );
 142
 143                 # Internal image links
 144                 $pat2 = "/\\[\\[image:([{$uc}]+)\\.(gif|png|jpg|jpeg)([^{$uc}])/i";
 145                 $text = preg_replace( $pat2, " \\1 \\3", $text );
 146
 147                 $text = preg_replace( "/([^{$lc}])([{$lc}]+)]]([a-z]+)/",
 148                         "\\1\\2 \\2\\3", $text ); # Handle [[game]]s
 149
 150                 # Strip all remaining non-search characters
 151                 $text = preg_replace( "/[^{$lc}]+/", " ", $text );
 152
 153                 /**
 154                  * Handle 's, s'
 155                  *
 156                  *   $text = preg_replace( "/([{$lc}]+)'s /", "\\1 \\1's ", $text );
 157                  *   $text = preg_replace( "/([{$lc}]+)s' /", "\\1s ", $text );
 158                  *
 159                  * These tail-anchored regexps are insanely slow. The worst case comes
 160                  * when Japanese or Chinese text (ie, no word spacing) is written on
 161                  * a wiki configured for Western UTF-8 mode. The Unicode characters are
 162                  * expanded to hex codes and the "words" are very long paragraph-length
 163                  * monstrosities. On a large page the above regexps may take over 20
 164                  * seconds *each* on a 1GHz-level processor.
 165                  *
 166                  * Following are reversed versions which are consistently fast
 167                  * (about 3 milliseconds on 1GHz-level processor).
 168                  */
 169                 $text = strrev( preg_replace( "/ s'([{$lc}]+)/", " s'\\1 \\1", strrev( $text ) ) );
 170                 $text = strrev( preg_replace( "/ 's([{$lc}]+)/", " s\\1", strrev( $text ) ) );
 171
 172                 # Strip wiki '' and '''
 173                 $text = preg_replace( "/''[']*/", " ", $text );
 174
 175                 return $text;
 176         }
 177
 178         /**
 179          * Get WikiPage for the SearchUpdate $id using WikiPage::READ_LATEST
 180          * and ensure using the same WikiPage object if there are multiple
 181          * SearchEngine types.
 182          *
 183          * Returns null if a page has been deleted or is not found.
 184          *
 185          * @return WikiPage|null
 186          */
 187         private function getLatestPage() {
 188                 if ( !isset( $this->page ) ) {
 189                         $this->page = WikiPage::newFromID( $this->id, WikiPage::READ_LATEST );
 190                 }
 191
 192                 return $this->page;
 193         }
 194
 195         /**
 196          * Get a normalized string representation of a title suitable for
 197          * including in a search index
 198          *
 199          * @param SearchEngine $search
 200          * @return string A stripped-down title string ready for the search index
 201          */
 202         private function getNormalizedTitle( SearchEngine $search ) {
 203                 $contLang = MediaWikiServices::getInstance()->getContentLanguage();
 204                 $ns = $this->title->getNamespace();
 205                 $title = $this->title->getText();
 206
 207                 $lc = $search->legalSearchChars() . '&#;';
 208                 $t = $contLang->normalizeForSearch( $title );
 209                 $t = preg_replace( "/[^{$lc}]+/", ' ', $t );
 210                 $t = $contLang->lc( $t );
 211
 212                 # Handle 's, s'
 213                 $t = preg_replace( "/([{$lc}]+)'s( |$)/", "\\1 \\1's ", $t );
 214                 $t = preg_replace( "/([{$lc}]+)s'( |$)/", "\\1s ", $t );
 215
 216                 $t = preg_replace( "/\\s+/", ' ', $t );
 217
 218                 if ( $ns == NS_FILE ) {
 219                         $t = preg_replace( "/ (png|gif|jpg|jpeg|ogg)$/", "", $t );
 220                 }
 221
 222                 return $search->normalizeText( trim( $t ) );
 223         }
 224 }