maintenance/generateSitemap.php

   1 <?php
   2 define( 'GS_MAIN', -2 );
   3 define( 'GS_TALK', -1 );
   4 /**
   5  * Creates a Google sitemap for the site
   6  *
   7  * @package MediaWiki
   8  * @subpackage Maintenance
   9  *
  10  * @copyright Copyright © 2005, Ævar Arnfjörð Bjarmason
  11  * @copyright Copyright © 2005, Jens Frank <jeluf@gmx.de>
  12  * @copyright Copyright © 2005, Brion Vibber <brion@pobox.com>
  13  *
  14  * @link http://www.google.com/webmasters/sitemaps/docs/en/about.html
  15  * @link http://www.google.com/schemas/sitemap/0.84/sitemap.xsd
  16  *
  17  * @license http://www.gnu.org/copyleft/gpl.html GNU General Public License 2.0 or later
  18  */
  19
  20 class GenerateSitemap {
  21         /**
  22          * The path to prepend to the filename
  23          *
  24          * @var string
  25          */
  26         var $fspath;
  27
  28         /**
  29          * The path to append to the domain name
  30          *
  31          * @var string
  32          */
  33         var $path;
  34
  35         /**
  36          * Whether or not to use compression
  37          *
  38          * @var bool
  39          */
  40         var $compress;
  41
  42         /**
  43          * The number of entries to save in each sitemap file
  44          *
  45          * @var int
  46          */
  47         var $limit;
  48
  49         /**
  50          * Key => value entries of namespaces and their priorities
  51          *
  52          * @var array
  53          */
  54         var $priorities = array(
  55                 // Custom main namespaces
  56                 GS_MAIN                 => '0.5',
  57                 // Custom talk namesspaces
  58                 GS_TALK                 => '0.1',
  59                 // MediaWiki standard namespaces
  60                 NS_MAIN                 => '1.0',
  61                 NS_TALK                 => '0.1',
  62                 NS_USER                 => '0.5',
  63                 NS_USER_TALK            => '0.1',
  64                 NS_PROJECT              => '0.5',
  65                 NS_PROJECT_TALK         => '0.1',
  66                 NS_IMAGE                => '0.5',
  67                 NS_IMAGE_TALK           => '0.1',
  68                 NS_MEDIAWIKI            => '0.0',
  69                 NS_MEDIAWIKI_TALK       => '0.1',
  70                 NS_TEMPLATE             => '0.0',
  71                 NS_TEMPLATE_TALK        => '0.1',
  72                 NS_HELP                 => '0.5',
  73                 NS_HELP_TALK            => '0.1',
  74                 NS_CATEGORY             => '0.5',
  75                 NS_CATEGORY_TALK        => '0.1',
  76         );
  77
  78         /**
  79          * A one-dimensional array of namespaces in the wiki
  80          *
  81          * @var array
  82          */
  83         var $namespaces = array();
  84
  85         /**
  86          * When this sitemap batch was generated
  87          *
  88          * @var string
  89          */
  90         var $timestamp;
  91
  92         /**
  93          * A database slave object
  94          *
  95          * @var object
  96          */
  97         var $dbr;
  98
  99         /**
 100          * A resource pointing to the sitemap index file
 101          *
 102          * @var resource
 103          */
 104         var $findex;
 105
 106
 107         /**
 108          * A resource pointing to a sitemap file
 109          *
 110          * @var resource
 111          */
 112         var $file;
 113
 114         /**
 115          * A resource pointing to php://stderr
 116          *
 117          * @var resource
 118          */
 119         var $stderr;
 120
 121         /**
 122          * Constructor
 123          *
 124          * @param string $fspath The path to prepend to the filenames, used to
 125          *                     save them somewhere else than in the root directory
 126          * @param string $path The path to append to the domain name
 127          * @param bool $compress Whether to compress the sitemap files
 128          */
 129         function GenerateSitemap( $fspath, $path, $compress ) {
 130                 global $wgDBname, $wgScriptPath;
 131
 132                 $this->fspath = isset( $fspath ) ? $fspath : '';
 133                 $this->path = isset( $path ) ? $path : $wgScriptPath;
 134                 $this->compress = $compress;
 135
 136                 $this->stderr = fopen( 'php://stderr', 'wt' );
 137                 $this->dbr =& wfGetDB( DB_SLAVE );
 138                 $this->generateNamespaces();
 139                 $this->generateLimit( NS_MAIN );
 140                 $this->findex = fopen( "{$this->fspath}sitemap-index-$wgDBname.xml", 'wb' );
 141         }
 142
 143         /**
 144          * Generate a one-dimensional array of existing namespaces
 145          */
 146         function generateNamespaces() {
 147                 $fname = 'GenerateSitemap::generateNamespaces';
 148
 149                 $res = $this->dbr->select( 'page',
 150                         array( 'page_namespace' ),
 151                         array(),
 152                         $fname,
 153                         array(
 154                                 'GROUP BY' => 'page_namespace',
 155                                 'ORDER BY' => 'page_namespace',
 156                         )
 157                 );
 158
 159                 while ( $row = $this->dbr->fetchObject( $res ) )
 160                         $this->namespaces[] = $row->page_namespace;
 161         }
 162
 163         /**
 164          * Get the priority of a given namespace
 165          *
 166          * @param int $namespace The namespace to get the priority for
 167          +
 168          * @return string
 169          */
 170
 171         function priority( $namespace ) {
 172                 return isset( $this->priorities[$namespace] ) ? $this->priorities[$namespace] : $this->guessPriority( $namespace );
 173         }
 174
 175         /**
 176          * If the namespace isn't listed on the priority list return the
 177          * default priority for the namespace, varies depending on whether it's
 178          * a talkpage or not.
 179          *
 180          * @param int $namespace The namespace to get the priority for
 181          *
 182          * @return string
 183          */
 184         function guessPriority( $namespace ) {
 185                 return Namespace::isMain( $namespace ) ? $this->priorities[GS_MAIN] : $this->priorities[GS_TALK];
 186         }
 187
 188         /**
 189          * Return a database resolution of all the pages in a given namespace
 190          *
 191          * @param int $namespace Limit the query to this namespace
 192          *
 193          * @return resource
 194          */
 195         function getPageRes( $namespace ) {
 196                 $fname = 'GenerateSitemap::getPageRes';
 197
 198                 return $this->dbr->select( 'page',
 199                         array(
 200                                 'page_namespace',
 201                                 'page_title',
 202                                 'page_is_redirect',
 203                                 'page_touched',
 204                         ),
 205                         array( 'page_namespace' => $namespace ),
 206                         $fname
 207                 );
 208         }
 209
 210         /**
 211          * Main loop
 212          *
 213          * @access public
 214          */
 215         function main() {
 216                 global $wgDBname;
 217
 218                 fwrite( $this->findex, $this->openIndex() );
 219
 220                 foreach ( $this->namespaces as $namespace ) {
 221                         $res = $this->getPageRes( $namespace );
 222                         $this->file = false;
 223                         $i = $smcount = 0;
 224
 225                         $this->debug( $namespace );
 226                         while ( $row = $this->dbr->fetchObject( $res ) ) {
 227                                 if ( $i++ % $this->limit === 0 ) {
 228                                         if ( $this->file !== false ) {
 229                                                 $this->write( $this->file, $this->closeFile() );
 230                                                 $this->close( $this->file );
 231                                         }
 232                                         $this->generateLimit( $namespace );
 233                                         $this->generateTimestamp();
 234                                         $filename = $this->sitemapFilename( $namespace, $smcount++ );
 235                                         $this->file = $this->open( $this->fspath . $filename, 'wb' );
 236                                         $this->write( $this->file, $this->openFile() );
 237                                         fwrite( $this->findex, $this->indexEntry( $filename ) );
 238                                         $this->debug( "\t$filename" );
 239                                 }
 240                                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 241                                 $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
 242                                 $this->write( $this->file, $this->fileEntry( $title->getFullURL(), $date, $this->priority( $namespace ) ) );
 243                         }
 244                         if ( $this->file ) {
 245                                 $this->write( $this->file, $this->closeFile() );
 246                                 $this->close( $this->file );
 247                         }
 248                 }
 249                 fwrite( $this->findex, $this->closeIndex() );
 250                 fclose( $this->findex );
 251         }
 252
 253         /**
 254          * gzopen() / fopen() wrapper
 255          *
 256          * @return resource
 257          */
 258         function open( $file, $flags ) {
 259                 return $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
 260         }
 261
 262         /**
 263          * gzwrite() / fwrite() wrapper
 264          */
 265         function write( &$handle, $str ) {
 266                 if ( $this->compress )
 267                         gzwrite( $handle, $str );
 268                 else
 269                         fwrite( $handle, $str );
 270         }
 271
 272         /**
 273          * gzclose() / fclose() wrapper
 274          */
 275         function close( &$handle ) {
 276                 if ( $this->compress )
 277                         gzclose( $handle );
 278                 else
 279                         fclose( $handle );
 280         }
 281
 282         /**
 283          * Get a sitemap filename
 284          *
 285          * @static
 286          *
 287          * @param int $namespace The namespace
 288          * @param int $count The count
 289          *
 290          * @return string
 291          */
 292         function sitemapFilename( $namespace, $count ) {
 293                 global $wgDBname;
 294
 295                 $ext = $this->compress ? '.gz' : '';
 296
 297                 return "sitemap-$wgDBname-NS_$namespace-$count.xml$ext";
 298         }
 299
 300         /**
 301          * Return the XML required to open an XML file
 302          *
 303          * @static
 304          *
 305          * @return string
 306          */
 307         function xmlHead() {
 308                 return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
 309         }
 310
 311         /**
 312          * Return the XML schema being used
 313          *
 314          * @static
 315          *
 316          * @returns string
 317          */
 318         function xmlSchema() {
 319                 return 'http://www.google.com/schemas/sitemap/0.84';
 320         }
 321
 322         /**
 323          * Return the XML required to open a sitemap index file
 324          *
 325          * @return string
 326          */
 327         function openIndex() {
 328                 return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
 329         }
 330
 331         /**
 332          * Return the XML for a single sitemap indexfile entry
 333          *
 334          * @static
 335          *
 336          * @param string $filename The filename of the sitemap file
 337          *
 338          * @return string
 339          */
 340         function indexEntry( $filename ) {
 341                 global $wgServer;
 342
 343                 return
 344                         "\t<sitemap>\n" .
 345                         "\t\t<loc>$wgServer{$this->path}/$filename</log>\n" .
 346                         "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
 347                         "\t</sitemap>\n";
 348         }
 349
 350         /**
 351          * Return the XML required to close a sitemap index file
 352          *
 353          * @static
 354          *
 355          * @return string
 356          */
 357         function closeIndex() {
 358                 return "</sitemapindex>\n";
 359         }
 360
 361         /**
 362          * Return the XML required to open a sitemap file
 363          *
 364          * @return string
 365          */
 366         function openFile() {
 367                 return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
 368         }
 369
 370         /**
 371          * Return the XML for a single sitemap entry
 372          *
 373          * @static
 374          *
 375          * @param string $url An RFC 2396 compilant URL
 376          * @param string $date A ISO 8601 date
 377          * @param string $priority A priority indicator, 0.0 - 1.0 inclusive with a 0.1 stepsize
 378          *
 379          * @return string
 380          */
 381         function fileEntry( $url, $date, $priority ) {
 382                 return
 383                         "\t<url>\n" .
 384                         "\t\t<loc>$url</loc>\n" .
 385                         "\t\t<lastmod>$date</lastmod>\n" .
 386                         "\t\t<priority>$priority</priority>\n" .
 387                         "\t</url>\n";
 388         }
 389
 390         /**
 391          * Return the XML required to close sitemap file
 392          *
 393          * @static
 394          * @return string
 395          */
 396         function closeFile() {
 397                 return "</urlset>\n";
 398         }
 399
 400         /**
 401          * Write a string to stderr followed by a UNIX newline
 402          */
 403         function debug( $str ) {
 404                 fwrite( $this->stderr, "$str\n" );
 405         }
 406
 407         /**
 408          * According to the sitemap specification each sitemap must contain no
 409          * more than 50,000 urls and no more than 2^20 bytes (10MB), this
 410          * function calculates how many urls we can have in each file assuming
 411          * that we have the worst case of 63 four byte characters and 1 three
 412          * byte character in the title (63*4+1*3 = 255)
 413          */
 414         function generateLimit( $namespace ) {
 415                 $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
 416
 417                 $olen = strlen( $this->openFile() );
 418                 $elen = strlen( $this->fileEntry( $title->getFullUrl(), wfTimestamp( TS_ISO_8601, wfTimestamp() ), '1.0' ) );
 419                 $clen = strlen( $this->closeFile() );
 420
 421                 for ( $i = 1, $etot = $elen; ( $olen + $clen + $etot + $elen ) <= pow( 2, 20 ); ++$i )
 422                         $etot += $elen;
 423
 424                 $this->limit = $i;
 425         }
 426
 427         /**
 428          * Update $this->timestamp to the current time
 429          */
 430         function generateTimestamp() {
 431                 $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
 432         }
 433 }
 434
 435 if ( in_array( '--help', $argv ) )
 436         die(
 437                 "Usage: php generateSitemap.php [host] [options]\n" .
 438                 "\thost = hostname\n" .
 439                 "\toptions:\n" .
 440                 "\t\t--help\tshow this message\n" .
 441                 "\t\t--fspath\tThe file system path to save to, e.g /tmp/sitemap/\n" .
 442                 "\t\t--path\tThe http path to use, e.g. /wiki\n" .
 443                 "\t\t--compress=[yes|no]\tcompress the sitemap files, default yes\n"
 444         );
 445
 446 if ( isset( $argv[1] ) && strpos( $argv[1], '--' ) !== 0 )
 447         $_SERVER['SERVER_NAME'] = $argv[1];
 448
 449 $optionsWithArgs = array( 'fspath', 'path', 'compress' );
 450 require_once 'commandLine.inc';
 451
 452 $gs = new GenerateSitemap( @$options['fspath'], @$options['path'], @$options['compress'] !== 'no' );
 453 $gs->main();
 454 ?>