* Added wfDie() wrapper, and some manual die(-1), to force the return code
[lhc/web/wiklou.git] / maintenance / generateSitemap.php
1 <?php
2 define( 'GS_MAIN', -2 );
3 define( 'GS_TALK', -1 );
4 /**
5 * Creates a Google sitemap for the site
6 *
7 * @package MediaWiki
8 * @subpackage Maintenance
9 *
10 * @copyright Copyright © 2005, Ævar Arnfjörð Bjarmason
11 * @copyright Copyright © 2005, Jens Frank <jeluf@gmx.de>
12 * @copyright Copyright © 2005, Brion Vibber <brion@pobox.com>
13 *
14 * @link http://www.google.com/webmasters/sitemaps/docs/en/about.html
15 * @link http://www.google.com/schemas/sitemap/0.84/sitemap.xsd
16 *
17 * @license http://www.gnu.org/copyleft/gpl.html GNU General Public License 2.0 or later
18 */
19
20 class GenerateSitemap {
21 /**
22 * The maximum amount of urls in a sitemap file
23 *
24 * @link http://www.google.com/schemas/sitemap/0.84/sitemap.xsd
25 *
26 * @var int
27 */
28 var $url_limit;
29
30 /**
31 * The maximum size of a sitemap file
32 *
33 * @link http://www.google.com/webmasters/sitemaps/docs/en/protocol.html#faq_sitemap_size
34 *
35 * @var int
36 */
37 var $size_limit;
38
39 /**
40 * The path to prepend to the filename
41 *
42 * @var string
43 */
44 var $fspath;
45
46 /**
47 * The path to append to the domain name
48 *
49 * @var string
50 */
51 var $path;
52
53 /**
54 * Whether or not to use compression
55 *
56 * @var bool
57 */
58 var $compress;
59
60 /**
61 * The number of entries to save in each sitemap file
62 *
63 * @var array
64 */
65 var $limit = array();
66
67 /**
68 * Key => value entries of namespaces and their priorities
69 *
70 * @var array
71 */
72 var $priorities = array(
73 // Custom main namespaces
74 GS_MAIN => '0.5',
75 // Custom talk namesspaces
76 GS_TALK => '0.1',
77 // MediaWiki standard namespaces
78 NS_MAIN => '1.0',
79 NS_TALK => '0.1',
80 NS_USER => '0.5',
81 NS_USER_TALK => '0.1',
82 NS_PROJECT => '0.5',
83 NS_PROJECT_TALK => '0.1',
84 NS_IMAGE => '0.5',
85 NS_IMAGE_TALK => '0.1',
86 NS_MEDIAWIKI => '0.0',
87 NS_MEDIAWIKI_TALK => '0.1',
88 NS_TEMPLATE => '0.0',
89 NS_TEMPLATE_TALK => '0.1',
90 NS_HELP => '0.5',
91 NS_HELP_TALK => '0.1',
92 NS_CATEGORY => '0.5',
93 NS_CATEGORY_TALK => '0.1',
94 );
95
96 /**
97 * A one-dimensional array of namespaces in the wiki
98 *
99 * @var array
100 */
101 var $namespaces = array();
102
103 /**
104 * When this sitemap batch was generated
105 *
106 * @var string
107 */
108 var $timestamp;
109
110 /**
111 * A database slave object
112 *
113 * @var object
114 */
115 var $dbr;
116
117 /**
118 * A resource pointing to the sitemap index file
119 *
120 * @var resource
121 */
122 var $findex;
123
124
125 /**
126 * A resource pointing to a sitemap file
127 *
128 * @var resource
129 */
130 var $file;
131
132 /**
133 * A resource pointing to php://stderr
134 *
135 * @var resource
136 */
137 var $stderr;
138
139 /**
140 * Constructor
141 *
142 * @param string $fspath The path to prepend to the filenames, used to
143 * save them somewhere else than in the root directory
144 * @param string $path The path to append to the domain name
145 * @param bool $compress Whether to compress the sitemap files
146 */
147 function GenerateSitemap( $fspath, $path, $compress ) {
148 global $wgDBname, $wgScriptPath;
149
150 $this->url_limit = 50000;
151 $this->size_limit = pow( 2, 20 ) * 10;
152 $this->fspath = isset( $fspath ) ? $fspath : '';
153 $this->path = isset( $path ) ? $path : $wgScriptPath;
154 $this->compress = $compress;
155
156 $this->stderr = fopen( 'php://stderr', 'wt' );
157 $this->dbr =& wfGetDB( DB_SLAVE );
158 $this->generateNamespaces();
159 $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
160 $this->findex = fopen( "{$this->fspath}sitemap-index-$wgDBname.xml", 'wb' );
161 }
162
163 /**
164 * Generate a one-dimensional array of existing namespaces
165 */
166 function generateNamespaces() {
167 $fname = 'GenerateSitemap::generateNamespaces';
168
169 $res = $this->dbr->select( 'page',
170 array( 'page_namespace' ),
171 array(),
172 $fname,
173 array(
174 'GROUP BY' => 'page_namespace',
175 'ORDER BY' => 'page_namespace',
176 )
177 );
178
179 while ( $row = $this->dbr->fetchObject( $res ) )
180 $this->namespaces[] = $row->page_namespace;
181 }
182
183 /**
184 * Get the priority of a given namespace
185 *
186 * @param int $namespace The namespace to get the priority for
187 +
188 * @return string
189 */
190
191 function priority( $namespace ) {
192 return isset( $this->priorities[$namespace] ) ? $this->priorities[$namespace] : $this->guessPriority( $namespace );
193 }
194
195 /**
196 * If the namespace isn't listed on the priority list return the
197 * default priority for the namespace, varies depending on whether it's
198 * a talkpage or not.
199 *
200 * @param int $namespace The namespace to get the priority for
201 *
202 * @return string
203 */
204 function guessPriority( $namespace ) {
205 return Namespace::isMain( $namespace ) ? $this->priorities[GS_MAIN] : $this->priorities[GS_TALK];
206 }
207
208 /**
209 * Return a database resolution of all the pages in a given namespace
210 *
211 * @param int $namespace Limit the query to this namespace
212 *
213 * @return resource
214 */
215 function getPageRes( $namespace ) {
216 $fname = 'GenerateSitemap::getPageRes';
217
218 return $this->dbr->select( 'page',
219 array(
220 'page_namespace',
221 'page_title',
222 'page_touched',
223 ),
224 array( 'page_namespace' => $namespace ),
225 $fname
226 );
227 }
228
229 /**
230 * Main loop
231 *
232 * @access public
233 */
234 function main() {
235 global $wgDBname, $wgContLang;
236
237 fwrite( $this->findex, $this->openIndex() );
238
239 foreach ( $this->namespaces as $namespace ) {
240 $res = $this->getPageRes( $namespace );
241 $this->file = false;
242 $this->generateLimit( $namespace );
243 $length = $this->limit[0];
244 $i = $smcount = 0;
245
246 $fns = $wgContLang->getFormattedNsText( $namespace );
247 $this->debug( "$namespace ($fns)" );
248 while ( $row = $this->dbr->fetchObject( $res ) ) {
249 if ( $i++ === 0 || $i === $this->url_limit + 1 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit ) {
250 if ( $this->file !== false ) {
251 $this->write( $this->file, $this->closeFile() );
252 $this->close( $this->file );
253 }
254 $filename = $this->sitemapFilename( $namespace, $smcount++ );
255 $this->file = $this->open( $this->fspath . $filename, 'wb' );
256 $this->write( $this->file, $this->openFile() );
257 fwrite( $this->findex, $this->indexEntry( $filename ) );
258 $this->debug( "\t$filename" );
259 $length = $this->limit[0];
260 $i = 1;
261 }
262 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
263 $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
264 $entry = $this->fileEntry( $title->getFullURL(), $date, $this->priority( $namespace ) );
265 $length += strlen( $entry );
266 $this->write( $this->file, $entry );
267 }
268 if ( $this->file ) {
269 $this->write( $this->file, $this->closeFile() );
270 $this->close( $this->file );
271 }
272 }
273 fwrite( $this->findex, $this->closeIndex() );
274 fclose( $this->findex );
275 }
276
277 /**
278 * gzopen() / fopen() wrapper
279 *
280 * @return resource
281 */
282 function open( $file, $flags ) {
283 return $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
284 }
285
286 /**
287 * gzwrite() / fwrite() wrapper
288 */
289 function write( &$handle, $str ) {
290 if ( $this->compress )
291 gzwrite( $handle, $str );
292 else
293 fwrite( $handle, $str );
294 }
295
296 /**
297 * gzclose() / fclose() wrapper
298 */
299 function close( &$handle ) {
300 if ( $this->compress )
301 gzclose( $handle );
302 else
303 fclose( $handle );
304 }
305
306 /**
307 * Get a sitemap filename
308 *
309 * @static
310 *
311 * @param int $namespace The namespace
312 * @param int $count The count
313 *
314 * @return string
315 */
316 function sitemapFilename( $namespace, $count ) {
317 global $wgDBname;
318
319 $ext = $this->compress ? '.gz' : '';
320
321 return "sitemap-$wgDBname-NS_$namespace-$count.xml$ext";
322 }
323
324 /**
325 * Return the XML required to open an XML file
326 *
327 * @static
328 *
329 * @return string
330 */
331 function xmlHead() {
332 return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
333 }
334
335 /**
336 * Return the XML schema being used
337 *
338 * @static
339 *
340 * @returns string
341 */
342 function xmlSchema() {
343 return 'http://www.google.com/schemas/sitemap/0.84';
344 }
345
346 /**
347 * Return the XML required to open a sitemap index file
348 *
349 * @return string
350 */
351 function openIndex() {
352 return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
353 }
354
355 /**
356 * Return the XML for a single sitemap indexfile entry
357 *
358 * @static
359 *
360 * @param string $filename The filename of the sitemap file
361 *
362 * @return string
363 */
364 function indexEntry( $filename ) {
365 return
366 "\t<sitemap>\n" .
367 "\t\t<loc>$filename</loc>\n" .
368 "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
369 "\t</sitemap>\n";
370 }
371
372 /**
373 * Return the XML required to close a sitemap index file
374 *
375 * @static
376 *
377 * @return string
378 */
379 function closeIndex() {
380 return "</sitemapindex>\n";
381 }
382
383 /**
384 * Return the XML required to open a sitemap file
385 *
386 * @return string
387 */
388 function openFile() {
389 return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
390 }
391
392 /**
393 * Return the XML for a single sitemap entry
394 *
395 * @static
396 *
397 * @param string $url An RFC 2396 compilant URL
398 * @param string $date A ISO 8601 date
399 * @param string $priority A priority indicator, 0.0 - 1.0 inclusive with a 0.1 stepsize
400 *
401 * @return string
402 */
403 function fileEntry( $url, $date, $priority ) {
404 return
405 "\t<url>\n" .
406 "\t\t<loc>$url</loc>\n" .
407 "\t\t<lastmod>$date</lastmod>\n" .
408 "\t\t<priority>$priority</priority>\n" .
409 "\t</url>\n";
410 }
411
412 /**
413 * Return the XML required to close sitemap file
414 *
415 * @static
416 * @return string
417 */
418 function closeFile() {
419 return "</urlset>\n";
420 }
421
422 /**
423 * Write a string to stderr followed by a UNIX newline
424 */
425 function debug( $str ) {
426 fwrite( $this->stderr, "$str\n" );
427 }
428
429 /**
430 * Populate $this->limit
431 */
432 function generateLimit( $namespace ) {
433 $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
434
435 $this->limit = array(
436 strlen( $this->openFile() ),
437 strlen( $this->fileEntry( $title->getFullUrl(), wfTimestamp( TS_ISO_8601, wfTimestamp() ), $this->priority( $namespace ) ) ),
438 strlen( $this->closeFile() )
439 );
440 }
441 }
442
443 if ( in_array( '--help', $argv ) )
444 wfDie(
445 "Usage: php generateSitemap.php [host] [options]\n" .
446 "\thost = hostname\n" .
447 "\toptions:\n" .
448 "\t\t--help\tshow this message\n" .
449 "\t\t--fspath\tThe file system path to save to, e.g /tmp/sitemap/\n" .
450 "\t\t--path\tThe http path to use, e.g. /wiki\n" .
451 "\t\t--compress=[yes|no]\tcompress the sitemap files, default yes\n"
452 );
453
454 if ( isset( $argv[1] ) && strpos( $argv[1], '--' ) !== 0 )
455 $_SERVER['SERVER_NAME'] = $argv[1];
456
457 $optionsWithArgs = array( 'fspath', 'path', 'compress' );
458 require_once 'commandLine.inc';
459
460 $gs = new GenerateSitemap( @$options['fspath'], @$options['path'], @$options['compress'] !== 'no' );
461 $gs->main();
462 ?>