maintenance/importUseModWiki.php

   1 <?php
   2
   3 /**
   4  * Import data from a UseModWiki into a MediaWiki wiki
   5  * 2003-02-09 Brion VIBBER <brion@pobox.com>
   6  * Based loosely on Magnus's code from 2001-2002
   7  *
   8  * Updated limited version to get something working temporarily
   9  * 2003-10-09
  10  * Be sure to run the link & index rebuilding scripts!
  11  *
  12  * Some more munging for charsets etc
  13  * 2003-11-28
  14  *
  15  * Partial fix for pages starting with lowercase letters (??)
  16  * and CamelCase and /Subpage link conversion
  17  * 2004-11-17
  18  *
  19  * Rewrite output to create Special:Export format for import
  20  * instead of raw SQL. Should be 'future-proof' against future
  21  * schema changes.
  22  * 2005-03-14
  23  *
  24  * @todo document
  25  * @package MediaWiki
  26  * @subpackage Maintenance
  27  */
  28
  29 if( php_sapi_name() != 'cli' ) {
  30         die( "Please customize the settings and run me from the command line." );
  31 }
  32
  33 /** Set these correctly! */
  34 $wgImportEncoding = "CP1252"; /* We convert all to UTF-8 */
  35 $wgRootDirectory = "/kalman/Projects/wiki2002/wiki/lib-http/db/wiki";
  36
  37 /* On a large wiki, you might run out of memory */
  38 @ini_set( 'memory_limit', '40M' );
  39
  40 /* globals */
  41 $wgFieldSeparator = "\xb3"; # Some wikis may use different char
  42         $FS = $wgFieldSeparator ;
  43         $FS1 = $FS."1" ;
  44         $FS2 = $FS."2" ;
  45         $FS3 = $FS."3" ;
  46
  47 # Unicode sanitization tools
  48 require_once( '../includes/normal/UtfNormal.php' );
  49
  50 $usercache = array();
  51
  52 importPages();
  53
  54 # ------------------------------------------------------------------------------
  55
  56 function importPages()
  57 {
  58         global $wgRootDirectory;
  59
  60         $gt = '>';
  61         echo <<<END
  62 <?xml version="1.0" encoding="UTF-8" ?$gt
  63 <mediawiki xmlns="http://www.mediawiki.org/xml/export-0.1/"
  64            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  65            xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.1/
  66                                http://www.mediawiki.org/xml/export-0.1.xsd"
  67            version="0.1"
  68            xml:lang="en">
  69 <!-- generated by importUseModWiki.php -->
  70
  71 END;
  72         $letters = array(
  73                 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
  74                 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
  75                 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'other' );
  76         foreach( $letters as $letter ) {
  77                 $dir = "$wgRootDirectory/page/$letter";
  78                 if( is_dir( $dir ) )
  79                         importPageDirectory( $dir );
  80         }
  81         echo <<<END
  82 </mediawiki>
  83
  84 END;
  85 }
  86
  87 function importPageDirectory( $dir, $prefix = "" )
  88 {
  89         echo "\n<!-- Checking page directory " . xmlCommentSafe( $dir ) . " -->\n";
  90         $mydir = opendir( $dir );
  91         while( $entry = readdir( $mydir ) ) {
  92                 if( preg_match( '/^(.+)\.db$/', $entry, $m ) ) {
  93                         echo importPage( $prefix . $m[1] );
  94                 } else {
  95                         if( is_dir( "$dir/$entry" ) ) {
  96                                 if( $entry != '.' && $entry != '..' ) {
  97                                         importPageDirectory( "$dir/$entry", "$entry/" );
  98                                 }
  99                         } else {
 100                                 echo "<!-- File '" . xmlCommentSafe( $entry ) . "' doesn't seem to contain an article. Skipping. -->\n";
 101                         }
 102                 }
 103         }
 104 }
 105
 106
 107 # ------------------------------------------------------------------------------
 108
 109 /* fetch_ functions
 110         Grab a given item from the database
 111         */
 112
 113 function useModFilename( $title ) {
 114         $c = substr( $title, 0, 1 );
 115         if(preg_match( '/[A-Z]/i', $c ) ) {
 116                 return strtoupper( $c ) . "/$title";
 117         }
 118         return "other/$title";
 119 }
 120
 121 function fetchPage( $title )
 122 {
 123         global $FS,$FS1,$FS2,$FS3, $wgRootDirectory;
 124
 125         $fname = $wgRootDirectory . "/page/" . useModFilename( $title ) . ".db";
 126         if( !file_exists( $fname ) ) {
 127                 die( "Couldn't open file '$fname' for page '$title'.\n" );
 128         }
 129
 130         $page = splitHash( $FS1, file_get_contents( $fname ) );
 131         $section = splitHash( $FS2, $page["text_default"] );
 132         $text = splitHash( $FS3, $section["data"] );
 133
 134         return array2object( array( "text" => $text["text"] , "summary" => $text["summary"] ,
 135                 "minor" => $text["minor"] , "ts" => $section["ts"] ,
 136                 "username" => $section["username"] , "host" => $section["host"] ) );
 137 }
 138
 139 function fetchKeptPages( $title )
 140 {
 141         global $FS,$FS1,$FS2,$FS3, $wgRootDirectory, $wgTimezoneCorrection;
 142
 143         $fname = $wgRootDirectory . "/keep/" . useModFilename( $title ) . ".kp";
 144         if( !file_exists( $fname ) ) return array();
 145
 146         $keptlist = explode( $FS1, file_get_contents( $fname ) );
 147         array_shift( $keptlist ); # Drop the junk at beginning of file
 148
 149         $revisions = array();
 150         foreach( $keptlist as $rev ) {
 151                 $section = splitHash( $FS2, $rev );
 152                 $text = splitHash( $FS3, $section["data"] );
 153                 if ( $text["text"] && $text["minor"] != "" && ( $section["ts"]*1 > 0 ) ) {
 154                         array_push( $revisions, array2object( array ( "text" => $text["text"] , "summary" => $text["summary"] ,
 155                                 "minor" => $text["minor"] , "ts" => $section["ts"] ,
 156                                 "username" => $section["username"] , "host" => $section["host"] ) ) );
 157                 } else {
 158                         echo "<!-- skipped a bad old revision -->\n";
 159                 }
 160         }
 161         return $revisions;
 162 }
 163
 164 function splitHash ( $sep , $str ) {
 165         $temp = explode ( $sep , $str ) ;
 166         $ret = array () ;
 167         for ( $i = 0; $i+1 < count ( $temp ) ; $i++ ) {
 168                 $ret[$temp[$i]] = $temp[++$i] ;
 169                 }
 170         return $ret ;
 171         }
 172
 173
 174 /* import_ functions
 175         Take a fetched item and produce SQL
 176         */
 177
 178 function checkUserCache( $name, $host )
 179 {
 180         global $usercache;
 181
 182         if( $name ) {
 183                 if( in_array( $name, $usercache ) ) {
 184                         $userid = $usercache[$name];
 185                 } else {
 186                         # If we haven't imported user accounts
 187                         $userid = 0;
 188                 }
 189                 $username = str_replace( '_', ' ', $name );
 190         } else {
 191                 $userid = 0;
 192                 $username = $host;
 193         }
 194         return array( $userid, $username );
 195 }
 196
 197 function importPage( $title )
 198 {
 199         global $usercache;
 200
 201         echo "\n<!-- Importing page " . xmlCommentSafe( $title ) . " -->\n";
 202         $page = fetchPage( $title );
 203
 204         $newtitle = xmlsafe( str_replace( '_', ' ', recodeText( $title ) ) );
 205
 206         $munged = mungeFormat( $page->text );
 207         if( $munged != $page->text ) {
 208                 /**
 209                  * Save a *new* revision with the conversion, and put the
 210                  * previous last version into the history.
 211                  */
 212                 $next = array2object( array(
 213                         'text'     => $munged,
 214                         'minor'    => 1,
 215                         'username' => 'Conversion script',
 216                         'host'     => '127.0.0.1',
 217                         'ts'       => time(),
 218                         'summary'  => 'link fix',
 219                         ) );
 220                 $revisions = array( $page, $next );
 221         } else {
 222                 /**
 223                  * Current revision:
 224                  */
 225                 $revisions = array( $page );
 226         }
 227         $xml = <<<END
 228         <page>
 229                 <title>$newtitle</title>
 230
 231 END;
 232
 233         # History
 234         $revisions = array_merge( $revisions, fetchKeptPages( $title ) );
 235         if(count( $revisions ) == 0 ) {
 236                 return $sql;
 237         }
 238
 239         foreach( $revisions as $rev ) {
 240                 $text      = xmlsafe( recodeText( $rev->text ) );
 241                 $minor     = ($rev->minor ? '<minor/>' : '');
 242                 list( $userid, $username ) = checkUserCache( $rev->username, $rev->host );
 243                 $username  = xmlsafe( recodeText( $username ) );
 244                 $timestamp = xmlsafe( timestamp2ISO8601( $rev->ts ) );
 245                 $comment   = xmlsafe( recodeText( $rev->summary ) );
 246
 247                 $xml .= <<<END
 248                 <revision>
 249                         <timestamp>$timestamp</timestamp>
 250                         <contributor><username>$username</username></contributor>
 251                         $minor
 252                         <comment>$comment</comment>
 253                         <text>$text</text>
 254                 </revision>
 255
 256 END;
 257         }
 258         $xml .= "</page>\n\n";
 259         return $xml;
 260 }
 261
 262 # Whee!
 263 function recodeText( $string ) {
 264         global $wgImportEncoding;
 265         # For currently latin-1 wikis
 266         $string = str_replace( "\r\n", "\n", $string );
 267         $string = @iconv( $wgImportEncoding, "UTF-8", $string );
 268         $string = wfMungeToUtf8( $string ); # Any old &#1234; stuff
 269         return $string;
 270 }
 271
 272 function wfUtf8Sequence($codepoint) {
 273         if($codepoint <     0x80) return chr($codepoint);
 274         if($codepoint <    0x800) return chr($codepoint >>  6 & 0x3f | 0xc0) .
 275                                      chr($codepoint       & 0x3f | 0x80);
 276     if($codepoint <  0x10000) return chr($codepoint >> 12 & 0x0f | 0xe0) .
 277                                      chr($codepoint >>  6 & 0x3f | 0x80) .
 278                                      chr($codepoint       & 0x3f | 0x80);
 279         if($codepoint < 0x100000) return chr($codepoint >> 18 & 0x07 | 0xf0) . # Double-check this
 280                                          chr($codepoint >> 12 & 0x3f | 0x80) .
 281                                      chr($codepoint >>  6 & 0x3f | 0x80) .
 282                                      chr($codepoint       & 0x3f | 0x80);
 283         # Doesn't yet handle outside the BMP
 284         return "&#$codepoint;";
 285 }
 286
 287 function wfMungeToUtf8($string) {
 288         $string = preg_replace ( '/&#([0-9]+);/e', 'wfUtf8Sequence($1)', $string );
 289         $string = preg_replace ( '/&#x([0-9a-f]+);/ie', 'wfUtf8Sequence(0x$1)', $string );
 290         # Should also do named entities here
 291         return $string;
 292 }
 293
 294 function timestamp2ISO8601( $ts ) {
 295         #2003-08-05T18:30:02Z
 296         return gmdate( 'Y-m-d', $ts ) . 'T' . gmdate( 'H:i:s', $ts ) . 'Z';
 297 }
 298
 299 function xmlsafe( $string ) {
 300         /**
 301          * The page may contain old data which has not been properly normalized.
 302          * Invalid UTF-8 sequences or forbidden control characters will make our
 303          * XML output invalid, so be sure to strip them out.
 304          */
 305         $string = UtfNormal::cleanUp( $string );
 306
 307         $string = htmlspecialchars( $string );
 308         return $string;
 309 }
 310
 311 function xmlCommentSafe( $text ) {
 312         return str_replace( '--', '\\-\\-', xmlsafe( recodeText( $text ) ) );
 313 }
 314
 315
 316 function array2object( $arr ) {
 317         $o = (object)0;
 318         foreach( $arr as $x => $y ) {
 319                 $o->$x = $y;
 320         }
 321         return $o;
 322 }
 323
 324
 325 /**
 326  * Make CamelCase and /Talk links work
 327  */
 328 function mungeFormat( $text ) {
 329         global $nowiki;
 330         $nowiki = array();
 331         $staged = preg_replace_callback(
 332                 '/(<nowiki>.*?<\\/nowiki>|(?:http|https|ftp):\\S+|\[\[[^]\\n]+]])/s',
 333                 'nowikiPlaceholder', $text );
 334
 335         # This is probably not  100% correct, I'm just
 336         # glancing at the UseModWiki code.
 337         $upper   = "[A-Z]";
 338         $lower   = "[a-z_0-9]";
 339         $any     = "[A-Za-z_0-9]";
 340         $camel   = "(?:$upper+$lower+$upper+$any*)";
 341         $subpage = "(?:\\/$any+)";
 342         $substart = "(?:\\/$upper$any*)";
 343
 344         $munged = preg_replace( "/(?!\\[\\[)($camel$subpage*|$substart$subpage*)\\b(?!\\]\\]|>)/",
 345                 '[[$1]]', $staged );
 346
 347         $final = preg_replace( '/' . preg_quote( placeholder() ) . '/es',
 348                 'array_shift( $nowiki )', $munged );
 349         return $final;
 350 }
 351
 352
 353 function placeholder( $x = null ) {
 354         return '\xffplaceholder\xff';
 355 }
 356
 357 function nowikiPlaceholder( $matches ) {
 358         global $nowiki;
 359         $nowiki[] = $matches[1];
 360         return placeholder();
 361 }
 362
 363 ?>