maintenance/convertUtf8.php

   1 <?php
   2 /**
   3  * @package MediaWiki
   4  * @subpackage Maintenance
   5  */
   6
   7 die("This file is not complete; it's checked in so I don't forget it.");
   8
   9 /**
  10  * UTF-8 conversion of DOOOOOOOM
  11  *
  12  * 1. Lock the wiki
  13  * 2. Make a convertlist of all pages
  14  * 3. Enable CONVERTLOCK mode and switch to UTF-8
  15  * 4. As quick as possible, convert the cur, images, *links, user, etc tables.
  16  * Clear cache tables.
  17  * 5. Unlock the wiki. Attempts to access pages on the convertlist will be
  18  * trapped to read-only.
  19  * 6. Go through the list, fixing up old revisions. Remove pages from the
  20  * convertlist.
  21  */
  22 class UtfUpdater {
  23         /** Constructor, set the database */
  24         function UtfUpdater() {
  25                 $this->db =& wfGetDB( DB_MASTER );
  26         }
  27
  28         /**
  29          * @param string $string A string to be converted to UTF-8
  30          */
  31         function toUtf8( $string ) {
  32                 if( function_exists( 'iconv' ) ) {
  33                         # There are likely to be Windows code page 1252 chars in there.
  34                         # Convert them to the proper UTF-8 chars if possible.
  35                         return iconv( 'CP1252', 'UTF-8', $string );
  36                 } else {
  37                         # Will work from plain iso 8859-1 and may corrupt these chars
  38                         return utf8_encode( $string );
  39                 }
  40         }
  41
  42         /**
  43          * Truncate a table.
  44          * @param string $table The table name to be truncated
  45          */
  46         function clearTable( $table ) {
  47                 print "Clearing $table...\n";
  48                 $tableName = $this->db->tableName( $table );
  49                 $this->db->query( 'TRUNCATE $tableName' );
  50         }
  51
  52         /**
  53          * @param string $table Table to be converted
  54          * @param string $key Primary key, to identify fields in the UPDATE. If NULL, all fields will be used to match.
  55          * @param array $fields List of all fields to grab and convert. If null, will assume you want the $key, and will ask for DISTINCT.
  56          * @param array $timestamp A field which should be updated to the current timestamp on changed records.
  57          * @access private
  58          */
  59         function convertTable( $table, $key, $fields = null, $timestamp = null ) {
  60                 $fname = 'UtfUpdater::convertTable';
  61                 if( $fields ) {
  62                         $distinct = '';
  63                 } else {
  64                         # If working on one key only, there will be multiple rows.
  65                         # Use DISTINCT to return only one and save us some trouble.
  66                         $fields = array( $key );
  67                         $distinct = 'DISTINCT';
  68                 }
  69                 $condition = '';
  70                 foreach( $fields as $field ) {
  71                         if( $condition ) $condition .= ' OR ';
  72                         $condition .= "$field RLIKE '[\x80-\xff]'";
  73                 }
  74                 $res = $this->db->selectArray(
  75                         $table,
  76                         array_merge( $fields, array( $key ) ),
  77                         $condition,
  78                         $fname,
  79                         $distinct );
  80                 print "Converting " . $this->db->numResults( $res ) . " rows from $table:\n";
  81                 $n = 0;
  82                 while( $s = $this->db->fetchObject( $res ) ) {
  83                         $set = array();
  84                         foreach( $fields as $field ) {
  85                                 $set[] = $this->toUtf8( $s->$field );
  86                         }
  87                         if( $timestamp ) {
  88                                 $set[$timestamp] = $this->db->timestamp();
  89                         }
  90                         if( $key ) {
  91                                 $keyCond = array( $key, $s->$key );
  92                         } else {
  93                                 $keyCond = array();
  94                                 foreach( $fields as $field ) {
  95                                         $keyCond[$field] = $s->$field;
  96                                 }
  97                         }
  98                         $this->db->updateArray(
  99                                 $table,
 100                                 $set,
 101                                 $keyCond,
 102                                 $fname );
 103                         if( ++$n % 100 == 0 ) echo "$n\n";
 104                 }
 105                 echo "$n done.\n";
 106                 $this->db->freeResult( $res );
 107         }
 108
 109         /**
 110          * Lock tables.
 111          * @param array $tables An array of table to be locked.
 112          */
 113         function lockTables( $tables ) {
 114                 $query = '';
 115                 foreach( $tables as $table ) {
 116                         $tableName = $this->db->tableName( $table );
 117                         if( $query ) $query .= ', ';
 118                         $query .= '$tableName WRITE';
 119                 }
 120                 $this->db->query( 'LOCK TABLES ' . $query );
 121         }
 122
 123         /**
 124          * @todo document
 125          */
 126         function updateAll() {
 127                 $this->lockTables( array(
 128                         'linkscc', 'objectcache', 'searchindex', 'querycache',
 129                         'ipblocks', 'user', 'page', 'revision', 'recentchanges',
 130                         'brokenlinks', 'categorylinks', 'imagelinks', 'watchlist',
 131                         'image', 'oldimage', 'archive' ) );
 132
 133                 # These are safe to clear out:
 134                 $this->clearTable( 'linkscc' );
 135                 $this->clearTable( 'objectcache' );
 136
 137                 # These need to be rebuild if used:
 138                 $this->clearTable( 'searchindex' );
 139                 $this->clearTable( 'querycache' );
 140
 141                 # And convert the rest...
 142                 $this->convertTable( 'ipblocks', 'ipb_id', array( 'ipb_reason' ) );
 143                 $this->convertTable( 'user', 'user_id',
 144                         array( 'user_name', 'user_real_name', 'user_options' ),
 145                         'user_touched' );
 146                 $this->convertTable( 'page', 'page_id',
 147                         array( 'page_title' ), 'page_touched' );
 148                 $this->convertTable( 'revision', 'rev_id',
 149                         array( 'rev_user_text', 'rev_comment' ) );
 150
 151                 $this->convertTable( 'recentchanges', 'rc_id',
 152                         array( 'rc_user_text', 'rc_title', 'rc_comment' ) );
 153
 154                 $this->convertTable( 'brokenlinks', 'bl_to' );
 155                 $this->convertTable( 'categorylinks', 'cl_to' );
 156                 $this->convertTable( 'imagelinks', 'il_to' );
 157                 $this->convertTable( 'watchlist', 'wl_title' );
 158
 159                 # FIXME: We'll also need to change the files.
 160                 $this->convertTable( 'image', 'img_name',
 161                         array( 'img_name', 'img_description', 'img_user_text' ) );
 162                 $this->convertTable( 'oldimage', 'archive_name',
 163                         array( 'oi_name', 'oi_archive_name', 'oi_description', 'oi_user_text' ) );
 164
 165                 # Don't change the ar_text entries; use $wgLegacyEncoding to read them at runtime
 166                 $this->convertTable( 'archive', null,
 167                         array( 'ar_title', 'ar_comment', 'ar_user_text' ) );
 168                 echo "Not converting text table: be sure to set \$wgLegacyEncoding!\n";
 169
 170                 $this->db->query( 'UNLOCK TABLES' );
 171         }
 172
 173 }
 174 ?>