3 // Alternate 1.4 -> 1.5 schema upgrade
4 // This does only the main tables + UTF-8
5 // and is designed to allow upgrades to interleave
6 // with other updates on the replication stream so
7 // that large wikis can be upgraded without disrupting
10 require_once( 'commandLine.inc' );
12 $upgrade = new FiveUpgrade();
16 function FiveUpgrade() {
18 $this->conversionTables
= $this->prepareWindows1252();
19 $this->dbw
=& $this->newConnection();
20 $this->dbr
=& $this->newConnection();
21 $this->dbr
->bufferResults( false );
26 $this->upgradeLinks();
31 * Open a second connection to the master server, with buffering off.
32 * This will let us stream large datasets in and write in chunks on the
37 function &newConnection() {
38 global $wgDBadminuser, $wgDBadminpassword;
39 global $wgDBserver, $wgDBname;
40 $db =& new Database( $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname );
45 * Prepare a conversion array for converting Windows Code Page 1252 to
46 * UTF-8. This should provide proper conversion of text that was miscoded
47 * as Windows-1252 by naughty user-agents, and doesn't rely on an outside
53 function prepareWindows1252() {
55 # http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT
56 static $cp1252 = array(
57 0x80 => 0x20AC, #EURO SIGN
58 0x81 => UNICODE_REPLACEMENT
,
59 0x82 => 0x201A, #SINGLE LOW-9 QUOTATION MARK
60 0x83 => 0x0192, #LATIN SMALL LETTER F WITH HOOK
61 0x84 => 0x201E, #DOUBLE LOW-9 QUOTATION MARK
62 0x85 => 0x2026, #HORIZONTAL ELLIPSIS
63 0x86 => 0x2020, #DAGGER
64 0x87 => 0x2021, #DOUBLE DAGGER
65 0x88 => 0x02C6, #MODIFIER LETTER CIRCUMFLEX ACCENT
66 0x89 => 0x2030, #PER MILLE SIGN
67 0x8A => 0x0160, #LATIN CAPITAL LETTER S WITH CARON
68 0x8B => 0x2039, #SINGLE LEFT-POINTING ANGLE QUOTATION MARK
69 0x8C => 0x0152, #LATIN CAPITAL LIGATURE OE
70 0x8D => UNICODE_REPLACEMENT
,
71 0x8E => 0x017D, #LATIN CAPITAL LETTER Z WITH CARON
72 0x8F => UNICODE_REPLACEMENT
,
73 0x90 => UNICODE_REPLACEMENT
,
74 0x91 => 0x2018, #LEFT SINGLE QUOTATION MARK
75 0x92 => 0x2019, #RIGHT SINGLE QUOTATION MARK
76 0x93 => 0x201C, #LEFT DOUBLE QUOTATION MARK
77 0x94 => 0x201D, #RIGHT DOUBLE QUOTATION MARK
78 0x95 => 0x2022, #BULLET
79 0x96 => 0x2013, #EN DASH
80 0x97 => 0x2014, #EM DASH
81 0x98 => 0x02DC, #SMALL TILDE
82 0x99 => 0x2122, #TRADE MARK SIGN
83 0x9A => 0x0161, #LATIN SMALL LETTER S WITH CARON
84 0x9B => 0x203A, #SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
85 0x9C => 0x0153, #LATIN SMALL LIGATURE OE
86 0x9D => UNICODE_REPLACEMENT
,
87 0x9E => 0x017E, #LATIN SMALL LETTER Z WITH CARON
88 0x9F => 0x0178, #LATIN CAPITAL LETTER Y WITH DIAERESIS
91 for( $i = 0; $i < 0x100; $i++
) {
92 $unicode = isset( $cp1252[$i] ) ?
$cp1252[$i] : $i;
93 $pairs[chr( $i )] = codepointToUtf8( $unicode );
99 * Convert from 8-bit Windows-1252 to UTF-8 if necessary.
100 * @param string $text
104 function conv( $text ) {
107 return strtr( $text, $this->conversionTables
);
114 * Dump timestamp and message to output
115 * @param string $message
118 function log( $message ) {
119 echo wfTimestamp( TS_DB
) . ': ' . $message . "\n";
124 * Initialize the chunked-insert system.
125 * Rows will be inserted in chunks of the given number, rather
126 * than in a giant INSERT...SELECT query, to keep the serialized
127 * MySQL database replication from getting hung up. This way other
128 * things can be going on during conversion without waiting for
129 * slaves to catch up as badly.
131 * @param int $chunksize Number of rows to insert at once
132 * @param int $final Total expected number of rows / id of last row,
133 * used for progress reports.
136 function setChunkScale( $chunksize, $final ) {
137 $this->chunkSize
= $chunksize;
138 $this->chunkFinal
= $final;
139 $this->chunkCount
= 0;
140 $this->chunkStartTime
= wfTime();
141 $this->chunkOptions
= array();
145 * Chunked inserts: perform an insert if we've reached the chunk limit.
146 * Prints a progress report with estimated completion time.
147 * @param string $table
148 * @param array &$chunk -- This will be emptied if an insert is done.
149 * @param string $fname function name to report in SQL
150 * @param int $key A key identifier to use in progress estimation in
151 * place of the number of rows inserted. Use this if
152 * you provided a max key number instead of a count
153 * as the final chunk number in setChunkScale()
156 function addChunk( $table, &$chunk, $fname, $key = null ) {
157 if( count( $chunk ) >= $this->chunkSize
) {
158 $this->insertChunk( $table, $chunk, $fname );
160 $this->chunkCount +
= count( $chunk );
162 $delta = $now - $this->chunkStartTime
;
163 $rate = $this->chunkCount
/ $delta;
165 if( is_null( $key ) ) {
166 $completed = $this->chunkCount
;
170 $portion = $completed / $this->chunkFinal
;
172 $estimatedTotalTime = $delta / $portion;
173 $eta = $now +
$estimatedTotalTime;
175 printf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec\n",
176 wfTimestamp( TS_DB
, intval( $now ) ),
179 wfTimestamp( TS_DB
, intval( $eta ) ),
190 * Chunked inserts: perform an insert unconditionally, at the end, and log.
191 * @param string $table
192 * @param array &$chunk -- This will be emptied if an insert is done.
193 * @param string $fname function name to report in SQL
196 function lastChunk( $table, &$chunk, $fname ) {
197 $n = count( $chunk );
199 $this->insertChunk( $table, $chunk, $fname );
201 $this->log( "100.00% done on $table (last chunk $n rows)." );
205 * Chunked inserts: perform an insert.
206 * @param string $table
207 * @param array &$chunk -- This will be emptied if an insert is done.
208 * @param string $fname function name to report in SQL
211 function insertChunk( $table, &$chunk, $fname ) {
212 $this->dbw
->insert( $table, $chunk, $fname, $this->chunkOptions
);
215 function upgradePage() {
216 $fname = "FiveUpgrade::upgradePage";
219 $this->log( "...converting from cur/old to page/revision/text DB structure." );
221 extract( $this->dbw
->tableNames( 'cur', 'old', 'page', 'revision', 'text' ) );
223 $this->log( "Creating page and revision tables..." );
224 $this->dbw
->query("CREATE TABLE $page (
225 page_id int(8) unsigned NOT NULL auto_increment,
226 page_namespace int NOT NULL,
227 page_title varchar(255) binary NOT NULL,
228 page_restrictions tinyblob NOT NULL default '',
229 page_counter bigint(20) unsigned NOT NULL default '0',
230 page_is_redirect tinyint(1) unsigned NOT NULL default '0',
231 page_is_new tinyint(1) unsigned NOT NULL default '0',
232 page_random real unsigned NOT NULL,
233 page_touched char(14) binary NOT NULL default '',
234 page_latest int(8) unsigned NOT NULL,
235 page_len int(8) unsigned NOT NULL,
237 PRIMARY KEY page_id (page_id),
238 UNIQUE INDEX name_title (page_namespace,page_title),
241 ) TYPE=InnoDB", $fname );
242 $this->dbw
->query("CREATE TABLE $revision (
243 rev_id int(8) unsigned NOT NULL auto_increment,
244 rev_page int(8) unsigned NOT NULL,
245 rev_comment tinyblob NOT NULL default '',
246 rev_user int(5) unsigned NOT NULL default '0',
247 rev_user_text varchar(255) binary NOT NULL default '',
248 rev_timestamp char(14) binary NOT NULL default '',
249 rev_minor_edit tinyint(1) unsigned NOT NULL default '0',
250 rev_deleted tinyint(1) unsigned NOT NULL default '0',
252 PRIMARY KEY rev_page_id (rev_page, rev_id),
253 UNIQUE INDEX rev_id (rev_id),
254 INDEX rev_timestamp (rev_timestamp),
255 INDEX page_timestamp (rev_page,rev_timestamp),
256 INDEX user_timestamp (rev_user,rev_timestamp),
257 INDEX usertext_timestamp (rev_user_text,rev_timestamp)
258 ) TYPE=InnoDB", $fname );
260 $maxold = $this->dbw
->selectField( 'old', 'max(old_id)', '', $fname );
261 $this->log( "Last old record is {$maxold}" );
263 global $wgLegacySchemaConversion;
264 if( $wgLegacySchemaConversion ) {
265 // Create HistoryBlobCurStub entries.
266 // Text will be pulled from the leftover 'cur' table at runtime.
267 echo "......Moving metadata from cur; using blob references to text in cur table.\n";
268 $cur_text = "concat('O:18:\"historyblobcurstub\":1:{s:6:\"mCurId\";i:',cur_id,';}')";
269 $cur_flags = "'object'";
271 // Copy all cur text in immediately: this may take longer but avoids
272 // having to keep an extra table around.
273 echo "......Moving text from cur.\n";
274 $cur_text = 'cur_text';
278 $maxcur = $this->dbw
->selectField( 'cur', 'max(cur_id)', '', $fname );
279 $this->log( "Last cur entry is $maxcur" );
282 * Copy placeholder records for each page's current version into old
283 * Don't do any conversion here; text records are converted at runtime
284 * based on the flags (and may be originally binary!) while the meta
285 * fields will be converted in the old -> rev and cur -> page steps.
287 $this->setChunkScale( $chunksize, $maxcur );
288 $result = $this->dbr
->query(
289 "SELECT cur_id, cur_namespace, cur_title, $cur_text AS text, cur_comment,
290 cur_user, cur_user_text, cur_timestamp, cur_minor_edit, $cur_flags AS flags
292 ORDER BY cur_id", $fname );
294 while( $row = $this->dbr
->fetchObject( $result ) ) {
296 'old_namespace' => $row->cur_namespace
,
297 'old_title' => $row->cur_title
,
298 'old_text' => $row->text
,
299 'old_comment' => $row->cur_comment
,
300 'old_user' => $row->cur_user
,
301 'old_user_text' => $row->cur_user_text
,
302 'old_timestamp' => $row->cur_timestamp
,
303 'old_minor_edit' => $row->cur_minor_edit
,
304 'old_flags' => $row->flags
);
305 $this->addChunk( 'old', $add, $fname, $row->cur_id
);
307 $this->lastChunk( 'old', $add, $fname );
308 $this->dbr
->freeResult( $result );
311 * Copy revision metadata from old into revision.
312 * We'll also do UTF-8 conversion of usernames and comments.
314 $newmaxold = $this->dbw
->selectField( 'old', 'max(old_id)', '', $fname );
315 $this->setChunkScale( $chunksize, $newmaxold );
317 $this->log( "......Setting up revision table." );
318 $result = $this->dbr
->query(
319 "SELECT old_id, cur_id, old_comment, old_user, old_user_text,
320 old_timestamp, old_minor_edit
321 FROM $old,$cur WHERE old_namespace=cur_namespace AND old_title=cur_title
322 ORDER BY old_id", $fname );
325 while( $row = $this->dbr
->fetchObject( $result ) ) {
327 'rev_id' => $row->old_id
,
328 'rev_page' => $row->cur_id
,
329 'rev_comment' => $this->conv( $row->old_comment
),
330 'rev_user' => $row->old_user
,
331 'rev_user_text' => $this->conv( $row->old_user_text
),
332 'rev_timestamp' => $row->old_timestamp
,
333 'rev_minor_edit' => $row->old_minor_edit
);
334 $this->addChunk( 'revision', $add, $fname, $row->old_id
);
336 $this->lastChunk( 'revision', $add, $fname );
337 $this->dbr
->freeResult( $result );
341 * Copy revision metadata from cur into page.
342 * We'll also do UTF-8 conversion of titles.
344 $this->log( "......Setting up page table." );
345 $this->setChunkScale( $chunksize, $maxcur );
346 $result = $this->dbr
->query( "
347 SELECT cur_id, cur_namespace, cur_title, cur_restrictions, cur_counter, cur_is_redirect, cur_is_new,
348 cur_random, cur_touched, rev_id, LENGTH(cur_text) AS len
350 WHERE cur_id=rev_page AND rev_timestamp=cur_timestamp AND rev_id > {$maxold}
351 ORDER BY cur_id", $fname );
353 while( $row = $this->dbr
->fetchObject( $result ) ) {
355 'page_id' => $row->cur_id
,
356 'page_namespace' => $row->cur_namespace
,
357 'page_title' => $this->conv( $row->cur_title
),
358 'page_restrictions' => $row->cur_restrictions
,
359 'page_counter' => $row->cur_counter
,
360 'page_is_redirect' => $row->cur_is_redirect
,
361 'page_is_new' => $row->cur_is_new
,
362 'page_random' => $row->cur_random
,
363 'page_touched' => $this->dbw
->timestamp(),
364 'page_latest' => $row->rev_id
,
365 'page_len' => $row->len
);
366 $this->addChunk( 'page', $add, $fname, $row->cur_id
);
368 $this->lastChunk( 'page', $add, $fname );
369 $this->dbr
->freeResult( $result );
371 $this->log( "......Renaming old." );
372 $this->dbw
->query( "ALTER TABLE $old RENAME TO $text", $fname );
374 $this->log( "...done." );
377 function upgradeLinks() {
378 $fname = 'FiveUpgrade::upgradeLinks';
380 extract( $this->dbw
->tableNames( 'links', 'brokenlinks', 'pagelinks', 'page' ) );
382 $this->log( 'Creating pagelinks table...' );
384 CREATE TABLE $pagelinks (
385 -- Key to the page_id of the page containing the link.
386 pl_from int(8) unsigned NOT NULL default '0',
388 -- Key to page_namespace/page_title of the target page.
389 -- The target page may or may not exist, and due to renames
390 -- and deletions may refer to different page records as time
392 pl_namespace int NOT NULL default '0',
393 pl_title varchar(255) binary NOT NULL default '',
395 UNIQUE KEY pl_from(pl_from,pl_namespace,pl_title),
396 KEY (pl_namespace,pl_title)
400 $this->log( 'Importing live links -> pagelinks' );
401 $nlinks = $this->dbw
->selectField( 'links', 'count(*)', '', $fname );
403 $this->setChunkScale( $chunksize, $nlinks );
404 $result = $this->dbr
->query( "
405 SELECT l_from,page_namespace,page_title
407 WHERE l_to=page_id", $fname );
409 while( $row = $this->dbr
->fetchObject( $result ) ) {
411 'pl_from' => $row->l_from
,
412 'pl_namespace' => $row->page_namespace
,
413 'pl_title' => $row->page_title
);
414 $this->addChunk( 'pagelinks', $add, $fname );
416 $this->lastChunk( 'pagelinks', $add, $fname );
418 $this->log( 'no links!' );
421 $this->log( 'Importing brokenlinks -> pagelinks' );
422 $nbrokenlinks = $this->dbw
->selectField( 'brokenlinks', 'count(*)', '', $fname );
423 if( $nbrokenlinks ) {
424 $this->setChunkScale( $chunksize, $nbrokenlinks );
425 $this->chunkOptions
= array( 'IGNORE' );
426 $result = $this->dbr
->query(
427 "SELECT bl_from, bl_to FROM $brokenlinks",
430 while( $row = $this->dbr
->fetchObject( $result ) ) {
431 $pagename = $this->conv( $row->bl_to
);
432 $title = Title
::newFromText( $pagename );
433 if( is_null( $title ) ) {
434 $this->log( "** invalid brokenlink: $row->bl_from -> '$pagename' (converted from '$row->bl_to')" );
437 'pl_from' => $row->bl_from
,
438 'pl_namespace' => $title->getNamespace(),
439 'pl_title' => $title->getDBkey() );
440 $this->addChunk( 'pagelinks', $add, $fname );
443 $this->lastChunk( 'pagelinks', $add, $fname );
445 $this->log( 'no brokenlinks!' );
448 $this->log( 'Done with links.' );