maintenance/storage/testCompression.php

   1 <?php
   2 /**
   3  * Test revision text compression and decompression.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance ExternalStorage
  22  */
  23
  24 $optionsWithArgs = [ 'start', 'limit', 'type' ];
  25 require __DIR__ . '/../commandLine.inc';
  26
  27 if ( !isset( $args[0] ) ) {
  28         echo "Usage: php testCompression.php [--type=<type>] [--start=<start-date>] " .
  29                 "[--limit=<num-revs>] <page-title>\n";
  30         exit( 1 );
  31 }
  32
  33 $lang = Language::factory( 'en' );
  34 $title = Title::newFromText( $args[0] );
  35 if ( isset( $options['start'] ) ) {
  36         $start = wfTimestamp( TS_MW, strtotime( $options['start'] ) );
  37         echo "Starting from " . $lang->timeanddate( $start ) . "\n";
  38 } else {
  39         $start = '19700101000000';
  40 }
  41 if ( isset( $options['limit'] ) ) {
  42         $limit = $options['limit'];
  43         $untilHappy = false;
  44 } else {
  45         $limit = 1000;
  46         $untilHappy = true;
  47 }
  48 $type = isset( $options['type'] ) ? $options['type'] : 'ConcatenatedGzipHistoryBlob';
  49
  50 $dbr = $this->getDB( DB_REPLICA );
  51 $res = $dbr->select(
  52         [ 'page', 'revision', 'text' ],
  53         array_merge(
  54                 Revision::selectFields(),
  55                 Revision::selectPageFields(),
  56                 Revision::selectTextFields()
  57         ),
  58         [
  59                 'page_namespace' => $title->getNamespace(),
  60                 'page_title' => $title->getDBkey(),
  61                 'page_id=rev_page',
  62                 'rev_timestamp > ' . $dbr->addQuotes( $dbr->timestamp( $start ) ),
  63                 'rev_text_id=old_id'
  64         ], __FILE__, [ 'LIMIT' => $limit ]
  65 );
  66
  67 $blob = new $type;
  68 $hashes = [];
  69 $keys = [];
  70 $uncompressedSize = 0;
  71 $t = -microtime( true );
  72 foreach ( $res as $row ) {
  73         $revision = new Revision( $row );
  74         $text = $revision->getSerializedData();
  75         $uncompressedSize += strlen( $text );
  76         $hashes[$row->rev_id] = md5( $text );
  77         $keys[$row->rev_id] = $blob->addItem( $text );
  78         if ( $untilHappy && !$blob->isHappy() ) {
  79                 break;
  80         }
  81 }
  82
  83 $serialized = serialize( $blob );
  84 $t += microtime( true );
  85 # print_r( $blob->mDiffMap );
  86
  87 printf( "%s\nCompression ratio for %d revisions: %5.2f, %s -> %d\n",
  88         $type,
  89         count( $hashes ),
  90         $uncompressedSize / strlen( $serialized ),
  91         $lang->formatSize( $uncompressedSize ),
  92         strlen( $serialized )
  93 );
  94 printf( "Compression time: %5.2f ms\n", $t * 1000 );
  95
  96 $t = -microtime( true );
  97 $blob = unserialize( $serialized );
  98 foreach ( $keys as $id => $key ) {
  99         $text = $blob->getItem( $key );
 100         if ( md5( $text ) != $hashes[$id] ) {
 101                 echo "Content hash mismatch for rev_id $id\n";
 102                 # var_dump( $text );
 103         }
 104 }
 105 $t += microtime( true );
 106 printf( "Decompression time: %5.2f ms\n", $t * 1000 );