maintenance/postgres/compare_schemas.pl

   1 #!/usr/bin/perl
   2
   3 ## Rough check that the base and postgres "tables.sql" are in sync
   4 ## Should be run from maintenance/postgres
   5 ## Checks a few other things as well...
   6
   7 use strict;
   8 use warnings;
   9 use Data::Dumper;
  10
  11 check_includes_dir();
  12
  13 my @old = ('../tables.sql');
  14 my $new = 'tables.sql';
  15 my @xfile;
  16
  17 ## Read in exceptions and other metadata
  18 my %ok;
  19 while (<DATA>) {
  20         next unless /^(\w+)\s*:\s*([^#]+)/;
  21         my ($name,$val) = ($1,$2);
  22         chomp $val;
  23         if ($name eq 'RENAME') {
  24                 die "Invalid rename\n" unless $val =~ /(\w+)\s+(\w+)/;
  25                 $ok{OLD}{$1} = $2;
  26                 $ok{NEW}{$2} = $1;
  27                 next;
  28         }
  29         if ($name eq 'XFILE') {
  30                 push @xfile, $val;
  31                 next;
  32         }
  33         for (split /\s+/ => $val) {
  34                 $ok{$name}{$_} = 0;
  35         }
  36 }
  37
  38 my $datatype = join '|' => qw(
  39 bool
  40 tinyint int bigint real float
  41 tinytext mediumtext text char varchar varbinary binary
  42 timestamp datetime
  43 tinyblob mediumblob blob
  44 );
  45 $datatype .= q{|ENUM\([\"\w, ]+\)};
  46 $datatype = qr{($datatype)};
  47
  48 my $typeval = qr{(\(\d+\))?};
  49
  50 my $typeval2 = qr{ signed| unsigned| binary| NOT NULL| NULL| auto_increment| default ['\-\d\w"]+| REFERENCES .+CASCADE};
  51
  52 my $indextype = join '|' => qw(INDEX KEY FULLTEXT), 'PRIMARY KEY', 'UNIQUE INDEX', 'UNIQUE KEY';
  53 $indextype = qr{$indextype};
  54
  55 my $engine = qr{TYPE|ENGINE};
  56
  57 my $tabletype = qr{InnoDB|MyISAM|HEAP|HEAP MAX_ROWS=\d+|InnoDB MAX_ROWS=\d+ AVG_ROW_LENGTH=\d+};
  58
  59 my $charset = qr{utf8|binary};
  60
  61 open my $newfh, '<', $new or die qq{Could not open $new: $!\n};
  62
  63
  64 my ($table,%old);
  65
  66 ## Read in the xfiles
  67 my %xinfo;
  68 for my $xfile (@xfile) {
  69         print "Loading $xfile\n";
  70         my $info = parse_sql($xfile);
  71         for (keys %$info) {
  72                 $xinfo{$_} = $info->{$_};
  73         }
  74 }
  75
  76 for my $oldfile (@old) {
  77         print "Loading $oldfile\n";
  78         my $info = parse_sql($oldfile);
  79         for (keys %xinfo) {
  80                 $info->{$_} = $xinfo{$_};
  81         }
  82         $old{$oldfile} = $info;
  83 }
  84
  85 sub parse_sql {
  86
  87         my $oldfile = shift;
  88
  89         open my $oldfh, '<', $oldfile or die qq{Could not open $oldfile: $!\n};
  90
  91         my %info;
  92         while (<$oldfh>) {
  93                 next if /^\s*\-\-/ or /^\s+$/;
  94                 s/\s*\-\- [\w ]+$//;
  95                 chomp;
  96
  97                 if (/CREATE\s*TABLE/i) {
  98                         m{^CREATE TABLE /\*\$wgDBprefix\*/(\w+) \($}
  99                                 or die qq{Invalid CREATE TABLE at line $. of $oldfile\n};
 100                         $table = $1;
 101                         $info{$table}{name}=$table;
 102                 }
 103                 elsif (m{^\) /\*\$wgDBTableOptions\*/}) {
 104                         $info{$table}{engine} = 'ENGINE';
 105                         $info{$table}{type} = 'variable';
 106                 }
 107                 elsif (/^\) ($engine)=($tabletype);$/) {
 108                         $info{$table}{engine}=$1;
 109                         $info{$table}{type}=$2;
 110                 }
 111                 elsif (/^\) ($engine)=($tabletype), DEFAULT CHARSET=($charset);$/) {
 112                         $info{$table}{engine}=$1;
 113                         $info{$table}{type}=$2;
 114                         $info{$table}{charset}=$3;
 115                 }
 116                 elsif (/^  (\w+) $datatype$typeval$typeval2{0,3},?$/) {
 117                         $info{$table}{column}{$1} = $2;
 118                         my $extra = $3 || '';
 119                         $info{$table}{columnfull}{$1} = "$2$extra";
 120                 }
 121                 elsif (/^  ($indextype)(?: (\w+))? \(([\w, \(\)]+)\),?$/) {
 122                         $info{$table}{lc $1.'_name'} = $2 ? $2 : '';
 123                         $info{$table}{lc $1.'pk_target'} = $3;
 124                 }
 125                 else {
 126                         die "Cannot parse line $. of $oldfile:\n$_\n";
 127                 }
 128
 129         }
 130         close $oldfh or die qq{Could not close "$oldfile": $!\n};
 131
 132         return \%info;
 133
 134 } ## end of parse_sql
 135
 136 ## Read in the parser test information
 137 my $parsefile = '../parserTests.inc';
 138 open my $pfh, '<', $parsefile or die qq{Could not open "$parsefile": $!\n};
 139 my $stat = 0;
 140 my %ptable;
 141 while (<$pfh>) {
 142         if (!$stat) {
 143                 if (/function listTables/) {
 144                         $stat = 1;
 145                 }
 146                 next;
 147         }
 148         $ptable{$1}=2 while m{'(\w+)'}g;
 149         last if /\);/;
 150 }
 151 close $pfh or die qq{Could not close "$parsefile": $!\n};
 152
 153 my $OK_NOT_IN_PTABLE = '
 154 filearchive
 155 logging
 156 profiling
 157 querycache_info
 158 searchindex
 159 trackbacks
 160 transcache
 161 user_newtalk
 162 updatelog
 163 ';
 164
 165 ## Make sure all tables in main tables.sql are accounted for int the parsertest.
 166 for my $table (sort keys %{$old{'../tables.sql'}}) {
 167         $ptable{$table}++;
 168         next if $ptable{$table} > 2;
 169         next if $OK_NOT_IN_PTABLE =~ /\b$table\b/;
 170         print qq{Table "$table" is in the schema, but not used inside of parserTest.inc\n};
 171 }
 172 ## Any that are used in ptables but no longer exist in the schema?
 173 for my $table (sort grep { $ptable{$_} == 2 } keys %ptable) {
 174         print qq{Table "$table" ($ptable{$table}) used in parserTest.inc, but not found in schema\n};
 175 }
 176
 177 for my $oldfile (@old) {
 178
 179 ## Begin non-standard indent
 180
 181 ## MySQL sanity checks
 182 for my $table (sort keys %{$old{$oldfile}}) {
 183         my $t = $old{$oldfile}{$table};
 184         if ($t->{engine} eq 'TYPE') {
 185                 die "Invalid engine for $oldfile: $t->{engine}\n" unless $t->{name} eq 'profiling';
 186         }
 187         my $charset = $t->{charset} || '';
 188         if ($oldfile !~ /binary/ and $charset eq 'binary') {
 189                 die "Invalid charset for $oldfile: $charset\n";
 190         }
 191 }
 192
 193 my $dtype = join '|' => qw(
 194 SMALLINT INTEGER BIGINT NUMERIC SERIAL
 195 TEXT CHAR VARCHAR
 196 BYTEA
 197 TIMESTAMPTZ
 198 CIDR
 199 );
 200 $dtype = qr{($dtype)};
 201 my %new;
 202 my ($infunction,$inview,$inrule,$lastcomma) = (0,0,0,0);
 203 seek $newfh, 0, 0;
 204 while (<$newfh>) {
 205         next if /^\s*\-\-/ or /^\s*$/;
 206         s/\s*\-\- [\w ']+$//;
 207         next if /^BEGIN;/ or /^SET / or /^COMMIT;/;
 208         next if /^CREATE SEQUENCE/;
 209         next if /^CREATE(?: UNIQUE)? INDEX/;
 210         next if /^CREATE FUNCTION/;
 211         next if /^CREATE TRIGGER/ or /^  FOR EACH ROW/;
 212         next if /^INSERT INTO/ or /^  VALUES \(/;
 213         next if /^ALTER TABLE/;
 214         chomp;
 215
 216         if (/^\$mw\$;?$/) {
 217                 $infunction = $infunction ? 0 : 1;
 218                 next;
 219         }
 220         next if $infunction;
 221
 222         next if /^CREATE VIEW/ and $inview = 1;
 223         if ($inview) {
 224                 /;$/ and $inview = 0;
 225                 next;
 226         }
 227
 228         next if /^CREATE RULE/ and $inrule = 1;
 229         if ($inrule) {
 230                 /;$/ and $inrule = 0;
 231                 next;
 232         }
 233
 234         if (/^CREATE TABLE "?(\w+)"? \($/) {
 235                 $table = $1;
 236                 $new{$table}{name}=$table;
 237                 $lastcomma = 1;
 238         }
 239         elsif (/^\);$/) {
 240                 if ($lastcomma) {
 241                         warn "Stray comma before line $.\n";
 242                 }
 243         }
 244         elsif (/^  (\w+) +$dtype.*?(,?)(?: --.*)?$/) {
 245                 $new{$table}{column}{$1} = $2;
 246                 if (!$lastcomma) {
 247                         print "Missing comma before line $. of $new\n";
 248                 }
 249                 $lastcomma = $3 ? 1 : 0;
 250         }
 251         else {
 252                 die "Cannot parse line $. of $new:\n$_\n";
 253         }
 254 }
 255
 256 ## Which column types are okay to map from mysql to postgres?
 257 my $COLMAP = q{
 258 ## INTS:
 259 tinyint SMALLINT
 260 int INTEGER SERIAL
 261 bigint BIGINT
 262 real NUMERIC
 263 float NUMERIC
 264
 265 ## TEXT:
 266 varchar(15) TEXT
 267 varchar(32) TEXT
 268 varchar(70) TEXT
 269 varchar(255) TEXT
 270 varchar TEXT
 271 text TEXT
 272 tinytext TEXT
 273 ENUM TEXT
 274
 275 ## TIMESTAMPS:
 276 varbinary(14) TIMESTAMPTZ
 277 binary(14) TIMESTAMPTZ
 278 datetime TIMESTAMPTZ
 279 timestamp TIMESTAMPTZ
 280
 281 ## BYTEA:
 282 mediumblob BYTEA
 283
 284 ## OTHER:
 285 bool SMALLINT # Sigh
 286
 287 };
 288 ## Allow specific exceptions to the above
 289 my $COLMAPOK = q{
 290 ## User inputted text strings:
 291 ar_comment      tinyblob       TEXT
 292 fa_description  tinyblob       TEXT
 293 img_description tinyblob       TEXT
 294 ipb_reason      tinyblob       TEXT
 295 log_action      varbinary(10)  TEXT
 296 oi_description  tinyblob       TEXT
 297 rev_comment     tinyblob       TEXT
 298 rc_log_action   varbinary(255) TEXT
 299 rc_log_type     varbinary(255) TEXT
 300
 301 ## Simple text-only strings:
 302 ar_flags          tinyblob       TEXT
 303 fa_minor_mime     varbinary(32)  TEXT
 304 fa_storage_group  varbinary(16)  TEXT # Just 'deleted' for now, should stay plain text
 305 fa_storage_key    varbinary(64)  TEXT # sha1 plus text extension
 306 ipb_address       tinyblob       TEXT # IP address or username
 307 ipb_range_end     tinyblob       TEXT # hexadecimal
 308 ipb_range_start   tinyblob       TEXT # hexadecimal
 309 img_minor_mime    varbinary(32)  TEXT
 310 img_sha1          varbinary(32)  TEXT
 311 job_cmd           varbinary(60)  TEXT # Should we limit to 60 as well?
 312 keyname           varbinary(255) TEXT # No tablename prefix (objectcache)
 313 ll_lang           varbinary(20)  TEXT # Language code
 314 log_params        blob           TEXT # LF separated list of args
 315 log_type          varbinary(10)  TEXT
 316 oi_minor_mime     varbinary(32)  TEXT
 317 oi_sha1           varbinary(32)  TEXT
 318 old_flags         tinyblob       TEXT
 319 old_text          mediumblob     TEXT
 320 pp_propname       varbinary(60)  TEXT
 321 pp_value          blob           TEXT
 322 page_restrictions tinyblob       TEXT # CSV string
 323 pf_server         varchar(30)    TEXT
 324 pr_level          varbinary(60)  TEXT
 325 pr_type           varbinary(60)  TEXT
 326 pt_create_perm    varbinary(60)  TEXT
 327 pt_reason         tinyblob       TEXT
 328 qc_type           varbinary(32)  TEXT
 329 qcc_type          varbinary(32)  TEXT
 330 qci_type          varbinary(32)  TEXT
 331 rc_params         blob           TEXT
 332 rlc_to_blob       blob           TEXT
 333 ug_group          varbinary(16)  TEXT
 334 user_email_token  binary(32)     TEXT
 335 user_ip           varbinary(40)  TEXT
 336 user_newpassword  tinyblob       TEXT
 337 user_options      blob           TEXT
 338 user_password     tinyblob       TEXT
 339 user_token        binary(32)     TEXT
 340
 341 ## Text URLs:
 342 el_index blob           TEXT
 343 el_to    blob           TEXT
 344 iw_url   blob           TEXT
 345 tb_url   blob           TEXT
 346 tc_url   varbinary(255) TEXT
 347
 348 ## Deprecated or not yet used:
 349 ar_text     mediumblob TEXT
 350 job_params  blob       TEXT
 351 log_deleted tinyint    INTEGER # Not used yet, but keep it INTEGER for safety
 352 rc_type     tinyint    CHAR
 353
 354 ## Number tweaking:
 355 fa_bits   int SMALLINT # bits per pixel
 356 fa_height int SMALLINT
 357 fa_width  int SMALLINT # Hope we don't see an image this wide...
 358 hc_id     int BIGINT   # Odd that site_stats is all bigint...
 359 img_bits  int SMALLINT # bits per image should stay sane
 360 oi_bits   int SMALLINT
 361
 362 ## True binary fields, usually due to gzdeflate and/or serialize:
 363 math_inputhash  varbinary(16) BYTEA
 364 math_outputhash varbinary(16) BYTEA
 365
 366 ## Namespaces: not need for such a high range
 367 ar_namespace     int SMALLINT
 368 job_namespace    int SMALLINT
 369 log_namespace    int SMALLINT
 370 page_namespace   int SMALLINT
 371 pl_namespace     int SMALLINT
 372 pt_namespace     int SMALLINT
 373 qc_namespace     int SMALLINT
 374 rc_namespace     int SMALLINT
 375 rd_namespace     int SMALLINT
 376 rlc_to_namespace int SMALLINT
 377 tl_namespace     int SMALLINT
 378 wl_namespace     int SMALLINT
 379
 380 ## Easy enough to change if a wiki ever does grow this big:
 381 ss_good_articles bigint INTEGER
 382 ss_total_edits   bigint INTEGER
 383 ss_total_pages   bigint INTEGER
 384 ss_total_views   bigint INTEGER
 385 ss_users         bigint INTEGER
 386
 387 ## True IP - keep an eye on these, coders tend to make textual assumptions
 388 rc_ip varbinary(40) CIDR # Want to keep an eye on this
 389
 390 ## Others:
 391 tc_time int TIMESTAMPTZ
 392
 393
 394 };
 395
 396 my %colmap;
 397 for (split /\n/ => $COLMAP) {
 398         next unless /^\w/;
 399         s/(.*?)#.*/$1/;
 400         my ($col,@maps) = split / +/, $_;
 401         for (@maps) {
 402                 $colmap{$col}{$_} = 1;
 403         }
 404 }
 405
 406 my %colmapok;
 407 for (split /\n/ => $COLMAPOK) {
 408         next unless /^\w/;
 409         my ($col,$old,$new) = split / +/, $_;
 410         $colmapok{$col}{$old}{$new} = 1;
 411 }
 412
 413 ## Old but not new
 414 for my $t (sort keys %{$old{$oldfile}}) {
 415         if (!exists $new{$t} and !exists $ok{OLD}{$t}) {
 416                 print "Table not in $new: $t\n";
 417                 next;
 418         }
 419         next if exists $ok{OLD}{$t} and !$ok{OLD}{$t};
 420         my $newt = exists $ok{OLD}{$t} ? $ok{OLD}{$t} : $t;
 421         my $oldcol = $old{$oldfile}{$t}{column};
 422         my $oldcolfull = $old{$oldfile}{$t}{columnfull};
 423         my $newcol = $new{$newt}{column};
 424         for my $c (keys %$oldcol) {
 425                 if (!exists $newcol->{$c}) {
 426                         print "Column $t.$c not in $new\n";
 427                         next;
 428                 }
 429         }
 430         for my $c (sort keys %$newcol) {
 431                 if (!exists $oldcol->{$c}) {
 432                         print "Column $t.$c not in $oldfile\n";
 433                         next;
 434                 }
 435                 ## Column types (roughly) match up?
 436                 my $new = $newcol->{$c};
 437                 my $old = $oldcolfull->{$c};
 438
 439                 ## Known exceptions:
 440                 next if exists $colmapok{$c}{$old}{$new};
 441
 442                 $old =~ s/ENUM.*/ENUM/;
 443                 if (! exists $colmap{$old}{$new}) {
 444                         print "Column types for $t.$c do not match: $old does not map to $new\n";
 445                 }
 446         }
 447 }
 448 ## New but not old:
 449 for (sort keys %new) {
 450         if (!exists $old{$oldfile}{$_} and !exists $ok{NEW}{$_}) {
 451                 print "Not in $oldfile: $_\n";
 452                 next;
 453         }
 454 }
 455
 456
 457 } ## end each file to be parsed
 458
 459
 460 sub check_includes_dir {
 461
 462         ## Check for some common errors in the files in the includes directory
 463
 464         print "Checking files in includes directory...\n";
 465         my $dir = '../../includes';
 466         opendir my $dh, $dir or die qq{Could not opendir $dir: $!\n};
 467         for my $file (grep { -f "$dir/$_" and /\.php$/ } readdir $dh) {
 468                 $file = "$dir/$file";
 469                 open my $fh, '<', $file or die qq{Could not open "$file": $!\n};
 470                 while (<$fh>) {
 471                         if (/FORCE INDEX/ and $file !~ /Database.php/) {
 472                                 warn "Found FORCE INDEX string at line $. of $file\n";
 473                         }
 474                         if (/REPLACE INTO/ and $file !~ /Database/) {
 475                                 warn "Found REPLACE INTO string at line $. of $file\n";
 476                         }
 477                         if (/\bIF\s*\(/ and $file !~ /Database.php/) {
 478                                 warn "Found IF string at line $. of $file\n";
 479                         }
 480                         if (/\bCONCAT\b/ and $file !~ /Database.php/) {
 481                                 warn "Found CONCAT string at line $. of $file\n";
 482                         }
 483                 }
 484                 close $fh or die qq{Could not close "$file": $!\n};
 485         }
 486         closedir $dh or die qq{Closedir failed?!\n};
 487
 488         return;
 489
 490 } ## end of check_includes_dir
 491
 492 __DATA__
 493 ## Known exceptions
 494 OLD: searchindex          ## We use tsearch2 directly on the page table instead
 495 RENAME: user mwuser       ## Reserved word causing lots of problems
 496 RENAME: text pagecontent  ## Reserved word
 497 NEW: mediawiki_version    ## Just us, for now
 498 XFILE: ../archives/patch-profiling.sql