maintenance/postgres/compare_schemas.pl

   1 #!/usr/bin/perl
   2
   3 ## Rough check that the base and postgres "tables.sql" are in sync
   4 ## Should be run from maintenance/postgres
   5
   6 use strict;
   7 use warnings;
   8 use Data::Dumper;
   9
  10 my @old = ('../tables.sql');
  11 my $new = 'tables.sql';
  12 my @xfile;
  13
  14 ## Read in exceptions and other metadata
  15 my %ok;
  16 while (<DATA>) {
  17         next unless /^(\w+)\s*:\s*([^#]+)/;
  18         my ($name,$val) = ($1,$2);
  19         chomp $val;
  20         if ($name eq 'RENAME') {
  21                 die "Invalid rename\n" unless $val =~ /(\w+)\s+(\w+)/;
  22                 $ok{OLD}{$1} = $2;
  23                 $ok{NEW}{$2} = $1;
  24                 next;
  25         }
  26         if ($name eq 'XFILE') {
  27                 push @xfile, $val;
  28                 next;
  29         }
  30         for (split /\s+/ => $val) {
  31                 $ok{$name}{$_} = 0;
  32         }
  33 }
  34
  35 my $datatype = join '|' => qw(
  36 bool
  37 tinyint int bigint real float
  38 tinytext mediumtext text char varchar varbinary
  39 timestamp datetime
  40 tinyblob mediumblob blob
  41 );
  42 $datatype .= q{|ENUM\([\"\w, ]+\)};
  43 $datatype = qr{($datatype)};
  44
  45 my $typeval = qr{(\(\d+\))?};
  46
  47 my $typeval2 = qr{ unsigned| binary| NOT NULL| NULL| auto_increment| default ['\-\d\w"]+| REFERENCES .+CASCADE};
  48
  49 my $indextype = join '|' => qw(INDEX KEY FULLTEXT), 'PRIMARY KEY', 'UNIQUE INDEX', 'UNIQUE KEY';
  50 $indextype = qr{$indextype};
  51
  52 my $engine = qr{TYPE|ENGINE};
  53
  54 my $tabletype = qr{InnoDB|MyISAM|HEAP|HEAP MAX_ROWS=\d+|InnoDB MAX_ROWS=\d+ AVG_ROW_LENGTH=\d+};
  55
  56 my $charset = qr{utf8|binary};
  57
  58 open my $newfh, '<', $new or die qq{Could not open $new: $!\n};
  59
  60
  61 my ($table,%old);
  62
  63 ## Read in the xfiles
  64 my %xinfo;
  65 for my $xfile (@xfile) {
  66         print "Loading $xfile\n";
  67         my $info = &parse_sql($xfile);
  68         for (keys %$info) {
  69                 $xinfo{$_} = $info->{$_};
  70         }
  71 }
  72
  73 for my $oldfile (@old) {
  74         print "Loading $oldfile\n";
  75         my $info = &parse_sql($oldfile);
  76         for (keys %xinfo) {
  77                 $info->{$_} = $xinfo{$_};
  78         }
  79         $old{$oldfile} = $info;
  80 }
  81
  82 sub parse_sql {
  83
  84         my $oldfile = shift;
  85
  86         open my $oldfh, '<', $oldfile or die qq{Could not open $oldfile: $!\n};
  87
  88         my %info;
  89         while (<$oldfh>) {
  90                 next if /^\s*\-\-/ or /^\s+$/;
  91                 s/\s*\-\- [\w ]+$//;
  92                 chomp;
  93
  94                 if (/CREATE\s*TABLE/i) {
  95                         m{^CREATE TABLE /\*\$wgDBprefix\*/(\w+) \($}
  96                                 or die qq{Invalid CREATE TABLE at line $. of $oldfile\n};
  97                         $table = $1;
  98                         $info{$table}{name}=$table;
  99                 }
 100                 elsif (m#^\) /\*\$wgDBTableOptions\*/#) {
 101                         $info{$table}{engine} = 'TYPE';
 102                         $info{$table}{type} = 'variable';
 103                 }
 104                 elsif (/^\) ($engine)=($tabletype);$/) {
 105                         $info{$table}{engine}=$1;
 106                         $info{$table}{type}=$2;
 107                 }
 108                 elsif (/^\) ($engine)=($tabletype), DEFAULT CHARSET=($charset);$/) {
 109                         $info{$table}{engine}=$1;
 110                         $info{$table}{type}=$2;
 111                         $info{$table}{charset}=$3;
 112                 }
 113                 elsif (/^  (\w+) $datatype$typeval$typeval2{0,3},?$/) {
 114                         $info{$table}{column}{$1} = $2;
 115                 }
 116                 elsif (/^  ($indextype)(?: (\w+))? \(([\w, \(\)]+)\),?$/) {
 117                         $info{$table}{lc $1.'_name'} = $2 ? $2 : '';
 118                         $info{$table}{lc $1.'pk_target'} = $3;
 119                 }
 120                 else {
 121                         die "Cannot parse line $. of $oldfile:\n$_\n";
 122                 }
 123
 124         }
 125         close $oldfh;
 126
 127         return \%info;
 128
 129 } ## end of parse_sql
 130
 131 for my $oldfile (@old) {
 132
 133 ## Begin non-standard indent
 134
 135 ## MySQL sanity checks
 136 for my $table (sort keys %{$old{$oldfile}}) {
 137         my $t = $old{$oldfile}{$table};
 138         if (($oldfile =~ /5/ and $t->{engine} ne 'ENGINE')
 139                 or
 140                 ($oldfile !~ /5/ and $t->{engine} ne 'TYPE')) {
 141                 die "Invalid engine for $oldfile: $t->{engine}\n" unless $t->{name} eq 'profiling';
 142         }
 143         my $charset = $t->{charset} || '';
 144         if ($oldfile !~ /binary/ and $charset eq 'binary') {
 145                 die "Invalid charset for $oldfile: $charset\n";
 146         }
 147 }
 148
 149 my $dtype = join '|' => qw(
 150 SMALLINT INTEGER BIGINT NUMERIC SERIAL
 151 TEXT CHAR VARCHAR
 152 BYTEA
 153 TIMESTAMPTZ
 154 CIDR
 155 );
 156 $dtype = qr{($dtype)};
 157 my %new;
 158 my ($infunction,$inview,$inrule,$lastcomma) = (0,0,0,0);
 159 seek $newfh, 0, 0;
 160 while (<$newfh>) {
 161         next if /^\s*\-\-/ or /^\s*$/;
 162         s/\s*\-\- [\w ']+$//;
 163         next if /^BEGIN;/ or /^SET / or /^COMMIT;/;
 164         next if /^CREATE SEQUENCE/;
 165         next if /^CREATE(?: UNIQUE)? INDEX/;
 166         next if /^CREATE FUNCTION/;
 167         next if /^CREATE TRIGGER/ or /^  FOR EACH ROW/;
 168         next if /^INSERT INTO/ or /^  VALUES \(/;
 169         next if /^ALTER TABLE/;
 170         chomp;
 171
 172         if (/^\$mw\$;?$/) {
 173                 $infunction = $infunction ? 0 : 1;
 174                 next;
 175         }
 176         next if $infunction;
 177
 178         next if /^CREATE VIEW/ and $inview = 1;
 179         if ($inview) {
 180                 /;$/ and $inview = 0;
 181                 next;
 182         }
 183
 184         next if /^CREATE RULE/ and $inrule = 1;
 185         if ($inrule) {
 186                 /;$/ and $inrule = 0;
 187                 next;
 188         }
 189
 190         if (/^CREATE TABLE "?(\w+)"? \($/) {
 191                 $table = $1;
 192                 $new{$table}{name}=$table;
 193                 $lastcomma = 1;
 194         }
 195         elsif (/^\);$/) {
 196                 if ($lastcomma) {
 197                         warn "Stray comma before line $.\n";
 198                 }
 199         }
 200         elsif (/^  (\w+) +$dtype.*?(,?)(?: --.*)?$/) {
 201                 $new{$table}{column}{$1} = $2;
 202                 if (!$lastcomma) {
 203                         print "Missing comma before line $. of $new\n";
 204                 }
 205                 $lastcomma = $3 ? 1 : 0;
 206         }
 207         else {
 208                 die "Cannot parse line $. of $new:\n$_\n";
 209         }
 210 }
 211
 212 ## Old but not new
 213 for my $t (sort keys %{$old{$oldfile}}) {
 214         if (!exists $new{$t} and !exists $ok{OLD}{$t}) {
 215                 print "Table not in $new: $t\n";
 216                 next;
 217         }
 218         next if exists $ok{OLD}{$t} and !$ok{OLD}{$t};
 219         my $newt = exists $ok{OLD}{$t} ? $ok{OLD}{$t} : $t;
 220         my $oldcol = $old{$oldfile}{$t}{column};
 221         my $newcol = $new{$newt}{column};
 222         for my $c (keys %$oldcol) {
 223                 if (!exists $newcol->{$c}) {
 224                         print "Column $t.$c not in $new\n";
 225                         next;
 226                 }
 227         }
 228         for my $c (keys %$newcol) {
 229                 if (!exists $oldcol->{$c}) {
 230                         print "Column $t.$c not in $oldfile\n";
 231                         next;
 232                 }
 233         }
 234 }
 235 ## New but not old:
 236 for (sort keys %new) {
 237         if (!exists $old{$oldfile}{$_} and !exists $ok{NEW}{$_}) {
 238                 print "Not in $oldfile: $_\n";
 239                 next;
 240         }
 241 }
 242
 243
 244 } ## end each file to be parsed
 245
 246
 247 __DATA__
 248 ## Known exceptions
 249 OLD: searchindex          ## We use tsearch2 directly on the page table instead
 250 RENAME: user mwuser       ## Reserved word causing lots of problems
 251 RENAME: text pagecontent  ## Reserved word
 252 NEW: mediawiki_version    ## Just us, for now
 253 XFILE: ../archives/patch-profiling.sql