Populate externallinks.el_index_60 and drop default
authorBrad Jorsch <bjorsch@wikimedia.org>
Fri, 18 Nov 2016 20:42:11 +0000 (15:42 -0500)
committerBrad Jorsch <bjorsch@wikimedia.org>
Tue, 15 May 2018 16:08:35 +0000 (12:08 -0400)
Adds a maintenance script to populate the field, has that be
automatically run during update.php, and drops the no-longer-needed
default value on the column (where possible: mssql has some sort of
constraint thing going on that I have no idea how it works).

Bug: T59176
Change-Id: I971edf013a1a39466aca3b6e34c915cb24fd3aa7

16 files changed:
autoload.php
includes/deferred/LinksUpdate.php
includes/installer/DatabaseUpdater.php
includes/installer/MssqlUpdater.php
includes/installer/MysqlUpdater.php
includes/installer/OracleUpdater.php
includes/installer/PostgresUpdater.php
includes/installer/SqliteUpdater.php
maintenance/archives/patch-externallinks-el_index_60-drop-default.sql [new file with mode: 0644]
maintenance/fixExtLinksProtocolRelative.php
maintenance/mssql/archives/patch-externallinks-el_index_60-drop-default.sql [new file with mode: 0644]
maintenance/mssql/tables.sql
maintenance/populateExternallinksIndex60.php [new file with mode: 0644]
maintenance/postgres/tables.sql
maintenance/sqlite/archives/patch-externallinks-el_index_60-drop-default.sql [new file with mode: 0644]
maintenance/tables.sql

index 6e123a1..e0b810b 100644 (file)
@@ -1161,6 +1161,7 @@ $wgAutoloadLocalClasses = [
        'PopulateBacklinkNamespace' => __DIR__ . '/maintenance/populateBacklinkNamespace.php',
        'PopulateCategory' => __DIR__ . '/maintenance/populateCategory.php',
        'PopulateContentModel' => __DIR__ . '/maintenance/populateContentModel.php',
+       'PopulateExternallinksIndex60' => __DIR__ . '/maintenance/populateExternallinksIndex60.php',
        'PopulateFilearchiveSha1' => __DIR__ . '/maintenance/populateFilearchiveSha1.php',
        'PopulateImageSha1' => __DIR__ . '/maintenance/populateImageSha1.php',
        'PopulateInterwiki' => __DIR__ . '/maintenance/populateInterwiki.php',
index 4ddd151..398df01 100644 (file)
@@ -569,6 +569,7 @@ class LinksUpdate extends DataUpdate implements EnqueueableDataUpdate {
                                        'el_from' => $this->mId,
                                        'el_to' => $url,
                                        'el_index' => $index,
+                                       'el_index_60' => substr( $index, 0, 60 ),
                                ];
                        }
                }
index 1f6110b..e2deed1 100644 (file)
@@ -1287,4 +1287,21 @@ abstract class DatabaseUpdater {
                 }
         }
 
+       /**
+        * Populates the externallinks.el_index_60 field
+        * @since 1.32
+        */
+       protected function populateExternallinksIndex60() {
+               if ( !$this->updateRowExists( 'populate externallinks.el_index_60' ) ) {
+                       $this->output(
+                               "Populating el_index_60 field, printing progress markers. For large\n" .
+                               "databases, you may want to hit Ctrl-C and do this manually with\n" .
+                               "maintenance/populateExternallinksIndex60.php.\n"
+                       );
+                       $task = $this->maintenance->runChild( 'PopulateExternallinksIndex60',
+                               'populateExternallinksIndex60.php' );
+                       $task->execute();
+                       $this->output( "done.\n" );
+               }
+       }
 }
index e389135..e04ad8f 100644 (file)
@@ -131,6 +131,9 @@ class MssqlUpdater extends DatabaseUpdater {
 
                        // 1.32
                        [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+                       [ 'populateExternallinksIndex60' ],
+                       [ 'modifyfield', 'externallinks', 'el_index_60',
+                               'patch-externallinks-el_index_60-drop-default.sql' ],
                ];
        }
 
index b4bb194..1b9faf2 100644 (file)
@@ -351,6 +351,9 @@ class MysqlUpdater extends DatabaseUpdater {
 
                        // 1.32
                        [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+                       [ 'populateExternallinksIndex60' ],
+                       [ 'modifyfield', 'externallinks', 'el_index_60',
+                               'patch-externallinks-el_index_60-drop-default.sql' ],
                ];
        }
 
index b08e26f..d55b520 100644 (file)
@@ -150,6 +150,7 @@ class OracleUpdater extends DatabaseUpdater {
 
                        // 1.32
                        [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+                       [ 'populateExternallinksIndex60' ],
 
                        // KEEP THIS AT THE BOTTOM!!
                        [ 'doRebuildDuplicateFunction' ],
index c72e206..70547b4 100644 (file)
@@ -572,6 +572,8 @@ class PostgresUpdater extends DatabaseUpdater {
 
                        // 1.32
                        [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+                       [ 'populateExternallinksIndex60' ],
+                       [ 'dropDefault', 'externallinks', 'el_index_60' ],
                ];
        }
 
@@ -907,6 +909,20 @@ END;
                }
        }
 
+       /**
+        * Drop a default value from a field
+        * @since 1.32
+        * @param string $table
+        * @param string $field
+        */
+       protected function dropDefault( $table, $field ) {
+               $info = $this->db->fieldInfo( $table, $field );
+               if ( $info->defaultValue() !== false ) {
+                       $this->output( "Removing '$table.$field' default value\n" );
+                       $this->db->query( "ALTER TABLE $table ALTER $field DROP DEFAULT" );
+               }
+       }
+
        protected function changeNullableField( $table, $field, $null, $update = false ) {
                $fi = $this->db->fieldInfo( $table, $field );
                if ( is_null( $fi ) ) {
index 76e1ca9..c2f2213 100644 (file)
@@ -215,6 +215,9 @@ class SqliteUpdater extends DatabaseUpdater {
 
                        // 1.32
                        [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+                       [ 'populateExternallinksIndex60' ],
+                       [ 'modifyfield', 'externallinks', 'el_index_60',
+                               'patch-externallinks-el_index_60-drop-default.sql' ],
                ];
        }
 
diff --git a/maintenance/archives/patch-externallinks-el_index_60-drop-default.sql b/maintenance/archives/patch-externallinks-el_index_60-drop-default.sql
new file mode 100644 (file)
index 0000000..f9242c5
--- /dev/null
@@ -0,0 +1,2 @@
+-- @since 1.32
+ALTER TABLE /*$wgDBprefix*/externallinks ALTER COLUMN el_index_60 DROP DEFAULT;
index c70b8be..f5861e5 100644 (file)
@@ -71,11 +71,13 @@ class FixExtLinksProtocolRelative extends LoggedUpdateMaintenance {
                                                'el_from' => $row->el_from,
                                                'el_to' => $row->el_to,
                                                'el_index' => "http:{$row->el_index}",
+                                               'el_index_60' => substr( "http:{$row->el_index}", 0, 60 ),
                                        ],
                                        [
                                                'el_from' => $row->el_from,
                                                'el_to' => $row->el_to,
                                                'el_index' => "https:{$row->el_index}",
+                                               'el_index_60' => substr( "https:{$row->el_index}", 0, 60 ),
                                        ]
                                ], __METHOD__, [ 'IGNORE' ]
                        );
diff --git a/maintenance/mssql/archives/patch-externallinks-el_index_60-drop-default.sql b/maintenance/mssql/archives/patch-externallinks-el_index_60-drop-default.sql
new file mode 100644 (file)
index 0000000..7755e66
--- /dev/null
@@ -0,0 +1,13 @@
+DECLARE @sql nvarchar(max)
+SET @sql=''
+
+SELECT @sql= @sql + 'ALTER TABLE /*_*/externallinks DROP CONSTRAINT ' + df.name + '; '
+FROM sys.default_constraints df
+JOIN sys.columns c
+       ON c.object_id = df.parent_object_id
+       AND c.column_id = df.parent_column_id
+WHERE
+       df.parent_object_id =  OBJECT_ID('/*_*/externallinks')
+       AND c.name = 'el_index_60';--
+
+EXEC sp_executesql @sql;
index 858260b..315cb20 100644 (file)
@@ -549,8 +549,7 @@ CREATE TABLE /*_*/externallinks (
 
   -- This is el_index truncated to 60 bytes to allow for sortable queries that
   -- aren't supported by a partial index.
-  -- @todo Drop the default once this is deployed everywhere and code is populating it.
-  el_index_60 varbinary(60) NOT NULL default ''
+  el_index_60 varbinary(60) NOT NULL
 );
 
 CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from);
diff --git a/maintenance/populateExternallinksIndex60.php b/maintenance/populateExternallinksIndex60.php
new file mode 100644 (file)
index 0000000..9b029fe
--- /dev/null
@@ -0,0 +1,89 @@
+<?php
+/**
+ * Populates the el_index_60 field in the externallinks table.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+
+require_once __DIR__ . '/Maintenance.php';
+
+/**
+ * Maintenance script that populates the el_index_60 field in the externallinks
+ * table.
+ *
+ * @ingroup Maintenance
+ * @since 1.32
+ */
+class PopulateExternallinksIndex60 extends LoggedUpdateMaintenance {
+       public function __construct() {
+               parent::__construct();
+               $this->addDescription(
+                       'Populates the el_index_60 field in the externallinks table' );
+               $this->setBatchSize( 200 );
+       }
+
+       protected function getUpdateKey() {
+               return 'populate externallinks.el_index_60';
+       }
+
+       protected function updateSkippedMessage() {
+               return 'externallinks.el_index_60 already populated.';
+       }
+
+       protected function doDBUpdates() {
+               $dbw = $this->getDB( DB_MASTER );
+               $this->output( "Populating externallinks.el_index_60...\n" );
+
+               $count = 0;
+               $start = 0;
+               $last = $dbw->selectField( 'externallinks', 'MAX(el_id)', '', __METHOD__ );
+               while ( $start <= $last ) {
+                       $end = $start + $this->mBatchSize;
+                       $this->output( "el_id $start - $end of $last\n" );
+                       $res = $dbw->select( 'externallinks', [ 'el_id', 'el_index' ],
+                               [
+                                       "el_id > $start",
+                                       "el_id <= $end",
+                                       'el_index_60' => '',
+                               ],
+                               __METHOD__,
+                               [ 'ORDER BY' => 'el_id' ]
+                       );
+                       foreach ( $res as $row ) {
+                               $count++;
+                               $dbw->update( 'externallinks',
+                                       [
+                                               'el_index_60' => substr( $row->el_index, 0, 60 ),
+                                       ],
+                                       [
+                                               'el_id' => $row->el_id,
+                                       ], __METHOD__, [ 'IGNORE' ]
+                               );
+                       }
+                       wfWaitForSlaves();
+                       $start = $end;
+               }
+               $this->output( "Done, $count rows updated.\n" );
+
+               return true;
+       }
+}
+
+$maintClass = "PopulateExternallinksIndex60";
+require_once RUN_MAINTENANCE_IF_MAIN;
index 1a53359..492a0d5 100644 (file)
@@ -373,7 +373,7 @@ CREATE TABLE externallinks (
   el_from     INTEGER     NOT NULL  REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   el_to       TEXT        NOT NULL,
   el_index    TEXT        NOT NULL,
-  el_index_60 BYTEA       NOT NULL  DEFAULT ''
+  el_index_60 BYTEA       NOT NULL
 );
 ALTER SEQUENCE externallinks_el_id_seq OWNED BY externallinks.el_id;
 CREATE INDEX externallinks_from_to ON externallinks (el_from,el_to);
diff --git a/maintenance/sqlite/archives/patch-externallinks-el_index_60-drop-default.sql b/maintenance/sqlite/archives/patch-externallinks-el_index_60-drop-default.sql
new file mode 100644 (file)
index 0000000..1d82f4d
--- /dev/null
@@ -0,0 +1,21 @@
+-- To change the default on one column, sqlite requires we copy the whole table
+
+CREATE TABLE /*_*/externallinks_tmp (
+  el_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  el_from int unsigned NOT NULL default 0,
+  el_to blob NOT NULL,
+  el_index blob NOT NULL,
+  el_index_60 varbinary(60) NOT NULL
+) /*$wgDBTableOptions*/;
+
+INSERT INTO /*_*/externallinks_tmp
+       SELECT el_id, el_from, el_to, el_index, el_index_60 FROM /*_*/externallinks;
+
+DROP TABLE /*_*/externallinks;
+ALTER TABLE /*_*/externallinks_tmp RENAME TO /*_*/externallinks;
+
+CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from, el_to(40));
+CREATE INDEX /*i*/el_to ON /*_*/externallinks (el_to(60), el_from);
+CREATE INDEX /*i*/el_index ON /*_*/externallinks (el_index(60));
+CREATE INDEX /*i*/el_index_60 ON /*_*/externallinks (el_index_60, el_id);
+CREATE INDEX /*i*/el_from_index_60 ON /*_*/externallinks (el_from, el_index_60, el_id);
index 34c63ed..ed3bd43 100644 (file)
@@ -932,8 +932,7 @@ CREATE TABLE /*_*/externallinks (
 
   -- This is el_index truncated to 60 bytes to allow for sortable queries that
   -- aren't supported by a partial index.
-  -- @todo Drop the default once this is deployed everywhere and code is populating it.
-  el_index_60 varbinary(60) NOT NULL default ''
+  el_index_60 varbinary(60) NOT NULL
 ) /*$wgDBTableOptions*/;
 
 -- Forward index, for page edit, save