Add $wgCdnReboundPurgeDelay for more consistent CDN purges

Bug: T113192
Change-Id: I89deb4f8143c1cc6154cdd05bcee1f49d3e3a75a

diff --git a/RELEASE-NOTES-1.27 b/RELEASE-NOTES-1.27
index b41b8c2..ab136b1 100644 (file)
--- a/RELEASE-NOTES-1.27
+++ b/RELEASE-NOTES-1.27
@@ -97,6 +97,8 @@ production.
 * $wgMaxUserDBWriteDuration added to limit huge user-generated transactions.
   Regular web request transactions that takes longer than this are aborted.
 * Added a new hook, 'TitleMoveCompleting', which runs before a page move is committed.
+* $wgCdnReboundPurgeDelay was added to provide secondary delayed purges of URLs
+  from CDN to mitigate DB replication lag and WAN cache purge lag.
 
 === External library changes in 1.27 ===
 ==== Upgraded external libraries ====

diff --git a/autoload.php b/autoload.php
index fce9cd4..dea31ab 100644 (file)
--- a/autoload.php
+++ b/autoload.php
@@ -200,6 +200,7 @@ $wgAutoloadLocalClasses = array(
        'CdbReader' => __DIR__ . '/includes/compat/CdbCompat.php',
        'CdbWriter' => __DIR__ . '/includes/compat/CdbCompat.php',
        'CdnCacheUpdate' => __DIR__ . '/includes/deferred/CdnCacheUpdate.php',
+       'CdnPurgeJob' => __DIR__ . '/includes/jobqueue/jobs/CdnPurgeJob.php',
        'CentralIdLookup' => __DIR__ . '/includes/user/CentralIdLookup.php',
        'CgzCopyTransaction' => __DIR__ . '/maintenance/storage/recompressTracked.php',
        'ChangePassword' => __DIR__ . '/maintenance/changePassword.php',

diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index 236a3f9..bf85678 100644 (file)
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -2600,6 +2600,15 @@ $wgSquidMaxage = 18000;
  */
 $wgCdnMaxageLagged = 30;
 
+/**
+ * If set, any SquidPurge call on a URL or URLs will send a second purge no less than
+ * this many seconds later via the job queue. This requires delayed job support.
+ * This should be safely higher than the 'max lag' value in $wgLBFactoryConf.
+ *
+ * @since 1.27
+ */
+$wgCdnReboundPurgeDelay = 0;
+
 /**
  * Default maximum age for raw CSS/JS accesses
  *
@@ -6689,10 +6698,11 @@ $wgJobClasses = array(
        'PublishStashedFile' => 'PublishStashedFileJob',
        'ThumbnailRender' => 'ThumbnailRenderJob',
        'recentChangesUpdate' => 'RecentChangesUpdateJob',
-       'refreshLinksPrioritized' => 'RefreshLinksJob', // for cascading protection
-       'refreshLinksDynamic' => 'RefreshLinksJob', // for pages with dynamic content
+       'refreshLinksPrioritized' => 'RefreshLinksJob',
+       'refreshLinksDynamic' => 'RefreshLinksJob',
        'activityUpdateJob' => 'ActivityUpdateJob',
        'categoryMembershipChange' => 'CategoryMembershipChangeJob',
+       'cdnPurge' => 'CdnPurgeJob',
        'enqueue' => 'EnqueueJob', // local queue for multi-DC setups
        'null' => 'NullJob'
 );

diff --git a/includes/deferred/CdnCacheUpdate.php b/includes/deferred/CdnCacheUpdate.php
index bc2b05c..25c27e3 100644 (file)
--- a/includes/deferred/CdnCacheUpdate.php
+++ b/includes/deferred/CdnCacheUpdate.php
@@ -38,6 +38,13 @@ class CdnCacheUpdate implements DeferrableUpdate, MergeableUpdate {
                $this->urls = $urlArr;
        }
 
+       public function merge( MergeableUpdate $update ) {
+               /** @var CdnCacheUpdate $update */
+               Assert::parameterType( __CLASS__, $update, '$update' );
+
+               $this->urls = array_merge( $this->urls, $update->urls );
+       }
+
        /**
         * Create an update object from an array of Title objects, or a TitleArray object
         *
@@ -67,14 +74,19 @@ class CdnCacheUpdate implements DeferrableUpdate, MergeableUpdate {
         * Purges the list of URLs passed to the constructor.
         */
        public function doUpdate() {
-               self::purge( $this->urls );
-       }
+               global $wgCdnReboundPurgeDelay;
 
-       public function merge( MergeableUpdate $update ) {
-               /** @var CdnCacheUpdate $update */
-               Assert::parameterType( __CLASS__, $update, '$update' );
+               self::purge( $this->urls );
 
-               $this->urls = array_merge( $this->urls, $update->urls );
+               if ( $wgCdnReboundPurgeDelay > 0 ) {
+                       JobQueueGroup::singleton()->lazyPush( new CdnPurgeJob(
+                               Title::makeTitle( NS_SPECIAL, 'Badtitle/' . __CLASS__ ),
+                               array(
+                                       'urls' => $this->urls,
+                                       'jobReleaseTimestamp' => time() + $wgCdnReboundPurgeDelay
+                               )
+                       ) );
+               }
        }
 
        /**

diff --git a/includes/jobqueue/jobs/CdnPurgeJob.php b/includes/jobqueue/jobs/CdnPurgeJob.php
new file mode 100644 (file)
index 0000000..356eeba
--- /dev/null
+++ b/includes/jobqueue/jobs/CdnPurgeJob.php
@@ -0,0 +1,46 @@
+<?php
+/**
+ * Job to purge a set of URLs from CDN
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup JobQueue
+ */
+
+/**
+ * Job to purge a set of URLs from CDN
+ *
+ * @ingroup JobQueue
+ * @since 1.27
+ */
+class CdnPurgeJob extends Job {
+       /**
+        * @param Title $title
+        * @param array $params Job parameters (urls)
+        */
+       function __construct( Title $title, array $params ) {
+               parent::__construct( 'cdnPurge', $title, $params );
+               $this->removeDuplicates = false; // delay semantics are critical
+       }
+
+       public function run() {
+               // Use purge() directly to avoid infinite recursion
+               CdnCacheUpdate::purge( $this->params['urls'] );
+
+               return true;
+       }
+}