Merge "Use adaptive CDN TTLs for page views"
authorjenkins-bot <jenkins-bot@gerrit.wikimedia.org>
Fri, 21 Oct 2016 21:55:52 +0000 (21:55 +0000)
committerGerrit Code Review <gerrit@wikimedia.org>
Fri, 21 Oct 2016 21:55:52 +0000 (21:55 +0000)
1  2 
includes/OutputPage.php
includes/page/Article.php
includes/page/CategoryPage.php

diff --combined includes/OutputPage.php
@@@ -67,6 -67,13 +67,6 @@@ class OutputPage extends ContextSource 
         */
        public $mBodytext = '';
  
 -      /**
 -       * Holds the debug lines that will be output as comments in page source if
 -       * $wgDebugComments is enabled. See also $wgShowDebug.
 -       * @deprecated since 1.20; use MWDebug class instead.
 -       */
 -      public $mDebugtext = '';
 -
        /** @var string Stores contents of "<title>" tag */
        private $mHTMLtitle = '';
  
                $lb->setArray( $arr );
  
                # Fetch existence plus the hiddencat property
 -              $dbr = wfGetDB( DB_SLAVE );
 +              $dbr = wfGetDB( DB_REPLICA );
                $fields = array_merge(
                        LinkCache::getSelectFields(),
                        [ 'page_namespace', 'page_title', 'pp_value' ]
                }
  
                // Include profiling data
 -              $this->setLimitReportData( $parserOutput->getLimitReportData() );
 +              if ( !$this->limitReportData ) {
 +                      $this->setLimitReportData( $parserOutput->getLimitReportData() );
 +              }
  
                // Link flags are ignored for now, but may in the future be
                // used to mark individual language links.
                $this->setCdnMaxage( $this->mCdnMaxage );
        }
  
+       /**
+        * Get TTL in [$minTTL,$maxTTL] in pass it to lowerCdnMaxage()
+        *
+        * This sets and returns $minTTL if $mtime is false or null. Otherwise,
+        * the TTL is higher the older the $mtime timestamp is. Essentially, the
+        * TTL is 90% of the age of the object, subject to the min and max.
+        *
+        * @param string|integer|float|bool|null $mtime Last-Modified timestamp
+        * @param integer $minTTL Mimimum TTL in seconds [default: 1 minute]
+        * @param integer $maxTTL Maximum TTL in seconds [default: $wgSquidMaxage]
+        * @return integer TTL in seconds
+        * @since 1.28
+        */
+       public function adaptCdnTTL( $mtime, $minTTL = 0, $maxTTL = 0 ) {
+               $minTTL = $minTTL ?: IExpiringStore::TTL_MINUTE;
+               $maxTTL = $maxTTL ?: $this->getConfig()->get( 'SquidMaxage' );
+               if ( $mtime === null || $mtime === false ) {
+                       return $minTTL; // entity does not exist
+               }
+               $age = time() - wfTimestamp( TS_UNIX, $mtime );
+               $adaptiveTTL = max( .9 * $age, $minTTL );
+               $adaptiveTTL = min( $adaptiveTTL, $maxTTL );
+               $this->lowerCdnMaxage( (int)$adaptiveTTL );
+               return $adaptiveTTL;
+       }
        /**
         * Use enableClientCache(false) to force it to send nocache headers
         *
        /**
         * Finally, all the text has been munged and accumulated into
         * the object, let's actually output it:
 +       *
 +       * @param bool $return Set to true to get the result as a string rather than sending it
 +       * @return string|null
 +       * @throws Exception
 +       * @throws FatalError
 +       * @throws MWException
         */
 -      public function output() {
 +      public function output( $return = false ) {
 +              global $wgContLang;
 +
                if ( $this->mDoNothing ) {
 -                      return;
 +                      return $return ? '' : null;
                }
  
                $response = $this->getRequest()->response();
                                }
                        }
  
 -                      return;
 +                      return $return ? '' : null;
                } elseif ( $this->mStatusCode ) {
                        $response->statusHeader( $this->mStatusCode );
                }
                ob_start();
  
                $response->header( 'Content-type: ' . $config->get( 'MimeType' ) . '; charset=UTF-8' );
 -              $response->header( 'Content-language: ' . $config->get( 'LanguageCode' ) );
 +              $response->header( 'Content-language: ' . $wgContLang->getHtmlCode() );
  
                // Avoid Internet Explorer "compatibility view" in IE 8-10, so that
                // jQuery etc. can work correctly.
                        // Hook that allows last minute changes to the output page, e.g.
                        // adding of CSS or Javascript by extensions.
                        Hooks::run( 'BeforePageDisplay', [ &$this, &$sk ] );
 -                      $this->getSkin()->setupSkinUserCss( $this );
  
                        try {
                                $sk->outputPage();
  
                $this->sendCacheControl();
  
 -              ob_end_flush();
 -
 +              if ( $return ) {
 +                      return ob_get_clean();
 +              } else {
 +                      ob_end_flush();
 +                      return null;
 +              }
        }
  
        /**
         * Output a standard error page
         *
         * showErrorPage( 'titlemsg', 'pagetextmsg' );
 -       * showErrorPage( 'titlemsg', 'pagetextmsg', array( 'param1', 'param2' ) );
 +       * showErrorPage( 'titlemsg', 'pagetextmsg', [ 'param1', 'param2' ] );
         * showErrorPage( 'titlemsg', $messageObject );
         * showErrorPage( $titleMessageObject, $messageObject );
         *
        /**
         * Output a standard permission error page
         *
 -       * @param array $errors Error message keys
 +       * @param array $errors Error message keys or [key, param...] arrays
         * @param string $action Action that was denied or null if unknown
         */
        public function showPermissionsErrorPage( array $errors, $action = null ) {
 +              foreach ( $errors as $key => $error ) {
 +                      $errors[$key] = (array)$error;
 +              }
 +
                // For some action (read, edit, create and upload), display a "login to do this action"
                // error if all of the following conditions are met:
                // 1. the user is not logged in
        }
  
        /**
 -       * Show a warning about slave lag
 +       * Show a warning about replica DB lag
         *
         * If the lag is higher than $wgSlaveLagCritical seconds,
         * then the warning is a bit more obvious. If the lag is
        public function showLagWarning( $lag ) {
                $config = $this->getConfig();
                if ( $lag >= $config->get( 'SlaveLagWarning' ) ) {
 +                      $lag = floor( $lag ); // floor to avoid nano seconds to display
                        $message = $lag < $config->get( 'SlaveLagCritical' )
                                ? 'lag-warn-normal'
                                : 'lag-warn-high';
                                'user.styles',
                                'user.cssprefs',
                        ] );
 +                      $this->getSkin()->setupSkinUserCss( $this );
  
                        // Prepare exempt modules for buildExemptModules()
                        $exemptGroups = [ 'site' => [], 'noscript' => [], 'private' => [], 'user' => [] ];
                        $exemptStates = [];
 -                      $moduleStyles = array_filter( $this->getModuleStyles( /*filter*/ true ),
 +                      $moduleStyles = $this->getModuleStyles( /*filter*/ true );
 +
 +                      // Preload getTitleInfo for isKnownEmpty calls below and in ResourceLoaderClientHtml
 +                      // Separate user-specific batch for improved cache-hit ratio.
 +                      $userBatch = [ 'user.styles', 'user' ];
 +                      $siteBatch = array_diff( $moduleStyles, $userBatch );
 +                      $dbr = wfGetDB( DB_REPLICA );
 +                      ResourceLoaderWikiModule::preloadTitleInfo( $context, $dbr, $siteBatch );
 +                      ResourceLoaderWikiModule::preloadTitleInfo( $context, $dbr, $userBatch );
 +
 +                      // Filter out modules handled by buildExemptModules()
 +                      $moduleStyles = array_filter( $moduleStyles,
                                function ( $name ) use ( $rl, $context, &$exemptGroups, &$exemptStates ) {
                                        $module = $rl->getModule( $name );
                                        if ( $module ) {
 -                                              $group = $module->getGroup();
                                                if ( $name === 'user.styles' && $this->isUserCssPreview() ) {
                                                        $exemptStates[$name] = 'ready';
                                                        // Special case in buildExemptModules()
                                                        return false;
                                                }
 -                                              if ( $name === 'site.styles' ) {
 -                                                      // HACK: Technically, 'site.styles' isn't in a separate request group.
 -                                                      // But, in order to ensure its styles are in the right position,
 -                                                      // pretend it's in a group called 'site'.
 -                                                      $group = 'site';
 -                                              }
 +                                              $group = $module->getGroup();
                                                if ( isset( $exemptGroups[$group] ) ) {
                                                        $exemptStates[$name] = 'ready';
                                                        if ( !$module->isKnownEmpty( $context ) ) {
                        );
                        $this->rlExemptStyleModules = $exemptGroups;
  
 -                      // Manually handled by getBottomScripts()
 -                      $userModule = $rl->getModule( 'user' );
 -                      $userState = $userModule->isKnownEmpty( $context ) && !$this->isUserJsPreview()
 -                              ? 'ready'
 -                              : 'loading';
 -                      $this->rlUserModuleState = $exemptStates['user'] = $userState;
 +                      $isUserModuleFiltered = !$this->filterModules( [ 'user' ] );
 +                      // If this page filters out 'user', makeResourceLoaderLink will drop it.
 +                      // Avoid indefinite "loading" state or untrue "ready" state (T145368).
 +                      if ( !$isUserModuleFiltered ) {
 +                              // Manually handled by getBottomScripts()
 +                              $userModule = $rl->getModule( 'user' );
 +                              $userState = $userModule->isKnownEmpty( $context ) && !$this->isUserJsPreview()
 +                                      ? 'ready'
 +                                      : 'loading';
 +                              $this->rlUserModuleState = $exemptStates['user'] = $userState;
 +                      }
  
                        $rlClient = new ResourceLoaderClientHtml( $context, $this->getTarget() );
                        $rlClient->setConfig( $this->getJSVars() );
                        }
                }
  
 -              $chunks[] = ResourceLoader::makeInlineScript(
 -                      ResourceLoader::makeConfigSetScript(
 -                              [ 'wgPageParseReport' => $this->limitReportData ],
 -                              true
 -                      )
 -              );
 +              if ( $this->limitReportData ) {
 +                      $chunks[] = ResourceLoader::makeInlineScript(
 +                              ResourceLoader::makeConfigSetScript(
 +                                      [ 'wgPageParseReport' => $this->limitReportData ],
 +                                      true
 +                              )
 +                      );
 +              }
  
                return self::combineWrappedStrings( $chunks );
        }
                if ( $user->isLoggedIn() ) {
                        $vars['wgUserId'] = $user->getId();
                        $vars['wgUserEditCount'] = $user->getEditCount();
 -                      $userReg = wfTimestampOrNull( TS_UNIX, $user->getRegistration() );
 -                      $vars['wgUserRegistration'] = $userReg !== null ? ( $userReg * 1000 ) : null;
 +                      $userReg = $user->getRegistration();
 +                      $vars['wgUserRegistration'] = $userReg ? wfTimestamp( TS_UNIX, $userReg ) * 1000 : null;
                        // Get the revision ID of the oldest new message on the user's talk
                        // page. This can be used for constructing new message alerts on
                        // the client side.
@@@ -99,7 -99,9 +99,7 @@@ class Article implements Page 
         */
        public static function newFromID( $id ) {
                $t = Title::newFromID( $id );
 -              # @todo FIXME: Doesn't inherit right
 -              return $t == null ? null : new self( $t );
 -              # return $t == null ? null : new static( $t ); // PHP 5.3
 +              return $t == null ? null : new static( $t );
        }
  
        /**
                return $article;
        }
  
 +      /**
 +       * Get the page this view was redirected from
 +       * @return Title|null
 +       * @since 1.28
 +       */
 +      public function getRedirectedFrom() {
 +              return $this->mRedirectedFrom;
 +      }
 +
        /**
         * Tell the page view functions that this view was redirected
         * from another page on the wiki.
         * @return string Return the text of this revision
         */
        public function getContent() {
 -              ContentHandler::deprecated( __METHOD__, '1.21' );
 +              wfDeprecated( __METHOD__, '1.21' );
                $content = $this->getContentObject();
                return ContentHandler::getContentText( $content );
        }
        function fetchContent() {
                // BC cruft!
  
 -              ContentHandler::deprecated( __METHOD__, '1.21' );
 +              wfDeprecated( __METHOD__, '1.21' );
  
                if ( $this->mContentLoaded && $this->mContent ) {
                        return $this->mContent;
  
                // @todo Get rid of mContent everywhere!
                $this->mContent = ContentHandler::getContentText( $content );
 -              ContentHandler::runLegacyHooks( 'ArticleAfterFetchContent', [ &$this, &$this->mContent ] );
 +              ContentHandler::runLegacyHooks(
 +                      'ArticleAfterFetchContent',
 +                      [ &$this, &$this->mContent ],
 +                      '1.21'
 +              );
  
                return $this->mContent;
        }
                $this->mContentObject = $content;
                $this->mRevIdFetched = $this->mRevision->getId();
  
 -              Hooks::run( 'ArticleAfterFetchContentObject', [ &$this, &$this->mContentObject ] );
 +              ContentHandler::runLegacyHooks(
 +                      'ArticleAfterFetchContentObject',
 +                      [ &$this, &$this->mContentObject ],
 +                      '1.21'
 +              );
  
                return $this->mContentObject;
        }
         * page of the given title.
         */
        public function view() {
 -              global $wgUseFileCache, $wgDebugToolbar, $wgMaxRedirects;
 +              global $wgUseFileCache, $wgDebugToolbar;
  
                # Get variables from query string
                # As side effect this will load the revision and update the title
  
                # Try client and file cache
                if ( !$wgDebugToolbar && $oldid === 0 && $this->mPage->checkTouched() ) {
 -                      # Use the greatest of the page's timestamp or the timestamp of any
 -                      # redirect in the chain (bug 67849)
 -                      $timestamp = $this->mPage->getTouched();
 -                      if ( isset( $this->mRedirectedFrom ) ) {
 -                              $timestamp = max( $timestamp, $this->mRedirectedFrom->getTouched() );
 -
 -                              # If there can be more than one redirect in the chain, we have
 -                              # to go through the whole chain too in case an intermediate
 -                              # redirect was changed.
 -                              if ( $wgMaxRedirects > 1 ) {
 -                                      $titles = Revision::newFromTitle( $this->mRedirectedFrom )
 -                                              ->getContent( Revision::FOR_THIS_USER, $user )
 -                                              ->getRedirectChain();
 -                                      $thisTitle = $this->getTitle();
 -                                      foreach ( $titles as $title ) {
 -                                              if ( Title::compare( $title, $thisTitle ) === 0 ) {
 -                                                      break;
 -                                              }
 -                                              $timestamp = max( $timestamp, $title->getTouched() );
 -                                      }
 -                              }
 -                      }
 -
 -                      # Is it client cached?
 -                      if ( $outputPage->checkLastModified( $timestamp ) ) {
 -                              wfDebug( __METHOD__ . ": done 304\n" );
 -
 -                              return;
 -                      # Try file cache
 -                      } elseif ( $wgUseFileCache && $this->tryFileCache() ) {
 +                      # Try to stream the output from file cache
 +                      if ( $wgUseFileCache && $this->tryFileCache() ) {
                                wfDebug( __METHOD__ . ": done file cache\n" );
                                # tell wgOut that output is taken care of
                                $outputPage->disable();
  
                                                # Allow extensions do their own custom view for certain pages
                                                $outputDone = true;
 -                                      } elseif ( !ContentHandler::runLegacyHooks( 'ArticleViewCustom',
 -                                                      [ $this->fetchContentObject(), $this->getTitle(), $outputPage ] ) ) {
 -
 +                                      } elseif ( !ContentHandler::runLegacyHooks(
 +                                              'ArticleViewCustom',
 +                                              [ $this->fetchContentObject(), $this->getTitle(), $outputPage ],
 +                                              '1.21'
 +                                      ) ) {
                                                # Allow extensions do their own custom view for certain pages
                                                $outputDone = true;
                                        }
                        }
                }
  
+               # Use adaptive TTLs for CDN so delayed/failed purges are noticed less often.
+               # This could use getTouched(), but that could be scary for major template edits.
+               $outputPage->adaptCdnTTL( $this->mPage->getTimestamp(), IExpiringStore::TTL_DAY );
                # Check for any __NOINDEX__ tags on the page using $pOutput
                $policy = $this->getRobotPolicy( 'view', $pOutput );
                $outputPage->setIndexPolicy( $policy['index'] );
                $this->mPage->doViewUpdates( $user, $oldid );
  
                $outputPage->addModules( 'mediawiki.action.view.postEdit' );
        }
  
        /**
                        // Give hooks a chance to customise the output
                        if ( ContentHandler::runLegacyHooks(
                                'ShowRawCssJs',
 -                              [ $this->mContentObject, $this->getTitle(), $outputPage ] )
 -                      ) {
 +                              [ $this->mContentObject, $this->getTitle(), $outputPage ],
 +                              '1.24'
 +                      ) ) {
                                // If no legacy hooks ran, display the content of the parser output, including RL modules,
                                // but excluding metadata like categories and language links
                                $po = $this->mContentObject->getParserOutput( $this->getTitle() );
                        return false;
                }
  
 -              $dbr = wfGetDB( DB_SLAVE );
 +              $dbr = wfGetDB( DB_REPLICA );
                $oldestRevisionTimestamp = $dbr->selectField(
                        'revision',
                        'MIN( rev_timestamp )',
  
                if ( !$rc ) {
                        // Don't cache: This can be hit if the page gets accessed very fast after
 -                      // its creation / latest upload or in case we have high slave lag. In case
 +                      // its creation / latest upload or in case we have high replica DB lag. In case
                        // the revision is too old, we will already return above.
                        return false;
                }
                        // This, as a side-effect, also makes sure that the following query isn't being run for
                        // pages with a larger history, unless the user has the 'bigdelete' right
                        // (and is about to delete this page).
 -                      $dbr = wfGetDB( DB_SLAVE );
 +                      $dbr = wfGetDB( DB_REPLICA );
                        $revisions = $edits = (int)$dbr->selectField(
                                'revision',
                                'COUNT(rev_page)',
  
        /**
         * Check if the page can be cached
 +       * @param integer $mode One of the HTMLFileCache::MODE_* constants (since 1.28)
         * @return bool
         */
 -      public function isFileCacheable() {
 +      public function isFileCacheable( $mode = HTMLFileCache::MODE_NORMAL ) {
                $cacheable = false;
  
 -              if ( HTMLFileCache::useFileCache( $this->getContext() ) ) {
 +              if ( HTMLFileCache::useFileCache( $this->getContext(), $mode ) ) {
                        $cacheable = $this->mPage->getId()
                                && !$this->mRedirectedFrom && !$this->getTitle()->isRedirect();
                        // Extension may have reason to disable file caching on some pages.
         * @see WikiPage::doDeleteArticleReal
         */
        public function doDeleteArticleReal(
 -              $reason, $suppress = false, $u1 = null, $u2 = null, &$error = '', User $user = null
 +              $reason, $suppress = false, $u1 = null, $u2 = null, &$error = '', User $user = null,
 +              $tags = []
        ) {
                return $this->mPage->doDeleteArticleReal(
 -                      $reason, $suppress, $u1, $u2, $error, $user
 +                      $reason, $suppress, $u1, $u2, $error, $user, $tags
                );
        }
  
        /**
         * Call to WikiPage function for backwards compatibility.
         * @see WikiPage::doEdit
 +       *
 +       * @deprecated since 1.21: use doEditContent() instead.
         */
        public function doEdit( $text, $summary, $flags = 0, $baseRevId = false, $user = null ) {
 -              ContentHandler::deprecated( __METHOD__, '1.21' );
 +              wfDeprecated( __METHOD__, '1.21' );
                return $this->mPage->doEdit( $text, $summary, $flags, $baseRevId, $user );
        }
  
         * Call to WikiPage function for backwards compatibility.
         * @see WikiPage::doPurge
         */
 -      public function doPurge() {
 -              return $this->mPage->doPurge();
 +      public function doPurge( $flags = WikiPage::PURGE_ALL ) {
 +              return $this->mPage->doPurge( $flags );
        }
  
        /**
         * Call to WikiPage function for backwards compatibility.
 -       * @see WikiPage::doQuickEditContent
 +       * @see WikiPage::getLastPurgeTimestamp
         */
 -      public function doQuickEditContent(
 -              Content $content, User $user, $comment = '', $minor = false, $serialFormat = null
 -      ) {
 -              return $this->mPage->doQuickEditContent(
 -                      $content, $user, $comment, $minor, $serialFormat
 -              );
 +      public function getLastPurgeTimestamp() {
 +              return $this->mPage->getLastPurgeTimestamp();
        }
  
        /**
        /**
         * Call to WikiPage function for backwards compatibility.
         * @see WikiPage::getText
 +       * @deprecated since 1.21 use WikiPage::getContent() instead
         */
        public function getText( $audience = Revision::FOR_PUBLIC, User $user = null ) {
 -              ContentHandler::deprecated( __METHOD__, '1.21' );
 +              wfDeprecated( __METHOD__, '1.21' );
                return $this->mPage->getText( $audience, $user );
        }
  
  
        /**
         * Call to WikiPage function for backwards compatibility.
 +       * @deprecated since 1.21, use prepareContentForEdit
         * @see WikiPage::prepareTextForEdit
         */
        public function prepareTextForEdit( $text, $revid = null, User $user = null ) {
@@@ -43,6 -43,18 +43,6 @@@ class CategoryPage extends Article 
                return new WikiCategoryPage( $title );
        }
  
 -      /**
 -       * Constructor from a page id
 -       * @param int $id Article ID to load
 -       * @return CategoryPage|null
 -       */
 -      public static function newFromID( $id ) {
 -              $t = Title::newFromID( $id );
 -              # @todo FIXME: Doesn't inherit right
 -              return $t == null ? null : new self( $t );
 -              # return $t == null ? null : new static( $t ); // PHP 5.3
 -      }
 -
        function view() {
                $request = $this->getContext()->getRequest();
                $diff = $request->getVal( 'diff' );
                if ( $title->inNamespace( NS_CATEGORY ) ) {
                        $this->closeShowCategory();
                }
+               # Use adaptive TTLs for CDN so delayed/failed purges are noticed less often
+               $outputPage = $this->getContext()->getOutput();
+               $outputPage->adaptCdnTTL( $this->mPage->getTouched(), IExpiringStore::TTL_MINUTE );
        }
  
        function openShowCategory() {