abstract class DatabaseBase implements IDatabase {
/** Number of times to re-try an operation in case of deadlock */
const DEADLOCK_TRIES = 4;
-
/** Minimum time to wait before retry, in microseconds */
const DEADLOCK_DELAY_MIN = 500000;
-
/** Maximum time to wait before retry */
const DEADLOCK_DELAY_MAX = 1500000;
+ /** How long before it is worth doing a dummy query to test the connection */
+ const PING_TTL = 1.0;
+ const PING_QUERY = 'SELECT 1 AS ping';
+
+ const TINY_WRITE_SEC = .010;
+ const SLOW_WRITE_SEC = .500;
+ const SMALL_WRITE_ROWS = 100;
+
+ /** @var string SQL query */
protected $mLastQuery = '';
+ /** @var bool */
protected $mDoneWrites = false;
+ /** @var string|bool */
protected $mPHPError = false;
-
- protected $mServer, $mUser, $mPassword, $mDBname;
+ /** @var string */
+ protected $mServer;
+ /** @var string */
+ protected $mUser;
+ /** @var string */
+ protected $mPassword;
+ /** @var string */
+ protected $mDBname;
+ /** @var bool */
+ protected $cliMode;
/** @var BagOStuff APC cache */
protected $srvCache;
/** @var resource Database connection */
protected $mConn = null;
+ /** @var bool */
protected $mOpened = false;
- /** @var callable[] */
+ /** @var array[] List of (callable, method name) */
protected $mTrxIdleCallbacks = [];
- /** @var callable[] */
+ /** @var array[] List of (callable, method name) */
protected $mTrxPreCommitCallbacks = [];
-
+ /** @var array[] List of (callable, method name) */
+ protected $mTrxEndCallbacks = [];
+ /** @var array[] Map of (name => (callable, method name)) */
+ protected $mTrxRecurringCallbacks = [];
+ /** @var bool Whether to suppress triggering of transaction end callbacks */
+ protected $mTrxEndCallbacksSuppressed = false;
+
+ /** @var string */
protected $mTablePrefix;
+ /** @var string */
protected $mSchema;
+ /** @var integer */
protected $mFlags;
+ /** @var bool */
protected $mForeign;
+ /** @var array */
protected $mLBInfo = [];
+ /** @var bool|null */
protected $mDefaultBigSelects = null;
+ /** @var array|bool */
protected $mSchemaVars = false;
/** @var array */
protected $mSessionVars = [];
-
+ /** @var array|null */
protected $preparedArgs;
-
+ /** @var string|bool|null Stashed value of html_errors INI setting */
protected $htmlErrors;
-
+ /** @var string */
protected $delimiter = ';';
/**
* @var int
*/
protected $mTrxLevel = 0;
-
/**
* Either a short hexidecimal string if a transaction is active or ""
*
* @see DatabaseBase::mTrxLevel
*/
protected $mTrxShortId = '';
-
/**
* The UNIX time that the transaction started. Callers can assume that if
* snapshot isolation is used, then the data is *at least* up to date to that
* @see DatabaseBase::mTrxLevel
*/
private $mTrxTimestamp = null;
-
/** @var float Lag estimate at the time of BEGIN */
- private $mTrxSlaveLag = null;
-
+ private $mTrxReplicaLag = null;
/**
* Remembers the function name given for starting the most recent transaction via begin().
* Used to provide additional context for error reporting.
* @see DatabaseBase::mTrxLevel
*/
private $mTrxFname = null;
-
/**
* Record if possible write queries were done in the last transaction started
*
* @see DatabaseBase::mTrxLevel
*/
private $mTrxDoneWrites = false;
-
/**
* Record if the current transaction was started implicitly due to DBO_TRX being set.
*
* @see DatabaseBase::mTrxLevel
*/
private $mTrxAutomatic = false;
-
/**
* Array of levels of atomicity within transactions
*
* @var array
*/
private $mTrxAtomicLevels = [];
-
/**
* Record if the current transaction was started implicitly by DatabaseBase::startAtomic
*
* @var bool
*/
private $mTrxAutomaticAtomic = false;
-
/**
* Track the write query callers of the current transaction
*
* @var string[]
*/
private $mTrxWriteCallers = [];
-
/**
- * Track the seconds spent in write queries for the current transaction
- *
- * @var float
+ * @var float Seconds spent in write queries for the current transaction
*/
private $mTrxWriteDuration = 0.0;
+ /**
+ * @var integer Number of write queries for the current transaction
+ */
+ private $mTrxWriteQueryCount = 0;
+ /**
+ * @var float Like mTrxWriteQueryCount but excludes lock-bound, easy to replicate, queries
+ */
+ private $mTrxWriteAdjDuration = 0.0;
+ /**
+ * @var integer Number of write queries counted in mTrxWriteAdjDuration
+ */
+ private $mTrxWriteAdjQueryCount = 0;
+ /**
+ * @var float RTT time estimate
+ */
+ private $mRTTEstimate = 0.0;
/** @var array Map of (name => 1) for locks obtained via lock() */
private $mNamedLocksHeld = [];
*/
protected $allViews = null;
+ /** @var float UNIX timestamp */
+ protected $lastPing = 0.0;
+
+ /** @var int[] Prior mFlags values */
+ private $priorFlags = [];
+
+ /** @var Profiler */
+ protected $profiler;
/** @var TransactionProfiler */
protected $trxProfiler;
* @return TransactionProfiler
*/
protected function getTransactionProfiler() {
- if ( !$this->trxProfiler ) {
- $this->trxProfiler = new TransactionProfiler();
- }
-
return $this->trxProfiler;
}
);
}
- public function pendingWriteQueryDuration() {
- return $this->mTrxLevel ? $this->mTrxWriteDuration : false;
+ public function pendingWriteQueryDuration( $type = self::ESTIMATE_TOTAL ) {
+ if ( !$this->mTrxLevel ) {
+ return false;
+ } elseif ( !$this->mTrxDoneWrites ) {
+ return 0.0;
+ }
+
+ switch ( $type ) {
+ case self::ESTIMATE_DB_APPLY:
+ $this->ping( $rtt );
+ $rttAdjTotal = $this->mTrxWriteAdjQueryCount * $rtt;
+ $applyTime = max( $this->mTrxWriteAdjDuration - $rttAdjTotal, 0 );
+ // For omitted queries, make them count as something at least
+ $omitted = $this->mTrxWriteQueryCount - $this->mTrxWriteAdjQueryCount;
+ $applyTime += self::TINY_WRITE_SEC * $omitted;
+
+ return $applyTime;
+ default: // everything
+ return $this->mTrxWriteDuration;
+ }
}
public function pendingWriteCallers() {
return $this->mOpened;
}
- public function setFlag( $flag ) {
+ public function setFlag( $flag, $remember = self::REMEMBER_NOTHING ) {
+ if ( $remember === self::REMEMBER_PRIOR ) {
+ array_push( $this->priorFlags, $this->mFlags );
+ }
$this->mFlags |= $flag;
}
- public function clearFlag( $flag ) {
+ public function clearFlag( $flag, $remember = self::REMEMBER_NOTHING ) {
+ if ( $remember === self::REMEMBER_PRIOR ) {
+ array_push( $this->priorFlags, $this->mFlags );
+ }
$this->mFlags &= ~$flag;
}
+ public function restoreFlags( $state = self::RESTORE_PRIOR ) {
+ if ( !$this->priorFlags ) {
+ return;
+ }
+
+ if ( $state === self::RESTORE_INITIAL ) {
+ $this->mFlags = reset( $this->priorFlags );
+ $this->priorFlags = [];
+ } else {
+ $this->mFlags = array_pop( $this->priorFlags );
+ }
+ }
+
public function getFlag( $flag ) {
return !!( $this->mFlags & $flag );
}
* @param array $params Parameters passed from DatabaseBase::factory()
*/
function __construct( array $params ) {
- global $wgDBprefix, $wgDBmwschema, $wgCommandLineMode;
+ global $wgDBprefix, $wgDBmwschema;
$this->srvCache = ObjectCache::getLocalServerInstance( 'hash' );
$schema = $params['schema'];
$foreign = $params['foreign'];
+ $this->cliMode = isset( $params['cliMode'] )
+ ? $params['cliMode']
+ : ( PHP_SAPI === 'cli' );
+
$this->mFlags = $flags;
if ( $this->mFlags & DBO_DEFAULT ) {
- if ( $wgCommandLineMode ) {
+ if ( $this->cliMode ) {
$this->mFlags &= ~DBO_TRX;
} else {
$this->mFlags |= DBO_TRX;
$this->mForeign = $foreign;
- if ( isset( $params['trxProfiler'] ) ) {
- $this->trxProfiler = $params['trxProfiler']; // override
- }
+ $this->profiler = isset( $params['profiler'] )
+ ? $params['profiler']
+ : Profiler::instance(); // @TODO: remove global state
+ $this->trxProfiler = isset( $params['trxProfiler'] )
+ ? $params['trxProfiler']
+ : new TransactionProfiler();
if ( $user ) {
$this->open( $server, $user, $password, $dbName );
}
+
}
/**
* @return DatabaseBase|null DatabaseBase subclass or null
*/
final public static function factory( $dbType, $p = [] ) {
+ global $wgCommandLineMode;
+
$canonicalDBTypes = [
'mysql' => [ 'mysqli', 'mysql' ],
'postgres' => [],
$p['schema'] = isset( $defaultSchemas[$dbType] ) ? $defaultSchemas[$dbType] : null;
}
$p['foreign'] = isset( $p['foreign'] ) ? $p['foreign'] : false;
+ $p['cliMode'] = $wgCommandLineMode;
return new $class( $p );
} else {
}
public function close() {
- if ( count( $this->mTrxIdleCallbacks ) ) { // sanity
- throw new MWException( "Transaction idle callbacks still pending." );
- }
if ( $this->mConn ) {
if ( $this->trxLevel() ) {
if ( !$this->mTrxAutomatic ) {
" performing implicit commit before closing connection!" );
}
- $this->commit( __METHOD__, 'flush' );
+ $this->commit( __METHOD__, self::FLUSHING_INTERNAL );
}
$closed = $this->closeConnection();
$this->mConn = false;
+ } elseif ( $this->mTrxIdleCallbacks || $this->mTrxEndCallbacks ) { // sanity
+ throw new MWException( "Transaction callbacks still pending." );
} else {
$closed = true;
}
* @return bool
*/
protected function isWriteQuery( $sql ) {
- return !preg_match( '/^(?:SELECT|BEGIN|ROLLBACK|COMMIT|SET|SHOW|EXPLAIN|\(SELECT)\b/i', $sql );
+ return !preg_match(
+ '/^(?:SELECT|BEGIN|ROLLBACK|COMMIT|SET|SHOW|EXPLAIN|\(SELECT)\b/i', $sql );
+ }
+
+ /**
+ * @param $sql
+ * @return string|null
+ */
+ protected function getQueryVerb( $sql ) {
+ return preg_match( '/^\s*([a-z]+)/i', $sql, $m ) ? strtoupper( $m[1] ) : null;
}
/**
* @return bool
*/
protected function isTransactableQuery( $sql ) {
- $verb = substr( $sql, 0, strcspn( $sql, " \t\r\n" ) );
- return !in_array( $verb, [ 'BEGIN', 'COMMIT', 'ROLLBACK', 'SHOW', 'SET' ] );
+ $verb = $this->getQueryVerb( $sql );
+ return !in_array( $verb, [ 'BEGIN', 'COMMIT', 'ROLLBACK', 'SHOW', 'SET' ], true );
}
public function query( $sql, $fname = __METHOD__, $tempIgnore = false ) {
global $wgUser;
+ $priorWritesPending = $this->writesOrCallbacksPending();
$this->mLastQuery = $sql;
- $isWriteQuery = $this->isWriteQuery( $sql );
- if ( $isWriteQuery ) {
+ $isWrite = $this->isWriteQuery( $sql );
+ if ( $isWrite ) {
$reason = $this->getReadOnlyReason();
if ( $reason !== false ) {
throw new DBReadOnlyError( $this, "Database is read-only: $reason" );
// Or, for one-word queries (like "BEGIN" or COMMIT") add it to the end (bug 42598)
$commentedSql = preg_replace( '/\s|$/', " /* $fname $userName */ ", $sql, 1 );
- if ( !$this->mTrxLevel && $this->getFlag( DBO_TRX ) && $this->isTransactableQuery( $sql ) ) {
- $this->begin( __METHOD__ . " ($fname)" );
+ # Start implicit transactions that wrap the request if DBO_TRX is enabled
+ if ( !$this->mTrxLevel && $this->getFlag( DBO_TRX )
+ && $this->isTransactableQuery( $sql )
+ ) {
+ $this->begin( __METHOD__ . " ($fname)", self::TRANSACTION_INTERNAL );
$this->mTrxAutomatic = true;
}
# Keep track of whether the transaction has write queries pending
- if ( $this->mTrxLevel && !$this->mTrxDoneWrites && $isWriteQuery ) {
+ if ( $this->mTrxLevel && !$this->mTrxDoneWrites && $isWrite ) {
$this->mTrxDoneWrites = true;
$this->getTransactionProfiler()->transactionWritingIn(
$this->mServer, $this->mDBname, $this->mTrxShortId );
}
- $isMaster = !is_null( $this->getLBInfo( 'master' ) );
- # generalizeSQL will probably cut down the query to reasonable
- # logging size most of the time. The substr is really just a sanity check.
- if ( $isMaster ) {
- $queryProf = 'query-m: ' . substr( DatabaseBase::generalizeSQL( $sql ), 0, 255 );
- $totalProf = 'DatabaseBase::query-master';
- } else {
- $queryProf = 'query: ' . substr( DatabaseBase::generalizeSQL( $sql ), 0, 255 );
- $totalProf = 'DatabaseBase::query';
- }
- # Include query transaction state
- $queryProf .= $this->mTrxShortId ? " [TRX#{$this->mTrxShortId}]" : "";
-
- $profiler = Profiler::instance();
- if ( !$profiler instanceof ProfilerStub ) {
- $totalProfSection = $profiler->scopedProfileIn( $totalProf );
- $queryProfSection = $profiler->scopedProfileIn( $queryProf );
- }
-
if ( $this->debug() ) {
wfDebugLog( 'queries', sprintf( "%s: %s", $this->mDBname, $commentedSql ) );
}
- $queryId = MWDebug::query( $sql, $fname, $isMaster );
-
# Avoid fatals if close() was called
$this->assertOpen();
- # Do the query and handle errors
- $startTime = microtime( true );
- $ret = $this->doQuery( $commentedSql );
- $queryRuntime = microtime( true ) - $startTime;
- # Log the query time and feed it into the DB trx profiler
- $this->getTransactionProfiler()->recordQueryCompletion(
- $queryProf, $startTime, $isWriteQuery, $this->affectedRows() );
-
- MWDebug::queryTime( $queryId );
+ # Send the query to the server
+ $ret = $this->doProfiledQuery( $sql, $commentedSql, $isWrite, $fname );
# Try reconnecting if the connection was lost
if ( false === $ret && $this->wasErrorReissuable() ) {
- # Transaction is gone; this can mean lost writes or REPEATABLE-READ snapshots
- $hadTrx = $this->mTrxLevel;
- # T127428: for non-write transactions, a disconnect and a COMMIT are similar:
- # neither changed data and in both cases any read snapshots are reset anyway.
- $isNoopCommit = ( !$this->writesOrCallbacksPending() && $sql === 'COMMIT' );
- # Update state tracking to reflect transaction loss
- $this->mTrxLevel = 0;
- $this->mTrxIdleCallbacks = []; // bug 65263
- $this->mTrxPreCommitCallbacks = []; // bug 65263
- wfDebug( "Connection lost, reconnecting...\n" );
- # Stash the last error values since ping() might clear them
+ $recoverable = $this->canRecoverFromDisconnect( $sql, $priorWritesPending );
+ # Stash the last error values before anything might clear them
$lastError = $this->lastError();
$lastErrno = $this->lastErrno();
- if ( $this->ping() ) {
+ # Update state tracking to reflect transaction loss due to disconnection
+ $this->handleTransactionLoss();
+ wfDebug( "Connection lost, reconnecting...\n" );
+ if ( $this->reconnect() ) {
wfDebug( "Reconnected\n" );
- $server = $this->getServer();
- $msg = __METHOD__ . ": lost connection to $server; reconnected";
+ $msg = __METHOD__ . ": lost connection to {$this->getServer()}; reconnected";
wfDebugLog( 'DBPerformance', "$msg:\n" . wfBacktrace( true ) );
- if ( ( $hadTrx && !$isNoopCommit ) || $this->mNamedLocksHeld ) {
- # Leave $ret as false and let an error be reported.
- # Callers may catch the exception and continue to use the DB.
- $this->reportQueryError( $lastError, $lastErrno, $sql, $fname, $tempIgnore );
+ if ( !$recoverable ) {
+ # Callers may catch the exception and continue to use the DB
+ $this->reportQueryError( $lastError, $lastErrno, $sql, $fname );
} else {
- # Should be safe to silently retry (no trx/callbacks/locks)
- $startTime = microtime( true );
- $ret = $this->doQuery( $commentedSql );
- $queryRuntime = microtime( true ) - $startTime;
- # Log the query time and feed it into the DB trx profiler
- $this->getTransactionProfiler()->recordQueryCompletion(
- $queryProf, $startTime, $isWriteQuery, $this->affectedRows() );
+ # Should be safe to silently retry the query
+ $ret = $this->doProfiledQuery( $sql, $commentedSql, $isWrite, $fname );
}
} else {
wfDebug( "Failed\n" );
}
if ( false === $ret ) {
+ # Deadlocks cause the entire transaction to abort, not just the statement.
+ # http://dev.mysql.com/doc/refman/5.7/en/innodb-error-handling.html
+ # https://www.postgresql.org/docs/9.1/static/explicit-locking.html
+ if ( $this->wasDeadlock() ) {
+ if ( $this->explicitTrxActive() || $priorWritesPending ) {
+ $tempIgnore = false; // not recoverable
+ }
+ # Update state tracking to reflect transaction loss
+ $this->handleTransactionLoss();
+ }
+
$this->reportQueryError(
$this->lastError(), $this->lastErrno(), $sql, $fname, $tempIgnore );
}
$res = $this->resultObject( $ret );
- // Destroy profile sections in the opposite order to their creation
- ScopedCallback::consume( $queryProfSection );
- ScopedCallback::consume( $totalProfSection );
+ return $res;
+ }
- if ( $isWriteQuery && $this->mTrxLevel ) {
- $this->mTrxWriteDuration += $queryRuntime;
- $this->mTrxWriteCallers[] = $fname;
+ private function doProfiledQuery( $sql, $commentedSql, $isWrite, $fname ) {
+ $isMaster = !is_null( $this->getLBInfo( 'master' ) );
+ # generalizeSQL() will probably cut down the query to reasonable
+ # logging size most of the time. The substr is really just a sanity check.
+ if ( $isMaster ) {
+ $queryProf = 'query-m: ' . substr( DatabaseBase::generalizeSQL( $sql ), 0, 255 );
+ } else {
+ $queryProf = 'query: ' . substr( DatabaseBase::generalizeSQL( $sql ), 0, 255 );
}
- return $res;
+ # Include query transaction state
+ $queryProf .= $this->mTrxShortId ? " [TRX#{$this->mTrxShortId}]" : "";
+
+ $startTime = microtime( true );
+ $this->profiler->profileIn( $queryProf );
+ $ret = $this->doQuery( $commentedSql );
+ $this->profiler->profileOut( $queryProf );
+ $queryRuntime = max( microtime( true ) - $startTime, 0.0 );
+
+ unset( $queryProfSection ); // profile out (if set)
+
+ if ( $ret !== false ) {
+ $this->lastPing = $startTime;
+ if ( $isWrite && $this->mTrxLevel ) {
+ $this->updateTrxWriteQueryTime( $sql, $queryRuntime );
+ $this->mTrxWriteCallers[] = $fname;
+ }
+ }
+
+ if ( $sql === self::PING_QUERY ) {
+ $this->mRTTEstimate = $queryRuntime;
+ }
+
+ $this->getTransactionProfiler()->recordQueryCompletion(
+ $queryProf, $startTime, $isWrite, $this->affectedRows()
+ );
+ MWDebug::query( $sql, $fname, $isMaster, $queryRuntime );
+
+ return $ret;
+ }
+
+ /**
+ * Update the estimated run-time of a query, not counting large row lock times
+ *
+ * LoadBalancer can be set to rollback transactions that will create huge replication
+ * lag. It bases this estimate off of pendingWriteQueryDuration(). Certain simple
+ * queries, like inserting a row can take a long time due to row locking. This method
+ * uses some simple heuristics to discount those cases.
+ *
+ * @param string $sql A SQL write query
+ * @param float $runtime Total runtime, including RTT
+ */
+ private function updateTrxWriteQueryTime( $sql, $runtime ) {
+ // Whether this is indicative of replica DB runtime (except for RBR or ws_repl)
+ $indicativeOfReplicaRuntime = true;
+ if ( $runtime > self::SLOW_WRITE_SEC ) {
+ $verb = $this->getQueryVerb( $sql );
+ // insert(), upsert(), replace() are fast unless bulky in size or blocked on locks
+ if ( $verb === 'INSERT' ) {
+ $indicativeOfReplicaRuntime = $this->affectedRows() > self::SMALL_WRITE_ROWS;
+ } elseif ( $verb === 'REPLACE' ) {
+ $indicativeOfReplicaRuntime = $this->affectedRows() > self::SMALL_WRITE_ROWS / 2;
+ }
+ }
+
+ $this->mTrxWriteDuration += $runtime;
+ $this->mTrxWriteQueryCount += 1;
+ if ( $indicativeOfReplicaRuntime ) {
+ $this->mTrxWriteAdjDuration += $runtime;
+ $this->mTrxWriteAdjQueryCount += 1;
+ }
+ }
+
+ private function canRecoverFromDisconnect( $sql, $priorWritesPending ) {
+ # Transaction dropped; this can mean lost writes, or REPEATABLE-READ snapshots.
+ # Dropped connections also mean that named locks are automatically released.
+ # Only allow error suppression in autocommit mode or when the lost transaction
+ # didn't matter anyway (aside from DBO_TRX snapshot loss).
+ if ( $this->mNamedLocksHeld ) {
+ return false; // possible critical section violation
+ } elseif ( $sql === 'COMMIT' ) {
+ return !$priorWritesPending; // nothing written anyway? (T127428)
+ } elseif ( $sql === 'ROLLBACK' ) {
+ return true; // transaction lost...which is also what was requested :)
+ } elseif ( $this->explicitTrxActive() ) {
+ return false; // don't drop atomocity
+ } elseif ( $priorWritesPending ) {
+ return false; // prior writes lost from implicit transaction
+ }
+
+ return true;
+ }
+
+ private function handleTransactionLoss() {
+ $this->mTrxLevel = 0;
+ $this->mTrxIdleCallbacks = []; // bug 65263
+ $this->mTrxPreCommitCallbacks = []; // bug 65263
+ try {
+ // Handle callbacks in mTrxEndCallbacks
+ $this->runOnTransactionIdleCallbacks( self::TRIGGER_ROLLBACK );
+ $this->runTransactionListenerCallbacks( self::TRIGGER_ROLLBACK );
+ return null;
+ } catch ( Exception $e ) {
+ // Already logged; move on...
+ return $e;
+ }
}
public function reportQueryError( $error, $errno, $sql, $fname, $tempIgnore = false ) {
return '(' . $this->selectSQLText( $table, $fld, $conds, null, [], $join_conds ) . ')';
}
+ /**
+ * @param string $field Field or column to cast
+ * @return string
+ * @since 1.28
+ */
+ public function buildStringCast( $field ) {
+ return $field;
+ }
+
public function selectDB( $db ) {
# Stub. Shouldn't cause serious problems if it's not overridden, but
# if your database engine supports a concept similar to MySQL's
/**
* Gets an array of aliased table names
*
- * @param array $tables Array( [alias] => table )
+ * @param array $tables [ [alias] => table ]
* @return string[] See tableNameWithAlias()
*/
public function tableNamesWithAlias( $tables ) {
/**
* Gets an array of aliased field names
*
- * @param array $fields Array( [alias] => field )
+ * @param array $fields [ [alias] => field ]
* @return string[] See fieldNameWithAlias()
*/
public function fieldNamesWithAlias( $fields ) {
$useTrx = !$this->mTrxLevel;
if ( $useTrx ) {
- $this->begin( $fname );
+ $this->begin( $fname, self::TRANSACTION_INTERNAL );
}
try {
# Update any existing conflicting row(s)
$ok = $this->insert( $table, $rows, $fname, [ 'IGNORE' ] ) && $ok;
} catch ( Exception $e ) {
if ( $useTrx ) {
- $this->rollback( $fname );
+ $this->rollback( $fname, self::FLUSHING_INTERNAL );
}
throw $e;
}
if ( $useTrx ) {
- $this->commit( $fname );
+ $this->commit( $fname, self::FLUSHING_INTERNAL );
}
return $ok;
return $this->query( $sql, $fname );
}
- public function insertSelect( $destTable, $srcTable, $varMap, $conds,
+ public function insertSelect(
+ $destTable, $srcTable, $varMap, $conds,
+ $fname = __METHOD__, $insertOptions = [], $selectOptions = []
+ ) {
+ if ( $this->cliMode ) {
+ // For massive migrations with downtime, we don't want to select everything
+ // into memory and OOM, so do all this native on the server side if possible.
+ return $this->nativeInsertSelect(
+ $destTable,
+ $srcTable,
+ $varMap,
+ $conds,
+ $fname,
+ $insertOptions,
+ $selectOptions
+ );
+ }
+
+ // For web requests, do a locking SELECT and then INSERT. This puts the SELECT burden
+ // on only the master (without needing row-based-replication). It also makes it easy to
+ // know how big the INSERT is going to be.
+ $fields = [];
+ foreach ( $varMap as $dstColumn => $sourceColumnOrSql ) {
+ $fields[] = $this->fieldNameWithAlias( $sourceColumnOrSql, $dstColumn );
+ }
+ $selectOptions[] = 'FOR UPDATE';
+ $res = $this->select( $srcTable, implode( ',', $fields ), $conds, $fname, $selectOptions );
+ if ( !$res ) {
+ return false;
+ }
+
+ $rows = [];
+ foreach ( $res as $row ) {
+ $rows[] = (array)$row;
+ }
+
+ return $this->insert( $destTable, $rows, $fname, $insertOptions );
+ }
+
+ public function nativeInsertSelect( $destTable, $srcTable, $varMap, $conds,
$fname = __METHOD__,
$insertOptions = [], $selectOptions = []
) {
* queries. If a deadlock occurs during the processing, the transaction
* will be rolled back and the callback function will be called again.
*
+ * Avoid using this method outside of Job or Maintenance classes.
+ *
* Usage:
* $dbw->deadlockLoop( callback, ... );
*
* Extra arguments are passed through to the specified callback function.
+ * This method requires that no transactions are already active to avoid
+ * causing premature commits or exceptions.
*
* Returns whatever the callback function returned on its successful,
* iteration, or false on error, for example if the retry limit was
* reached.
+ *
* @return mixed
* @throws DBUnexpectedError
* @throws Exception
return false;
}
- final public function onTransactionIdle( $callback ) {
+ public function serverIsReadOnly() {
+ return false;
+ }
+
+ final public function onTransactionResolution( callable $callback ) {
+ if ( !$this->mTrxLevel ) {
+ throw new DBUnexpectedError( $this, "No transaction is active." );
+ }
+ $this->mTrxEndCallbacks[] = [ $callback, wfGetCaller() ];
+ }
+
+ final public function onTransactionIdle( callable $callback ) {
$this->mTrxIdleCallbacks[] = [ $callback, wfGetCaller() ];
if ( !$this->mTrxLevel ) {
- $this->runOnTransactionIdleCallbacks();
+ $this->runOnTransactionIdleCallbacks( self::TRIGGER_IDLE );
}
}
- final public function onTransactionPreCommitOrIdle( $callback ) {
+ final public function onTransactionPreCommitOrIdle( callable $callback ) {
if ( $this->mTrxLevel ) {
$this->mTrxPreCommitCallbacks[] = [ $callback, wfGetCaller() ];
} else {
- $this->onTransactionIdle( $callback ); // this will trigger immediately
+ // If no transaction is active, then make one for this callback
+ $this->startAtomic( __METHOD__ );
+ try {
+ call_user_func( $callback );
+ $this->endAtomic( __METHOD__ );
+ } catch ( Exception $e ) {
+ $this->rollback( __METHOD__, self::FLUSHING_INTERNAL );
+ throw $e;
+ }
+ }
+ }
+
+ final public function setTransactionListener( $name, callable $callback = null ) {
+ if ( $callback ) {
+ $this->mTrxRecurringCallbacks[$name] = [ $callback, wfGetCaller() ];
+ } else {
+ unset( $this->mTrxRecurringCallbacks[$name] );
}
}
/**
- * Actually any "on transaction idle" callbacks.
+ * Whether to disable running of post-COMMIT/ROLLBACK callbacks
*
+ * This method should not be used outside of Database/LoadBalancer
+ *
+ * @param bool $suppress
+ * @since 1.28
+ */
+ final public function setTrxEndCallbackSuppression( $suppress ) {
+ $this->mTrxEndCallbacksSuppressed = $suppress;
+ }
+
+ /**
+ * Actually run and consume any "on transaction idle/resolution" callbacks.
+ *
+ * This method should not be used outside of Database/LoadBalancer
+ *
+ * @param integer $trigger IDatabase::TRIGGER_* constant
* @since 1.20
+ * @throws Exception
*/
- protected function runOnTransactionIdleCallbacks() {
- $autoTrx = $this->getFlag( DBO_TRX ); // automatic begin() enabled?
+ public function runOnTransactionIdleCallbacks( $trigger ) {
+ if ( $this->mTrxEndCallbacksSuppressed ) {
+ return;
+ }
- $e = $ePrior = null; // last exception
+ $autoTrx = $this->getFlag( DBO_TRX ); // automatic begin() enabled?
+ /** @var Exception $e */
+ $e = null; // first exception
do { // callbacks may add callbacks :)
- $callbacks = $this->mTrxIdleCallbacks;
- $this->mTrxIdleCallbacks = []; // recursion guard
+ $callbacks = array_merge(
+ $this->mTrxIdleCallbacks,
+ $this->mTrxEndCallbacks // include "transaction resolution" callbacks
+ );
+ $this->mTrxIdleCallbacks = []; // consumed (and recursion guard)
+ $this->mTrxEndCallbacks = []; // consumed (recursion guard)
foreach ( $callbacks as $callback ) {
try {
list( $phpCallback ) = $callback;
$this->clearFlag( DBO_TRX ); // make each query its own transaction
- call_user_func( $phpCallback );
+ call_user_func_array( $phpCallback, [ $trigger ] );
if ( $autoTrx ) {
$this->setFlag( DBO_TRX ); // restore automatic begin()
} else {
$this->clearFlag( DBO_TRX ); // restore auto-commit
}
- } catch ( Exception $e ) {
- if ( $ePrior ) {
- MWExceptionHandler::logException( $ePrior );
- }
- $ePrior = $e;
+ } catch ( Exception $ex ) {
+ MWExceptionHandler::logException( $ex );
+ $e = $e ?: $ex;
// Some callbacks may use startAtomic/endAtomic, so make sure
// their transactions are ended so other callbacks don't fail
if ( $this->trxLevel() ) {
- $this->rollback( __METHOD__ );
+ $this->rollback( __METHOD__, self::FLUSHING_INTERNAL );
}
}
}
} while ( count( $this->mTrxIdleCallbacks ) );
if ( $e instanceof Exception ) {
- throw $e; // re-throw any last exception
+ throw $e; // re-throw any first exception
}
}
/**
- * Actually any "on transaction pre-commit" callbacks.
+ * Actually run and consume any "on transaction pre-commit" callbacks.
+ *
+ * This method should not be used outside of Database/LoadBalancer
*
* @since 1.22
+ * @throws Exception
*/
- protected function runOnTransactionPreCommitCallbacks() {
- $e = $ePrior = null; // last exception
+ public function runOnTransactionPreCommitCallbacks() {
+ $e = null; // first exception
do { // callbacks may add callbacks :)
$callbacks = $this->mTrxPreCommitCallbacks;
- $this->mTrxPreCommitCallbacks = []; // recursion guard
+ $this->mTrxPreCommitCallbacks = []; // consumed (and recursion guard)
foreach ( $callbacks as $callback ) {
try {
list( $phpCallback ) = $callback;
call_user_func( $phpCallback );
- } catch ( Exception $e ) {
- if ( $ePrior ) {
- MWExceptionHandler::logException( $ePrior );
- }
- $ePrior = $e;
+ } catch ( Exception $ex ) {
+ MWExceptionHandler::logException( $ex );
+ $e = $e ?: $ex;
}
}
} while ( count( $this->mTrxPreCommitCallbacks ) );
if ( $e instanceof Exception ) {
- throw $e; // re-throw any last exception
+ throw $e; // re-throw any first exception
+ }
+ }
+
+ /**
+ * Actually run any "transaction listener" callbacks.
+ *
+ * This method should not be used outside of Database/LoadBalancer
+ *
+ * @param integer $trigger IDatabase::TRIGGER_* constant
+ * @throws Exception
+ * @since 1.20
+ */
+ public function runTransactionListenerCallbacks( $trigger ) {
+ if ( $this->mTrxEndCallbacksSuppressed ) {
+ return;
+ }
+
+ /** @var Exception $e */
+ $e = null; // first exception
+
+ foreach ( $this->mTrxRecurringCallbacks as $callback ) {
+ try {
+ list( $phpCallback ) = $callback;
+ $phpCallback( $trigger, $this );
+ } catch ( Exception $ex ) {
+ MWExceptionHandler::logException( $ex );
+ $e = $e ?: $ex;
+ }
+ }
+
+ if ( $e instanceof Exception ) {
+ throw $e; // re-throw any first exception
}
}
final public function startAtomic( $fname = __METHOD__ ) {
if ( !$this->mTrxLevel ) {
- $this->begin( $fname );
- $this->mTrxAutomatic = true;
+ $this->begin( $fname, self::TRANSACTION_INTERNAL );
// If DBO_TRX is set, a series of startAtomic/endAtomic pairs will result
// in all changes being in one transaction to keep requests transactional.
if ( !$this->getFlag( DBO_TRX ) ) {
final public function endAtomic( $fname = __METHOD__ ) {
if ( !$this->mTrxLevel ) {
- throw new DBUnexpectedError( $this, 'No atomic transaction is open.' );
+ throw new DBUnexpectedError( $this, "No atomic transaction is open (got $fname)." );
}
if ( !$this->mTrxAtomicLevels ||
array_pop( $this->mTrxAtomicLevels ) !== $fname
) {
- throw new DBUnexpectedError( $this, 'Invalid atomic section ended.' );
+ throw new DBUnexpectedError( $this, "Invalid atomic section ended (got $fname)." );
}
if ( !$this->mTrxAtomicLevels && $this->mTrxAutomaticAtomic ) {
- $this->commit( $fname, 'flush' );
+ $this->commit( $fname, self::FLUSHING_INTERNAL );
}
}
final public function doAtomicSection( $fname, callable $callback ) {
$this->startAtomic( $fname );
try {
- call_user_func_array( $callback, [ $this, $fname ] );
+ $res = call_user_func_array( $callback, [ $this, $fname ] );
} catch ( Exception $e ) {
- $this->rollback( $fname );
+ $this->rollback( $fname, self::FLUSHING_INTERNAL );
throw $e;
}
$this->endAtomic( $fname );
+
+ return $res;
}
- final public function begin( $fname = __METHOD__ ) {
- if ( $this->mTrxLevel ) { // implicit commit
+ final public function begin( $fname = __METHOD__, $mode = self::TRANSACTION_EXPLICIT ) {
+ // Protect against mismatched atomic section, transaction nesting, and snapshot loss
+ if ( $this->mTrxLevel ) {
if ( $this->mTrxAtomicLevels ) {
- // If the current transaction was an automatic atomic one, then we definitely have
- // a problem. Same if there is any unclosed atomic level.
$levels = implode( ', ', $this->mTrxAtomicLevels );
- throw new DBUnexpectedError(
- $this,
- "Got explicit BEGIN from $fname while atomic section(s) $levels are open."
- );
+ $msg = "$fname: Got explicit BEGIN while atomic section(s) $levels are open.";
+ throw new DBUnexpectedError( $this, $msg );
} elseif ( !$this->mTrxAutomatic ) {
- // We want to warn about inadvertently nested begin/commit pairs, but not about
- // auto-committing implicit transactions that were started by query() via DBO_TRX
- throw new DBUnexpectedError(
- $this,
- "$fname: Transaction already in progress (from {$this->mTrxFname}), " .
- " performing implicit commit!"
- );
+ $msg = "$fname: Explicit transaction already active (from {$this->mTrxFname}).";
+ throw new DBUnexpectedError( $this, $msg );
} else {
- // The transaction was automatic and has done write operations
- if ( $this->mTrxDoneWrites ) {
- wfDebug( "$fname: Automatic transaction with writes in progress" .
- " (from {$this->mTrxFname}), performing implicit commit!\n"
- );
- }
- }
-
- $this->runOnTransactionPreCommitCallbacks();
- $writeTime = $this->pendingWriteQueryDuration();
- $this->doCommit( $fname );
- if ( $this->mTrxDoneWrites ) {
- $this->mDoneWrites = microtime( true );
- $this->getTransactionProfiler()->transactionWritingOut(
- $this->mServer, $this->mDBname, $this->mTrxShortId, $writeTime );
+ // @TODO: make this an exception at some point
+ $msg = "$fname: Implicit transaction already active (from {$this->mTrxFname}).";
+ wfLogDBError( $msg );
+ wfWarn( $msg );
+ return; // join the main transaction set
}
- $this->runOnTransactionIdleCallbacks();
+ } elseif ( $this->getFlag( DBO_TRX ) && $mode !== self::TRANSACTION_INTERNAL ) {
+ // @TODO: make this an exception at some point
+ $msg = "$fname: Implicit transaction expected (DBO_TRX set).";
+ wfLogDBError( $msg );
+ wfWarn( $msg );
+ return; // let any writes be in the main transaction
}
- # Avoid fatals if close() was called
+ // Avoid fatals if close() was called
$this->assertOpen();
$this->doBegin( $fname );
$this->mTrxTimestamp = microtime( true );
$this->mTrxFname = $fname;
$this->mTrxDoneWrites = false;
- $this->mTrxAutomatic = false;
+ $this->mTrxAutomatic = ( $mode === self::TRANSACTION_INTERNAL );
$this->mTrxAutomaticAtomic = false;
$this->mTrxAtomicLevels = [];
- $this->mTrxIdleCallbacks = [];
- $this->mTrxPreCommitCallbacks = [];
$this->mTrxShortId = wfRandomString( 12 );
$this->mTrxWriteDuration = 0.0;
+ $this->mTrxWriteQueryCount = 0;
+ $this->mTrxWriteAdjDuration = 0.0;
+ $this->mTrxWriteAdjQueryCount = 0;
$this->mTrxWriteCallers = [];
// First SELECT after BEGIN will establish the snapshot in REPEATABLE-READ.
- // Get an estimate of the slave lag before then, treating estimate staleness
+ // Get an estimate of the replica DB lag before then, treating estimate staleness
// as lag itself just to be safe
$status = $this->getApproximateLagStatus();
- $this->mTrxSlaveLag = $status['lag'] + ( microtime( true ) - $status['since'] );
+ $this->mTrxReplicaLag = $status['lag'] + ( microtime( true ) - $status['since'] );
}
/**
$levels = implode( ', ', $this->mTrxAtomicLevels );
throw new DBUnexpectedError(
$this,
- "Got COMMIT while atomic sections $levels are still open"
+ "$fname: Got COMMIT while atomic sections $levels are still open."
);
}
- if ( $flush === 'flush' ) {
+ if ( $flush === self::FLUSHING_INTERNAL || $flush === self::FLUSHING_ALL_PEERS ) {
if ( !$this->mTrxLevel ) {
return; // nothing to do
} elseif ( !$this->mTrxAutomatic ) {
throw new DBUnexpectedError(
$this,
- "$fname: Flushing an explicit transaction, getting out of sync!"
+ "$fname: Flushing an explicit transaction, getting out of sync."
);
}
} else {
if ( !$this->mTrxLevel ) {
- wfWarn( "$fname: No transaction to commit, something got out of sync!" );
+ wfWarn( "$fname: No transaction to commit, something got out of sync." );
return; // nothing to do
} elseif ( $this->mTrxAutomatic ) {
- wfWarn( "$fname: Explicit commit of implicit transaction. Something may be out of sync!" );
+ // @TODO: make this an exception at some point
+ $msg = "$fname: Explicit commit of implicit transaction.";
+ wfLogDBError( $msg );
+ wfWarn( $msg );
+ return; // wait for the main transaction set commit round
}
}
- # Avoid fatals if close() was called
+ // Avoid fatals if close() was called
$this->assertOpen();
$this->runOnTransactionPreCommitCallbacks();
- $writeTime = $this->pendingWriteQueryDuration();
+ $writeTime = $this->pendingWriteQueryDuration( self::ESTIMATE_DB_APPLY );
$this->doCommit( $fname );
if ( $this->mTrxDoneWrites ) {
$this->mDoneWrites = microtime( true );
$this->getTransactionProfiler()->transactionWritingOut(
$this->mServer, $this->mDBname, $this->mTrxShortId, $writeTime );
}
- $this->runOnTransactionIdleCallbacks();
+
+ $this->runOnTransactionIdleCallbacks( self::TRIGGER_COMMIT );
+ $this->runTransactionListenerCallbacks( self::TRIGGER_COMMIT );
}
/**
}
final public function rollback( $fname = __METHOD__, $flush = '' ) {
- if ( $flush !== 'flush' ) {
+ if ( $flush === self::FLUSHING_INTERNAL || $flush === self::FLUSHING_ALL_PEERS ) {
if ( !$this->mTrxLevel ) {
- wfWarn( "$fname: No transaction to rollback, something got out of sync!" );
return; // nothing to do
}
} else {
if ( !$this->mTrxLevel ) {
+ wfWarn( "$fname: No transaction to rollback, something got out of sync." );
return; // nothing to do
+ } elseif ( $this->getFlag( DBO_TRX ) ) {
+ throw new DBUnexpectedError(
+ $this,
+ "$fname: Expected mass rollback of all peer databases (DBO_TRX set)."
+ );
}
}
- # Avoid fatals if close() was called
+ // Avoid fatals if close() was called
$this->assertOpen();
$this->doRollback( $fname );
- $this->mTrxIdleCallbacks = []; // cancel
- $this->mTrxPreCommitCallbacks = []; // cancel
$this->mTrxAtomicLevels = [];
if ( $this->mTrxDoneWrites ) {
$this->getTransactionProfiler()->transactionWritingOut(
$this->mServer, $this->mDBname, $this->mTrxShortId );
}
+
+ $this->mTrxIdleCallbacks = []; // clear
+ $this->mTrxPreCommitCallbacks = []; // clear
+ $this->runOnTransactionIdleCallbacks( self::TRIGGER_ROLLBACK );
+ $this->runTransactionListenerCallbacks( self::TRIGGER_ROLLBACK );
}
/**
*/
protected function doRollback( $fname ) {
if ( $this->mTrxLevel ) {
- $this->query( 'ROLLBACK', $fname, true );
+ # Disconnects cause rollback anyway, so ignore those errors
+ $ignoreErrors = true;
+ $this->query( 'ROLLBACK', $fname, $ignoreErrors );
$this->mTrxLevel = 0;
}
}
+ public function flushSnapshot( $fname = __METHOD__ ) {
+ if ( $this->writesOrCallbacksPending() || $this->explicitTrxActive() ) {
+ // This only flushes transactions to clear snapshots, not to write data
+ throw new DBUnexpectedError(
+ $this,
+ "$fname: Cannot COMMIT to clear snapshot because writes are pending."
+ );
+ }
+
+ $this->commit( $fname, self::FLUSHING_INTERNAL );
+ }
+
+ public function explicitTrxActive() {
+ return $this->mTrxLevel && ( $this->mTrxAtomicLevels || !$this->mTrxAutomatic );
+ }
+
/**
* Creates a new table with structure copied from existing table
* Note that unlike most database abstraction functions, this function does not
}
}
- public function ping() {
- # Stub. Not essential to override.
- return true;
+ public function ping( &$rtt = null ) {
+ // Avoid hitting the server if it was hit recently
+ if ( $this->isOpen() && ( microtime( true ) - $this->lastPing ) < self::PING_TTL ) {
+ if ( !func_num_args() || $this->mRTTEstimate > 0 ) {
+ $rtt = $this->mRTTEstimate;
+ return true; // don't care about $rtt
+ }
+ }
+
+ // This will reconnect if possible or return false if not
+ $this->clearFlag( DBO_TRX, self::REMEMBER_PRIOR );
+ $ok = ( $this->query( self::PING_QUERY, __METHOD__, true ) !== false );
+ $this->restoreFlags( self::RESTORE_PRIOR );
+
+ if ( $ok ) {
+ $rtt = $this->mRTTEstimate;
+ }
+
+ return $ok;
+ }
+
+ /**
+ * @return bool
+ */
+ protected function reconnect() {
+ $this->closeConnection();
+ $this->mOpened = false;
+ $this->mConn = false;
+ try {
+ $this->open( $this->mServer, $this->mUser, $this->mPassword, $this->mDBname );
+ $this->lastPing = microtime( true );
+ $ok = true;
+ } catch ( DBConnectionError $e ) {
+ $ok = false;
+ }
+
+ return $ok;
}
public function getSessionLagStatus() {
}
/**
- * Get the slave lag when the current transaction started
+ * Get the replica DB lag when the current transaction started
*
* This is useful when transactions might use snapshot isolation
* (e.g. REPEATABLE-READ in innodb), so the "real" lag of that data
*/
public function getTransactionLagStatus() {
return $this->mTrxLevel
- ? [ 'lag' => $this->mTrxSlaveLag, 'since' => $this->trxTimestamp() ]
+ ? [ 'lag' => $this->mTrxReplicaLag, 'since' => $this->trxTimestamp() ]
: null;
}
/**
- * Get a slave lag estimate for this server
+ * Get a replica DB lag estimate for this server
*
* @return array ('lag': seconds or false on error, 'since': UNIX timestamp of estimate)
* @since 1.27
*/
public function getApproximateLagStatus() {
return [
- 'lag' => $this->getLBInfo( 'slave' ) ? $this->getLag() : 0,
+ 'lag' => $this->getLBInfo( 'replica' ) ? $this->getLag() : 0,
'since' => microtime( true )
];
}
}
public function getScopedLockAndFlush( $lockKey, $fname, $timeout ) {
+ if ( $this->writesOrCallbacksPending() ) {
+ // This only flushes transactions to clear snapshots, not to write data
+ throw new DBUnexpectedError(
+ $this,
+ "$fname: Cannot COMMIT to clear snapshot because writes are pending."
+ );
+ }
+
if ( !$this->lock( $lockKey, $fname, $timeout ) ) {
return null;
}
$unlocker = new ScopedCallback( function () use ( $lockKey, $fname ) {
- $this->commit( __METHOD__, 'flush' );
- $this->unlock( $lockKey, $fname );
+ if ( $this->trxLevel() ) {
+ // There is a good chance an exception was thrown, causing any early return
+ // from the caller. Let any error handler get a chance to issue rollback().
+ // If there isn't one, let the error bubble up and trigger server-side rollback.
+ $this->onTransactionResolution( function () use ( $lockKey, $fname ) {
+ $this->unlock( $lockKey, $fname );
+ } );
+ } else {
+ $this->unlock( $lockKey, $fname );
+ }
} );
- $this->commit( __METHOD__, 'flush' );
+ $this->commit( __METHOD__, self::FLUSHING_INTERNAL );
return $unlocker;
}
if ( $this->mTrxLevel && $this->mTrxDoneWrites ) {
trigger_error( "Uncommitted DB writes (transaction from {$this->mTrxFname})." );
}
- if ( count( $this->mTrxIdleCallbacks ) || count( $this->mTrxPreCommitCallbacks ) ) {
+ $danglingCallbacks = array_merge(
+ $this->mTrxIdleCallbacks,
+ $this->mTrxPreCommitCallbacks,
+ $this->mTrxEndCallbacks
+ );
+ if ( $danglingCallbacks ) {
$callers = [];
- foreach ( $this->mTrxIdleCallbacks as $callbackInfo ) {
+ foreach ( $danglingCallbacks as $callbackInfo ) {
$callers[] = $callbackInfo[1];
}
$callers = implode( ', ', $callers );
dépôts / lhc / web / wiklou.git / blobdiff