includes/utils/UIDGenerator.php

   1 <?php
   2 /**
   3  * This file deals with UID generation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @author Aaron Schulz
  22  */
  23 use Wikimedia\Assert\Assert;
  24
  25 /**
  26  * Class for getting statistically unique IDs
  27  *
  28  * @since 1.21
  29  */
  30 class UIDGenerator {
  31         /** @var UIDGenerator */
  32         protected static $instance = null;
  33
  34         protected $nodeIdFile; // string; local file path
  35         protected $nodeId32; // string; node ID in binary (32 bits)
  36         protected $nodeId48; // string; node ID in binary (48 bits)
  37
  38         protected $lockFile88; // string; local file path
  39         protected $lockFile128; // string; local file path
  40
  41         /** @var array */
  42         protected $fileHandles = array(); // cache file handles
  43
  44         const QUICK_RAND = 1; // get randomness from fast and insecure sources
  45         const QUICK_VOLATILE = 2; // use an APC like in-memory counter if available
  46
  47         protected function __construct() {
  48                 $this->nodeIdFile = wfTempDir() . '/mw-' . __CLASS__ . '-UID-nodeid';
  49                 $nodeId = '';
  50                 if ( is_file( $this->nodeIdFile ) ) {
  51                         $nodeId = file_get_contents( $this->nodeIdFile );
  52                 }
  53                 // Try to get some ID that uniquely identifies this machine (RFC 4122)...
  54                 if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) {
  55                         MediaWiki\suppressWarnings();
  56                         if ( wfIsWindows() ) {
  57                                 // http://technet.microsoft.com/en-us/library/bb490913.aspx
  58                                 $csv = trim( wfShellExec( 'getmac /NH /FO CSV' ) );
  59                                 $line = substr( $csv, 0, strcspn( $csv, "\n" ) );
  60                                 $info = str_getcsv( $line );
  61                                 $nodeId = isset( $info[0] ) ? str_replace( '-', '', $info[0] ) : '';
  62                         } elseif ( is_executable( '/sbin/ifconfig' ) ) { // Linux/BSD/Solaris/OS X
  63                                 // See http://linux.die.net/man/8/ifconfig
  64                                 $m = array();
  65                                 preg_match( '/\s([0-9a-f]{2}(:[0-9a-f]{2}){5})\s/',
  66                                         wfShellExec( '/sbin/ifconfig -a' ), $m );
  67                                 $nodeId = isset( $m[1] ) ? str_replace( ':', '', $m[1] ) : '';
  68                         }
  69                         MediaWiki\restoreWarnings();
  70                         if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) {
  71                                 $nodeId = MWCryptRand::generateHex( 12, true );
  72                                 $nodeId[1] = dechex( hexdec( $nodeId[1] ) | 0x1 ); // set multicast bit
  73                         }
  74                         file_put_contents( $this->nodeIdFile, $nodeId ); // cache
  75                 }
  76                 $this->nodeId32 = wfBaseConvert( substr( sha1( $nodeId ), 0, 8 ), 16, 2, 32 );
  77                 $this->nodeId48 = wfBaseConvert( $nodeId, 16, 2, 48 );
  78                 // If different processes run as different users, they may have different temp dirs.
  79                 // This is dealt with by initializing the clock sequence number and counters randomly.
  80                 $this->lockFile88 = wfTempDir() . '/mw-' . __CLASS__ . '-UID-88';
  81                 $this->lockFile128 = wfTempDir() . '/mw-' . __CLASS__ . '-UID-128';
  82         }
  83
  84         /**
  85          * @return UIDGenerator
  86          */
  87         protected static function singleton() {
  88                 if ( self::$instance === null ) {
  89                         self::$instance = new self();
  90                 }
  91
  92                 return self::$instance;
  93         }
  94
  95         /**
  96          * Get a statistically unique 88-bit unsigned integer ID string.
  97          * The bits of the UID are prefixed with the time (down to the millisecond).
  98          *
  99          * These IDs are suitable as values for the shard key of distributed data.
 100          * If a column uses these as values, it should be declared UNIQUE to handle collisions.
 101          * New rows almost always have higher UIDs, which makes B-TREE updates on INSERT fast.
 102          * They can also be stored "DECIMAL(27) UNSIGNED" or BINARY(11) in MySQL.
 103          *
 104          * UID generation is serialized on each server (as the node ID is for the whole machine).
 105          *
 106          * @param int $base Specifies a base other than 10
 107          * @return string Number
 108          * @throws MWException
 109          */
 110         public static function newTimestampedUID88( $base = 10 ) {
 111                 Assert::parameterType( 'integer', $base, '$base' );
 112                 Assert::parameter( $base <= 36, '$base', 'must be <= 36' );
 113                 Assert::parameter( $base >= 2, '$base', 'must be >= 2' );
 114
 115                 $gen = self::singleton();
 116                 $time = $gen->getTimestampAndDelay( 'lockFile88', 1, 1024 );
 117
 118                 return wfBaseConvert( $gen->getTimestampedID88( $time ), 2, $base );
 119         }
 120
 121         /**
 122          * @param array $info (UIDGenerator::millitime(), counter, clock sequence)
 123          * @return string 88 bits
 124          * @throws MWException
 125          */
 126         protected function getTimestampedID88( array $info ) {
 127                 list( $time, $counter ) = $info;
 128                 // Take the 46 MSBs of "milliseconds since epoch"
 129                 $id_bin = $this->millisecondsSinceEpochBinary( $time );
 130                 // Add a 10 bit counter resulting in 56 bits total
 131                 $id_bin .= str_pad( decbin( $counter ), 10, '0', STR_PAD_LEFT );
 132                 // Add the 32 bit node ID resulting in 88 bits total
 133                 $id_bin .= $this->nodeId32;
 134                 // Convert to a 1-27 digit integer string
 135                 if ( strlen( $id_bin ) !== 88 ) {
 136                         throw new MWException( "Detected overflow for millisecond timestamp." );
 137                 }
 138
 139                 return $id_bin;
 140         }
 141
 142         /**
 143          * Get a statistically unique 128-bit unsigned integer ID string.
 144          * The bits of the UID are prefixed with the time (down to the millisecond).
 145          *
 146          * These IDs are suitable as globally unique IDs, without any enforced uniqueness.
 147          * New rows almost always have higher UIDs, which makes B-TREE updates on INSERT fast.
 148          * They can also be stored as "DECIMAL(39) UNSIGNED" or BINARY(16) in MySQL.
 149          *
 150          * UID generation is serialized on each server (as the node ID is for the whole machine).
 151          *
 152          * @param int $base Specifies a base other than 10
 153          * @return string Number
 154          * @throws MWException
 155          */
 156         public static function newTimestampedUID128( $base = 10 ) {
 157                 Assert::parameterType( 'integer', $base, '$base' );
 158                 Assert::parameter( $base <= 36, '$base', 'must be <= 36' );
 159                 Assert::parameter( $base >= 2, '$base', 'must be >= 2' );
 160
 161                 $gen = self::singleton();
 162                 $time = $gen->getTimestampAndDelay( 'lockFile128', 16384, 1048576 );
 163
 164                 return wfBaseConvert( $gen->getTimestampedID128( $time ), 2, $base );
 165         }
 166
 167         /**
 168          * @param array $info (UIDGenerator::millitime(), counter, clock sequence)
 169          * @return string 128 bits
 170          * @throws MWException
 171          */
 172         protected function getTimestampedID128( array $info ) {
 173                 list( $time, $counter, $clkSeq ) = $info;
 174                 // Take the 46 MSBs of "milliseconds since epoch"
 175                 $id_bin = $this->millisecondsSinceEpochBinary( $time );
 176                 // Add a 20 bit counter resulting in 66 bits total
 177                 $id_bin .= str_pad( decbin( $counter ), 20, '0', STR_PAD_LEFT );
 178                 // Add a 14 bit clock sequence number resulting in 80 bits total
 179                 $id_bin .= str_pad( decbin( $clkSeq ), 14, '0', STR_PAD_LEFT );
 180                 // Add the 48 bit node ID resulting in 128 bits total
 181                 $id_bin .= $this->nodeId48;
 182                 // Convert to a 1-39 digit integer string
 183                 if ( strlen( $id_bin ) !== 128 ) {
 184                         throw new MWException( "Detected overflow for millisecond timestamp." );
 185                 }
 186
 187                 return $id_bin;
 188         }
 189
 190         /**
 191          * Return an RFC4122 compliant v4 UUID
 192          *
 193          * @param int $flags Bitfield (supports UIDGenerator::QUICK_RAND)
 194          * @return string
 195          * @throws MWException
 196          */
 197         public static function newUUIDv4( $flags = 0 ) {
 198                 $hex = ( $flags & self::QUICK_RAND )
 199                         ? wfRandomString( 31 )
 200                         : MWCryptRand::generateHex( 31 );
 201
 202                 return sprintf( '%s-%s-%s-%s-%s',
 203                         // "time_low" (32 bits)
 204                         substr( $hex, 0, 8 ),
 205                         // "time_mid" (16 bits)
 206                         substr( $hex, 8, 4 ),
 207                         // "time_hi_and_version" (16 bits)
 208                         '4' . substr( $hex, 12, 3 ),
 209                         // "clk_seq_hi_res (8 bits, variant is binary 10x) and "clk_seq_low" (8 bits)
 210                         dechex( 0x8 | ( hexdec( $hex[15] ) & 0x3 ) ) . $hex[16] . substr( $hex, 17, 2 ),
 211                         // "node" (48 bits)
 212                         substr( $hex, 19, 12 )
 213                 );
 214         }
 215
 216         /**
 217          * Return an RFC4122 compliant v4 UUID
 218          *
 219          * @param int $flags Bitfield (supports UIDGenerator::QUICK_RAND)
 220          * @return string 32 hex characters with no hyphens
 221          * @throws MWException
 222          */
 223         public static function newRawUUIDv4( $flags = 0 ) {
 224                 return str_replace( '-', '', self::newUUIDv4( $flags ) );
 225         }
 226
 227         /**
 228          * Return an ID that is sequential *only* for this node and bucket
 229          *
 230          * These IDs are suitable for per-host sequence numbers, e.g. for some packet protocols.
 231          * If UIDGenerator::QUICK_VOLATILE is used the counter might reset on server restart.
 232          *
 233          * @param string $bucket Arbitrary bucket name (should be ASCII)
 234          * @param int $bits Bit size (<=48) of resulting numbers before wrap-around
 235          * @param int $flags (supports UIDGenerator::QUICK_VOLATILE)
 236          * @return float Integer value as float
 237          * @since 1.23
 238          */
 239         public static function newSequentialPerNodeID( $bucket, $bits = 48, $flags = 0 ) {
 240                 return current( self::newSequentialPerNodeIDs( $bucket, $bits, 1, $flags ) );
 241         }
 242
 243         /**
 244          * Return IDs that are sequential *only* for this node and bucket
 245          *
 246          * @see UIDGenerator::newSequentialPerNodeID()
 247          * @param string $bucket Arbitrary bucket name (should be ASCII)
 248          * @param int $bits Bit size (16 to 48) of resulting numbers before wrap-around
 249          * @param int $count Number of IDs to return (1 to 10000)
 250          * @param int $flags (supports UIDGenerator::QUICK_VOLATILE)
 251          * @return array Ordered list of float integer values
 252          * @since 1.23
 253          */
 254         public static function newSequentialPerNodeIDs( $bucket, $bits, $count, $flags = 0 ) {
 255                 $gen = self::singleton();
 256                 return $gen->getSequentialPerNodeIDs( $bucket, $bits, $count, $flags );
 257         }
 258
 259         /**
 260          * Return IDs that are sequential *only* for this node and bucket
 261          *
 262          * @see UIDGenerator::newSequentialPerNodeID()
 263          * @param string $bucket Arbitrary bucket name (should be ASCII)
 264          * @param int $bits Bit size (16 to 48) of resulting numbers before wrap-around
 265          * @param int $count Number of IDs to return (1 to 10000)
 266          * @param int $flags (supports UIDGenerator::QUICK_VOLATILE)
 267          * @return array Ordered list of float integer values
 268          * @throws MWException
 269          */
 270         protected function getSequentialPerNodeIDs( $bucket, $bits, $count, $flags ) {
 271                 if ( $count <= 0 ) {
 272                         return array(); // nothing to do
 273                 } elseif ( $count > 10000 ) {
 274                         throw new MWException( "Number of requested IDs ($count) is too high." );
 275                 } elseif ( $bits < 16 || $bits > 48 ) {
 276                         throw new MWException( "Requested bit size ($bits) is out of range." );
 277                 }
 278
 279                 $counter = null; // post-increment persistent counter value
 280
 281                 // Use APC/eAccelerator/xcache if requested, available, and not in CLI mode;
 282                 // Counter values would not survive accross script instances in CLI mode.
 283                 $cache = null;
 284                 if ( ( $flags & self::QUICK_VOLATILE ) && PHP_SAPI !== 'cli' ) {
 285                         try {
 286                                 $cache = ObjectCache::newAccelerator();
 287                         } catch ( Exception $e ) {
 288                                 // not supported
 289                         }
 290                 }
 291                 if ( $cache ) {
 292                         $counter = $cache->incr( $bucket, $count );
 293                         if ( $counter === false ) {
 294                                 if ( !$cache->add( $bucket, (int)$count ) ) {
 295                                         throw new MWException( 'Unable to set value to ' . get_class( $cache ) );
 296                                 }
 297                                 $counter = $count;
 298                         }
 299                 }
 300
 301                 // Note: use of fmod() avoids "division by zero" on 32 bit machines
 302                 if ( $counter === null ) {
 303                         $path = wfTempDir() . '/mw-' . __CLASS__ . '-' . rawurlencode( $bucket ) . '-48';
 304                         // Get the UID lock file handle
 305                         if ( isset( $this->fileHandles[$path] ) ) {
 306                                 $handle = $this->fileHandles[$path];
 307                         } else {
 308                                 $handle = fopen( $path, 'cb+' );
 309                                 $this->fileHandles[$path] = $handle ?: null; // cache
 310                         }
 311                         // Acquire the UID lock file
 312                         if ( $handle === false ) {
 313                                 throw new MWException( "Could not open '{$path}'." );
 314                         } elseif ( !flock( $handle, LOCK_EX ) ) {
 315                                 fclose( $handle );
 316                                 throw new MWException( "Could not acquire '{$path}'." );
 317                         }
 318                         // Fetch the counter value and increment it...
 319                         rewind( $handle );
 320                         $counter = floor( trim( fgets( $handle ) ) ) + $count; // fetch as float
 321                         // Write back the new counter value
 322                         ftruncate( $handle, 0 );
 323                         rewind( $handle );
 324                         fwrite( $handle, fmod( $counter, pow( 2, 48 ) ) ); // warp-around as needed
 325                         fflush( $handle );
 326                         // Release the UID lock file
 327                         flock( $handle, LOCK_UN );
 328                 }
 329
 330                 $ids = array();
 331                 $divisor = pow( 2, $bits );
 332                 $currentId = floor( $counter - $count ); // pre-increment counter value
 333                 for ( $i = 0; $i < $count; ++$i ) {
 334                         $ids[] = fmod( ++$currentId, $divisor );
 335                 }
 336
 337                 return $ids;
 338         }
 339
 340         /**
 341          * Get a (time,counter,clock sequence) where (time,counter) is higher
 342          * than any previous (time,counter) value for the given clock sequence.
 343          * This is useful for making UIDs sequential on a per-node bases.
 344          *
 345          * @param string $lockFile Name of a local lock file
 346          * @param int $clockSeqSize The number of possible clock sequence values
 347          * @param int $counterSize The number of possible counter values
 348          * @return array (result of UIDGenerator::millitime(), counter, clock sequence)
 349          * @throws MWException
 350          */
 351         protected function getTimestampAndDelay( $lockFile, $clockSeqSize, $counterSize ) {
 352                 // Get the UID lock file handle
 353                 $path = $this->$lockFile;
 354                 if ( isset( $this->fileHandles[$path] ) ) {
 355                         $handle = $this->fileHandles[$path];
 356                 } else {
 357                         $handle = fopen( $path, 'cb+' );
 358                         $this->fileHandles[$path] = $handle ?: null; // cache
 359                 }
 360                 // Acquire the UID lock file
 361                 if ( $handle === false ) {
 362                         throw new MWException( "Could not open '{$this->$lockFile}'." );
 363                 } elseif ( !flock( $handle, LOCK_EX ) ) {
 364                         fclose( $handle );
 365                         throw new MWException( "Could not acquire '{$this->$lockFile}'." );
 366                 }
 367                 // Get the current timestamp, clock sequence number, last time, and counter
 368                 rewind( $handle );
 369                 $data = explode( ' ', fgets( $handle ) ); // "<clk seq> <sec> <msec> <counter> <offset>"
 370                 $clockChanged = false; // clock set back significantly?
 371                 if ( count( $data ) == 5 ) { // last UID info already initialized
 372                         $clkSeq = (int)$data[0] % $clockSeqSize;
 373                         $prevTime = array( (int)$data[1], (int)$data[2] );
 374                         $offset = (int)$data[4] % $counterSize; // random counter offset
 375                         $counter = 0; // counter for UIDs with the same timestamp
 376                         // Delay until the clock reaches the time of the last ID.
 377                         // This detects any microtime() drift among processes.
 378                         $time = $this->timeWaitUntil( $prevTime );
 379                         if ( !$time ) { // too long to delay?
 380                                 $clockChanged = true; // bump clock sequence number
 381                                 $time = self::millitime();
 382                         } elseif ( $time == $prevTime ) {
 383                                 // Bump the counter if there are timestamp collisions
 384                                 $counter = (int)$data[3] % $counterSize;
 385                                 if ( ++$counter >= $counterSize ) { // sanity (starts at 0)
 386                                         flock( $handle, LOCK_UN ); // abort
 387                                         throw new MWException( "Counter overflow for timestamp value." );
 388                                 }
 389                         }
 390                 } else { // last UID info not initialized
 391                         $clkSeq = mt_rand( 0, $clockSeqSize - 1 );
 392                         $counter = 0;
 393                         $offset = mt_rand( 0, $counterSize - 1 );
 394                         $time = self::millitime();
 395                 }
 396                 // microtime() and gettimeofday() can drift from time() at least on Windows.
 397                 // The drift is immediate for processes running while the system clock changes.
 398                 // time() does not have this problem. See https://bugs.php.net/bug.php?id=42659.
 399                 if ( abs( time() - $time[0] ) >= 2 ) {
 400                         // We don't want processes using too high or low timestamps to avoid duplicate
 401                         // UIDs and clock sequence number churn. This process should just be restarted.
 402                         flock( $handle, LOCK_UN ); // abort
 403                         throw new MWException( "Process clock is outdated or drifted." );
 404                 }
 405                 // If microtime() is synced and a clock change was detected, then the clock went back
 406                 if ( $clockChanged ) {
 407                         // Bump the clock sequence number and also randomize the counter offset,
 408                         // which is useful for UIDs that do not include the clock sequence number.
 409                         $clkSeq = ( $clkSeq + 1 ) % $clockSeqSize;
 410                         $offset = mt_rand( 0, $counterSize - 1 );
 411                         trigger_error( "Clock was set back; sequence number incremented." );
 412                 }
 413                 // Update the (clock sequence number, timestamp, counter)
 414                 ftruncate( $handle, 0 );
 415                 rewind( $handle );
 416                 fwrite( $handle, "{$clkSeq} {$time[0]} {$time[1]} {$counter} {$offset}" );
 417                 fflush( $handle );
 418                 // Release the UID lock file
 419                 flock( $handle, LOCK_UN );
 420
 421                 return array( $time, ( $counter + $offset ) % $counterSize, $clkSeq );
 422         }
 423
 424         /**
 425          * Wait till the current timestamp reaches $time and return the current
 426          * timestamp. This returns false if it would have to wait more than 10ms.
 427          *
 428          * @param array $time Result of UIDGenerator::millitime()
 429          * @return array|bool UIDGenerator::millitime() result or false
 430          */
 431         protected function timeWaitUntil( array $time ) {
 432                 do {
 433                         $ct = self::millitime();
 434                         if ( $ct >= $time ) { // http://php.net/manual/en/language.operators.comparison.php
 435                                 return $ct; // current timestamp is higher than $time
 436                         }
 437                 } while ( ( ( $time[0] - $ct[0] ) * 1000 + ( $time[1] - $ct[1] ) ) <= 10 );
 438
 439                 return false;
 440         }
 441
 442         /**
 443          * @param array $time Result of UIDGenerator::millitime()
 444          * @return string 46 MSBs of "milliseconds since epoch" in binary (rolls over in 4201)
 445          * @throws MWException
 446          */
 447         protected function millisecondsSinceEpochBinary( array $time ) {
 448                 list( $sec, $msec ) = $time;
 449                 $ts = 1000 * $sec + $msec;
 450                 if ( $ts > pow( 2, 52 ) ) {
 451                         throw new MWException( __METHOD__ .
 452                                 ': sorry, this function doesn\'t work after the year 144680' );
 453                 }
 454
 455                 return substr( wfBaseConvert( $ts, 10, 2, 46 ), -46 );
 456         }
 457
 458         /**
 459          * @return array (current time in seconds, milliseconds since then)
 460          */
 461         protected static function millitime() {
 462                 list( $msec, $sec ) = explode( ' ', microtime() );
 463
 464                 return array( (int)$sec, (int)( $msec * 1000 ) );
 465         }
 466
 467         /**
 468          * Delete all cache files that have been created.
 469          *
 470          * This is a cleanup method primarily meant to be used from unit tests to
 471          * avoid poluting the local filesystem. If used outside of a unit test
 472          * environment it should be used with caution as it may destroy state saved
 473          * in the files.
 474          *
 475          * @see unitTestTearDown
 476          * @since 1.23
 477          */
 478         protected function deleteCacheFiles() {
 479                 // Bug: 44850
 480                 foreach ( $this->fileHandles as $path => $handle ) {
 481                         if ( $handle !== null ) {
 482                                 fclose( $handle );
 483                         }
 484                         if ( is_file( $path ) ) {
 485                                 unlink( $path );
 486                         }
 487                         unset( $this->fileHandles[$path] );
 488                 }
 489                 if ( is_file( $this->nodeIdFile ) ) {
 490                         unlink( $this->nodeIdFile );
 491                 }
 492         }
 493
 494         /**
 495          * Cleanup resources when tearing down after a unit test.
 496          *
 497          * This is a cleanup method primarily meant to be used from unit tests to
 498          * avoid poluting the local filesystem. If used outside of a unit test
 499          * environment it should be used with caution as it may destroy state saved
 500          * in the files.
 501          *
 502          * @see deleteCacheFiles
 503          * @since 1.23
 504          */
 505         public static function unitTestTearDown() {
 506                 // Bug: 44850
 507                 $gen = self::singleton();
 508                 $gen->deleteCacheFiles();
 509         }
 510
 511         function __destruct() {
 512                 array_map( 'fclose', array_filter( $this->fileHandles ) );
 513         }
 514 }