From e56f7b6c63acbb579888c0abf1c70c06f296d612 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Gerg=C5=91=20Tisza?= Date: Thu, 23 Jul 2015 20:13:27 +0000 Subject: [PATCH] Sample StatsD messages when instructed Bug: T106457 Change-Id: I8ddb5a53dcbaf398f5a114d2a4a862842e980db4 --- autoload.php | 1 + includes/GlobalFunctions.php | 2 +- includes/libs/BufferingStatsdDataFactory.php | 4 + includes/libs/SamplingStatsdClient.php | 133 ++++++++++++++++++ .../libs/SamplingStatsdClientTest.php | 43 ++++++ 5 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 includes/libs/SamplingStatsdClient.php create mode 100644 tests/phpunit/includes/libs/SamplingStatsdClientTest.php diff --git a/autoload.php b/autoload.php index 0d2e4a8115..c3c6375ea5 100644 --- a/autoload.php +++ b/autoload.php @@ -1058,6 +1058,7 @@ $wgAutoloadLocalClasses = array( 'SQLiteField' => __DIR__ . '/includes/db/DatabaseSqlite.php', 'SVGMetadataExtractor' => __DIR__ . '/includes/media/SVGMetadataExtractor.php', 'SVGReader' => __DIR__ . '/includes/media/SVGMetadataExtractor.php', + 'SamplingStatsdClient' => __DIR__ . '/includes/libs/SamplingStatsdClient.php', 'Sanitizer' => __DIR__ . '/includes/Sanitizer.php', 'SavepointPostgres' => __DIR__ . '/includes/db/DatabasePostgres.php', 'ScopedCallback' => __DIR__ . '/includes/libs/ScopedCallback.php', diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php index 167305d9ab..b7b733d2dd 100644 --- a/includes/GlobalFunctions.php +++ b/includes/GlobalFunctions.php @@ -1259,7 +1259,7 @@ function wfLogProfilingData() { $statsdHost = $statsdServer[0]; $statsdPort = isset( $statsdServer[1] ) ? $statsdServer[1] : 8125; $statsdSender = new SocketSender( $statsdHost, $statsdPort ); - $statsdClient = new StatsdClient( $statsdSender, true, false ); + $statsdClient = new SamplingStatsdClient( $statsdSender, true, false ); $statsdClient->send( $context->getStats()->getBuffer() ); } catch ( Exception $ex ) { MWExceptionHandler::logException( $ex ); diff --git a/includes/libs/BufferingStatsdDataFactory.php b/includes/libs/BufferingStatsdDataFactory.php index 3d7fad5179..100d2a4e38 100644 --- a/includes/libs/BufferingStatsdDataFactory.php +++ b/includes/libs/BufferingStatsdDataFactory.php @@ -20,6 +20,7 @@ * @file */ +use Liuggio\StatsdClient\Entity\StatsdData; use Liuggio\StatsdClient\Entity\StatsdDataInterface; use Liuggio\StatsdClient\Factory\StatsdDataFactory; @@ -75,6 +76,9 @@ class BufferingStatsdDataFactory extends StatsdDataFactory { return $entity; } + /** + * @return StatsdData[] + */ public function getBuffer() { return $this->buffer; } diff --git a/includes/libs/SamplingStatsdClient.php b/includes/libs/SamplingStatsdClient.php new file mode 100644 index 0000000000..d7791a87a6 --- /dev/null +++ b/includes/libs/SamplingStatsdClient.php @@ -0,0 +1,133 @@ +getSampleRate() === 1 ) { + $item->setSampleRate( $sampleRate ); + } + }); + } + + return $data; + } + + /** + * Sample the metrics according to their sample rate and send the remaining ones. + * + * {@inheritDoc} + */ + public function send( $data, $sampleRate = 1 ) { + if ( !is_array( $data ) ) { + $data = array( $data ); + } + if ( !$data ) { + return; + } + foreach ( $data as $item ) { + if ( !( $item instanceof StatsdDataInterface ) ) { + throw new InvalidArgumentException( + 'SamplingStatsdClient does not accept stringified messages' ); + } + } + + // add sampling + if ( $sampleRate < 1 ) { + $data = $this->appendSampleRate( $data, $sampleRate ); + } + $data = $this->sampleData( $data ); + + $messages = array_map( 'strval', $data ); + + // reduce number of packets + if ( $this->getReducePacket() ) { + $data = $this->reduceCount( $data ); + } + //failures in any of this should be silently ignored if .. + $written = 0; + try { + $fp = $this->getSender()->open(); + if ( !$fp ) { + return; + } + foreach ( $messages as $message ) { + $written += $this->getSender()->write( $fp, $message ); + } + $this->getSender()->close( $fp ); + } catch ( Exception $e ) { + $this->throwException( $e ); + } + + return $written; + } + + /** + * Throw away some of the data according to the sample rate. + * @param StatsdDataInterface[] $data + * @return array + * @throws LogicException + */ + protected function sampleData( $data ) { + $newData = array(); + $mt_rand_max = mt_getrandmax(); + foreach ( $data as $item ) { + $samplingRate = $item->getSampleRate(); + if ( $samplingRate <= 0.0 || $samplingRate > 1.0 ) { + throw new LogicException( 'Sampling rate shall be within ]0, 1]' ); + } + if ( + $samplingRate === 1 || + ( mt_rand() / $mt_rand_max <= $samplingRate ) + ) { + $newData[] = $item; + } + } + return $newData; + } + + /** + * {@inheritDoc} + */ + protected function throwException( Exception $exception ) { + if ( !$this->getFailSilently() ) { + throw $exception; + } + } +} diff --git a/tests/phpunit/includes/libs/SamplingStatsdClientTest.php b/tests/phpunit/includes/libs/SamplingStatsdClientTest.php new file mode 100644 index 0000000000..be6732d52f --- /dev/null +++ b/tests/phpunit/includes/libs/SamplingStatsdClientTest.php @@ -0,0 +1,43 @@ +getMock( 'Liuggio\StatsdClient\Sender\SenderInterface' ); + $sender->expects( $this->any() )->method( 'open' )->will( $this->returnValue( true ) ); + if ( $expectWrite ) { + $sender->expects( $this->once() )->method( 'write' ) + ->with( $this->anything(), $this->equalTo( $data ) ); + } else { + $sender->expects( $this->never() )->method( 'write' ); + } + mt_srand( $seed ); + $client = new SamplingStatsdClient( $sender ); + $client->send( $data, $sampleRate ); + } + + public function samplingDataProvider() { + $unsampled = new StatsdData(); + $unsampled->setKey( 'foo' ); + $unsampled->setValue( 1 ); + + $sampled = new StatsdData(); + $sampled->setKey( 'foo' ); + $sampled->setValue( 1 ); + $sampled->setSampleRate( '0.1' ); + + return array( + // $data, $sampleRate, $seed, $expectWrite + array( $unsampled, 1, 0 /*0.44*/, $unsampled ), + array( $sampled, 1, 0 /*0.44*/, null ), + array( $sampled, 1, 4 /*0.03*/, $sampled ), + array( $unsampled, 0.1, 4 /*0.03*/, $sampled ), + array( $sampled, 0.5, 0 /*0.44*/, null ), + array( $sampled, 0.5, 4 /*0.03*/, $sampled ), + ); + } +} -- 2.20.1