Sample StatsD messages when instructed

Bug: T106457
Change-Id: I8ddb5a53dcbaf398f5a114d2a4a862842e980db4

diff --git a/autoload.php b/autoload.php
index 0d2e4a8..c3c6375 100644 (file)
--- a/autoload.php
+++ b/autoload.php
@@ -1058,6 +1058,7 @@ $wgAutoloadLocalClasses = array(
        'SQLiteField' => __DIR__ . '/includes/db/DatabaseSqlite.php',
        'SVGMetadataExtractor' => __DIR__ . '/includes/media/SVGMetadataExtractor.php',
        'SVGReader' => __DIR__ . '/includes/media/SVGMetadataExtractor.php',
+       'SamplingStatsdClient' => __DIR__ . '/includes/libs/SamplingStatsdClient.php',
        'Sanitizer' => __DIR__ . '/includes/Sanitizer.php',
        'SavepointPostgres' => __DIR__ . '/includes/db/DatabasePostgres.php',
        'ScopedCallback' => __DIR__ . '/includes/libs/ScopedCallback.php',

diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php
index 167305d..b7b733d 100644 (file)
--- a/includes/GlobalFunctions.php
+++ b/includes/GlobalFunctions.php
@@ -1259,7 +1259,7 @@ function wfLogProfilingData() {
                        $statsdHost = $statsdServer[0];
                        $statsdPort = isset( $statsdServer[1] ) ? $statsdServer[1] : 8125;
                        $statsdSender = new SocketSender( $statsdHost, $statsdPort );
-                       $statsdClient = new StatsdClient( $statsdSender, true, false );
+                       $statsdClient = new SamplingStatsdClient( $statsdSender, true, false );
                        $statsdClient->send( $context->getStats()->getBuffer() );
                } catch ( Exception $ex ) {
                        MWExceptionHandler::logException( $ex );

diff --git a/includes/libs/BufferingStatsdDataFactory.php b/includes/libs/BufferingStatsdDataFactory.php
index 3d7fad5..100d2a4 100644 (file)
--- a/includes/libs/BufferingStatsdDataFactory.php
+++ b/includes/libs/BufferingStatsdDataFactory.php
@@ -20,6 +20,7 @@
  * @file
  */
 
+use Liuggio\StatsdClient\Entity\StatsdData;
 use Liuggio\StatsdClient\Entity\StatsdDataInterface;
 use Liuggio\StatsdClient\Factory\StatsdDataFactory;
 
@@ -75,6 +76,9 @@ class BufferingStatsdDataFactory extends StatsdDataFactory {
                return $entity;
        }
 
+       /**
+        * @return StatsdData[]
+        */
        public function getBuffer() {
                return $this->buffer;
        }

diff --git a/includes/libs/SamplingStatsdClient.php b/includes/libs/SamplingStatsdClient.php
new file mode 100644 (file)
index 0000000..d7791a8
--- /dev/null
+++ b/includes/libs/SamplingStatsdClient.php
@@ -0,0 +1,133 @@
+<?php
+/**
+ * Copyright 2015
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+use Liuggio\StatsdClient\StatsdClient;
+use Liuggio\StatsdClient\Entity\StatsdData;
+use Liuggio\StatsdClient\Entity\StatsdDataInterface;
+
+/**
+ * A statsd client that applies the sampling rate to the data items before sending them.
+ *
+ * @since 1.26
+ */
+class SamplingStatsdClient extends StatsdClient {
+       /**
+        * Sets sampling rate for all items in $data.
+        * The sample rate specified in a StatsdData entity overrides the sample rate specified here.
+        *
+        * {@inheritDoc}
+        */
+       public function appendSampleRate( $data, $sampleRate = 1 ) {
+               if ( $sampleRate < 1 ) {
+                       array_walk( $data, function( $item ) use ( $sampleRate ) {
+                               /** @var $item StatsdData */
+                               if ( $item->getSampleRate() === 1 ) {
+                                       $item->setSampleRate( $sampleRate );
+                               }
+                       });
+               }
+
+               return $data;
+       }
+
+       /**
+        * Sample the metrics according to their sample rate and send the remaining ones.
+        *
+        * {@inheritDoc}
+        */
+       public function send( $data, $sampleRate = 1 ) {
+               if ( !is_array( $data ) ) {
+                       $data = array( $data );
+               }
+               if ( !$data ) {
+                       return;
+               }
+               foreach ( $data as $item ) {
+                       if ( !( $item instanceof StatsdDataInterface ) ) {
+                               throw new InvalidArgumentException(
+                                       'SamplingStatsdClient does not accept stringified messages' );
+                       }
+               }
+
+               // add sampling
+               if ( $sampleRate < 1 ) {
+                       $data = $this->appendSampleRate( $data, $sampleRate );
+               }
+               $data = $this->sampleData( $data );
+
+               $messages = array_map( 'strval', $data );
+
+               // reduce number of packets
+               if ( $this->getReducePacket() ) {
+                       $data = $this->reduceCount( $data );
+               }
+               //failures in any of this should be silently ignored if ..
+               $written = 0;
+               try {
+                       $fp = $this->getSender()->open();
+                       if ( !$fp ) {
+                               return;
+                       }
+                       foreach ( $messages as $message ) {
+                               $written += $this->getSender()->write( $fp, $message );
+                       }
+                       $this->getSender()->close( $fp );
+               } catch ( Exception $e ) {
+                       $this->throwException( $e );
+               }
+
+               return $written;
+       }
+
+       /**
+        * Throw away some of the data according to the sample rate.
+        * @param StatsdDataInterface[] $data
+        * @return array
+        * @throws LogicException
+        */
+       protected function sampleData( $data ) {
+               $newData = array();
+               $mt_rand_max = mt_getrandmax();
+               foreach ( $data as $item ) {
+                       $samplingRate = $item->getSampleRate();
+                       if ( $samplingRate <= 0.0 || $samplingRate > 1.0 ) {
+                               throw new LogicException( 'Sampling rate shall be within ]0, 1]' );
+                       }
+                       if (
+                               $samplingRate === 1 ||
+                               ( mt_rand() / $mt_rand_max <= $samplingRate )
+                       ) {
+                               $newData[] = $item;
+                       }
+               }
+               return $newData;
+       }
+
+       /**
+        * {@inheritDoc}
+        */
+       protected function throwException( Exception $exception ) {
+               if ( !$this->getFailSilently() ) {
+                       throw $exception;
+               }
+       }
+}

diff --git a/tests/phpunit/includes/libs/SamplingStatsdClientTest.php b/tests/phpunit/includes/libs/SamplingStatsdClientTest.php
new file mode 100644 (file)
index 0000000..be6732d
--- /dev/null
+++ b/tests/phpunit/includes/libs/SamplingStatsdClientTest.php
@@ -0,0 +1,43 @@
+<?php
+
+use Liuggio\StatsdClient\Entity\StatsdData;
+
+class SamplingStatsdClientTest extends PHPUnit_Framework_TestCase {
+       /**
+        * @dataProvider samplingDataProvider
+        */
+       public function testSampling( $data, $sampleRate, $seed, $expectWrite ) {
+               $sender = $this->getMock( 'Liuggio\StatsdClient\Sender\SenderInterface' );
+               $sender->expects( $this->any() )->method( 'open' )->will( $this->returnValue( true ) );
+               if ( $expectWrite ) {
+                       $sender->expects( $this->once() )->method( 'write' )
+                               ->with( $this->anything(), $this->equalTo( $data ) );
+               } else {
+                       $sender->expects( $this->never() )->method( 'write' );
+               }
+               mt_srand( $seed );
+               $client = new SamplingStatsdClient( $sender );
+               $client->send( $data, $sampleRate );
+       }
+
+       public function samplingDataProvider() {
+               $unsampled = new StatsdData();
+               $unsampled->setKey( 'foo' );
+               $unsampled->setValue( 1 );
+
+               $sampled = new StatsdData();
+               $sampled->setKey( 'foo' );
+               $sampled->setValue( 1 );
+               $sampled->setSampleRate( '0.1' );
+
+               return array(
+                       // $data, $sampleRate, $seed, $expectWrite
+                       array( $unsampled, 1, 0 /*0.44*/, $unsampled ),
+                       array( $sampled, 1, 0 /*0.44*/, null ),
+                       array( $sampled, 1, 4 /*0.03*/, $sampled ),
+                       array( $unsampled, 0.1, 4 /*0.03*/, $sampled ),
+                       array( $sampled, 0.5, 0 /*0.44*/, null ),
+                       array( $sampled, 0.5, 4 /*0.03*/, $sampled ),
+               );
+       }
+}