From dafe195c282cb2cfaa6c137d9842b126c94d5fc6 Mon Sep 17 00:00:00 2001
From: Kunal Mehta <legoktm@member.fsf.org>
Date: Thu, 21 Dec 2017 23:24:06 -0800
Subject: [PATCH] Verify that all @covers tags are sane when running tests

PHPUnit only verifies that @covers tags are correct when actually
running coverage - but that's too slow to do on every commit. And if
even a single tag is incorrect, the entire coverage job will fail. It
also has some different requirements compared to normal tags, like all
namespaced classes must use their absolute name. All of those things
combined make it easy for developers to accidentally break the coverage
job.

There are some external tools that also do this, but those tools are
incompatible with our unusual PHPUnit setup (the phpunit.php wrapper
script).

The MediaWikiCoversValidator trait just calls the same method that
PHPUnit does to validate @covers tags. It is implemented as a trait so
that test cases that implement PHPUnit_Framework_TestCase directly can
still use this.

Bug: T171899
Change-Id: I1d564bcae2bfbedb004c440b90db6341148ed4ba
---
 tests/common/TestsAutoLoader.php           |  1 +
 tests/phpunit/MediaWikiCoversValidator.php | 50 ++++++++++++++++++++++
 tests/phpunit/MediaWikiTestCase.php        |  2 +
 3 files changed, 53 insertions(+)
 create mode 100644 tests/phpunit/MediaWikiCoversValidator.php

diff --git a/tests/common/TestsAutoLoader.php b/tests/common/TestsAutoLoader.php
index 44868b30d6..cbf4a1598a 100644
--- a/tests/common/TestsAutoLoader.php
+++ b/tests/common/TestsAutoLoader.php
@@ -62,6 +62,7 @@ $wgAutoloadClasses += [
 	'TestUser' => "$testDir/phpunit/includes/TestUser.php",
 	'TestUserRegistry' => "$testDir/phpunit/includes/TestUserRegistry.php",
 	'LessFileCompilationTest' => "$testDir/phpunit/LessFileCompilationTest.php",
+	'MediaWikiCoversValidator' => "$testDir/phpunit/MediaWikiCoversValidator.php",
 
 	# tests/phpunit/includes
 	'RevisionDbTestBase' => "$testDir/phpunit/includes/RevisionDbTestBase.php",
diff --git a/tests/phpunit/MediaWikiCoversValidator.php b/tests/phpunit/MediaWikiCoversValidator.php
new file mode 100644
index 0000000000..a79a139c8a
--- /dev/null
+++ b/tests/phpunit/MediaWikiCoversValidator.php
@@ -0,0 +1,50 @@
+<?php
+/**
+ * Copyright (C) 2017 Kunal Mehta <legoktm@member.fsf.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+/**
+ * Trait that checks that covers tags are valid, since PHPUnit
+ * won't do it unless you run it with coverage, which is super
+ * slow.
+ *
+ * @since 1.31
+ */
+trait MediaWikiCoversValidator {
+
+	/**
+	 * Test that all methods in this class that begin
+	 * with "test" have valid covers tags.
+	 */
+	public function testValidCovers() {
+		$methods = get_class_methods( $this );
+		$class = get_class( $this );
+		$bad = '';
+		foreach ( $methods as $method ) {
+			if ( strpos( $method, 'test' ) === 0 ) {
+				try {
+					PHPUnit_Util_Test::getLinesToBeCovered( $class, $method );
+				} catch ( PHPUnit_Framework_CodeCoverageException $e ) {
+					$bad .= "$class::$method: {$e->getMessage()}\n";
+				}
+			}
+		}
+
+		$this->assertEquals( '', $bad );
+	}
+}
diff --git a/tests/phpunit/MediaWikiTestCase.php b/tests/phpunit/MediaWikiTestCase.php
index 9f2e5f9395..ef13101196 100644
--- a/tests/phpunit/MediaWikiTestCase.php
+++ b/tests/phpunit/MediaWikiTestCase.php
@@ -14,6 +14,8 @@ use Wikimedia\TestingAccessWrapper;
  */
 abstract class MediaWikiTestCase extends PHPUnit_Framework_TestCase {
 
+	use MediaWikiCoversValidator;
+
 	/**
 	 * The service locator created by prepareServices(). This service locator will
 	 * be restored after each test. Tests that pollute the global service locator
-- 
2.20.1