From 326f5559073e5f07ccb319fd7e720e9132d793bc Mon Sep 17 00:00:00 2001
From: Chad Horohoe <chadh@wikimedia.org>
Date: Tue, 25 Jul 2017 13:32:31 -0700
Subject: [PATCH] Rewrite pref cleanup script

- Keep the current hidden pref cleanup stuff, that's not harmful
  and marginally useful
- Drop preferences we dunno wtf they're about. Cuz they're probably
  deprecated or otherwise unused
- Normalize preferences into accepted value ranges. This part is kinda
  hard and I haven't figured it out, so slap a TODO

More to come, stay tuned!

Change-Id: I70047adba0034136d107ce7534294cc6fa3c1860
---
 maintenance/cleanupPreferences.php | 130 ++++++++++++++++++++++++++---
 1 file changed, 119 insertions(+), 11 deletions(-)

diff --git a/maintenance/cleanupPreferences.php b/maintenance/cleanupPreferences.php
index 6e58ae9741..f16adcb311 100644
--- a/maintenance/cleanupPreferences.php
+++ b/maintenance/cleanupPreferences.php
@@ -1,6 +1,6 @@
 <?php
 /**
- * Remove hidden preferences from the database.
+ * Clean up user preferences from the database.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -19,6 +19,7 @@
  *
  * @file
  * @author TyA <tya.wiki@gmail.com>
+ * @author Chad <chad@wikimedia.org>
  * @see https://phabricator.wikimedia.org/T32976
  * @ingroup Maintenance
  */
@@ -26,25 +27,132 @@
 require_once __DIR__ . '/Maintenance.php';
 
 /**
- * Maintenance script that removes hidden preferences from the database.
+ * Maintenance script that removes bogus preferences from the database.
  *
  * @ingroup Maintenance
  */
 class CleanupPreferences extends Maintenance {
+	public function __construct() {
+		parent::__construct();
+		$this->mDescription = 'Clean up hidden preferences, removed preferences, and normalizes values';
+		$this->setBatchSize( 50 );
+		$this->addOption( 'dry-run', 'Print debug info instead of actually deleting' );
+		$this->addOption( 'hidden', 'Drop hidden preferences ($wgHiddenPrefs)' );
+		$this->addOption( 'unknown',
+			'Drop unknown preferences (not in $wgDefaultUserOptions or a gadget or userjs)' );
+		// TODO: actually implement this
+		// $this->addOption( 'bogus', 'Drop preferences that have invalid/unaccepted values' );
+	}
+
+	/**
+	 * We will do this in three passes
+	 *   1) The easiest is to drop the hidden preferences from the database. We
+	 *      don't actually want them
+	 *   2) Drop preference keys that we don't know about. They could've been
+	 *      removed from core, provided by a now-disabled extension, or the result
+	 *      of a bug. We don't want them.
+	 *   3) TODO: Normalize accepted preference values. This is the biggest part of the work.
+	 *      For each preference we know about, iterate over it and if it's got a
+	 *      limited set of accepted values (so it's not text, basically), make sure
+	 *      all values are in that range. Drop ones that aren't.
+	 */
 	public function execute() {
-		global $wgHiddenPrefs;
+		global $wgHiddenPrefs, $wgDefaultUserOptions;
 
 		$dbw = $this->getDB( DB_MASTER );
-		$this->beginTransaction( $dbw, __METHOD__ );
-		foreach ( $wgHiddenPrefs as $item ) {
-			$dbw->delete(
+		$didWork = false;
+		$hidden = $this->hasOption( 'hidden' );
+		$unknown = $this->hasOption( 'unknown' );
+		$bogus = $this->hasOption( 'bogus' );
+
+		if ( !$hidden && !$unknown && !$bogus ) {
+			$this->output( "Did not select one of --hidden, --unknown or --bogus, exiting\n" );
+			return;
+		}
+
+		// Remove hidden prefs. Iterate over them to avoid the IN on a large table
+		if ( $hidden ) {
+			if ( !$wgHiddenPrefs ) {
+				$this->output( "No hidden preferences, skipping\n" );
+			}
+			foreach ( $wgHiddenPrefs as $hiddenPref ) {
+				$this->deleteByWhere(
+					$dbw,
+					'Dropping hidden preferences',
+					[ 'up_property' => $hiddenPref ]
+				);
+			}
+		}
+
+		// Remove unknown preferences. Special-case gadget- and userjs- as we can't
+		// control those names.
+		if ( $unknown ) {
+			$this->deleteByWhere(
+				$dbw,
+				'Dropping unknown preferences',
+				[
+					'up_property NOT' . $dbw->buildLike( 'gadget-', $dbw->anyString() ),
+					'up_property NOT' . $dbw->buildLike( 'userjs-', $dbw->anyString() ),
+					'up_property NOT IN (' . $dbw->makeList( array_keys( $wgDefaultUserOptions ) ) . ')',
+				]
+			);
+		}
+
+		// Something something phase 3
+		if ( $bogus ) {
+		}
+	}
+
+	/**
+	 *
+	 */
+	private function deleteByWhere( $dbw, $startMessage, $where ) {
+		$this->output( $startMessage . "...\n" );
+		$total = 0;
+		while ( true ) {
+			$res = $dbw->select(
 				'user_properties',
-				[ 'up_property' => $item ],
-				__METHOD__
+				'*', // The table lacks a primary key, so select the whole row
+				$where,
+				__METHOD__,
+				[ 'LIMIT' => $this->mBatchSize ]
 			);
-		};
-		$this->commitTransaction( $dbw, __METHOD__ );
-		$this->output( "Finished!\n" );
+
+			$numRows = $res->numRows();
+			$total += $numRows;
+			if ( $res->numRows() <= 0 ) {
+				// All done!
+				$this->output( "DONE! (handled $total entries)\n" );
+				break;
+			}
+
+			// Progress or something
+			$this->output( "..doing $numRows entries\n" );
+
+			// Delete our batch, then wait
+			foreach ( $res as $row ) {
+				if ( $this->hasOption( 'dry-run' ) ) {
+					$this->output(
+						"    DRY RUN, would drop: " .
+						"[up_user] => '{$row->up_user}' " .
+						"[up_property] => '{$row->up_property}' " .
+						"[up_value] => '{$row->up_value}'\n"
+					);
+					continue;
+				}
+				$this->beginTransaction( $dbw, __METHOD__ );
+				$dbw->delete(
+					'user_properties',
+					[
+						'up_user'     => $row->up_user,
+						'up_property' => $row->up_property,
+						'up_value'    => $row->up_value,
+					],
+					__METHOD__
+				);
+				$this->commitTransaction( $dbw, __METHOD__ );
+			}
+		}
 	}
 }
 
-- 
2.20.1