MediaViki:Gadget-libGlobalReplace.js

/**
* [[MediaWiki:Gadget-libGlobalReplace.js]]
* Replaces a file on all wikis, including Wikimedia Commons
* Uses either CORS under the current user account
* or deputes the task to CommonsDelinker
*
* The method used is determined by
* -Browser capabilities (CORS required)
* -The usage count: More than the given number
*                   aren't attempted to be replaced
*                   under the user account
*
* It adds only one public method to the mw.libs - object:
* @example
*      var $jQuery_Deferred_Object;
*      $jQuery_Deferred_Object = mw.libs.globalReplace(oldFile, newFile, shortReason, fullReason);
*      $jQuery_Deferred_Object.done(function() { alert("Good news! " + oldFile + " has been replaced by " + newFile + "!") });
*
* Internal stuff:
* Since we don't use instances of classes, we have to pass around all the parameters
*
* TODO: I18n (progress messages) when Krinkle is ready with Gadgets 2.0 :-)
*
* @rev 1 (2012-11-26)
* @rev 5 (2017-12-15)
* @author Rillke, 2012-2015
* <nowiki>
*/
// List the global variables for jsHint-Validation. Please make sure that it passes http://jshint.com/
// Scheme: globalVariable:allowOverwriting[, globalVariable:allowOverwriting][, globalVariable:allowOverwriting]
/*global jQuery:false, mediaWiki:false*/

// Set jsHint-options. You should not set forin or undef to false if your script does not validate.
/*jshint forin:true, noarg:true, noempty:true, eqeqeq:true, bitwise:true, strict:true, undef:true, curly:false, browser:true*/

(function ($, mw) {
"use strict";

// Config
// When this number is exceeded or reached, use CommonsDelinker
// This number must not be higher than 50
// (can't query more than 50 titles at once)
var usageThreshold = 45;

// Internal stuff
var CORSsupported = false;

/**
 * TODO: Outsource to library as I often use them OR does jQuery provide something like that?
 **/
if (!Object.keys)
	Object.keys = function (o) {
		var k = [], p;
		for (p in o) if (o.hasOwnProperty.call(o, p)) k.push(p);
		return k;
	};

var _firstItem = function (o) {
	return o[Object.keys(o)[0]];
};

// TODO: Keep in sync with CommonsDelinker source
// https://bitbucket.org/magnusmanske/commons-delinquent/src/master/demon.php
var getFileRegEx = function (title, prefix) {
	prefix = prefix || '[\\n\\[\\:\\=\\>\\|]\\s*';
	return new RegExp('(' + prefix + ')[' + mw.RegExp.escape(title[0].toUpperCase() + title[0].toLowerCase()) + ']' + mw.RegExp.escape(
		title.slice(1)).replace(/ /g, '[ _]'), 'g');
};

var queryGET = function (params, cb, errCb) {
	mw.loader.using(['ext.gadget.libAPI'], function () {
		params.action = params.action || 'query';
		mw.libs.commons.api.query(params, {
			cache: true,
			cb: cb,
			errCb: errCb
		});
	});
};

var centralToken = {},
	edittoken = mw.user.tokens.get('csrfToken'),
	fbToken, // fallback
	fetchingFbToken;

function getCentralAuth(cb, errCb, wiki) {
	new mw.Api().get({
		action: 'centralauthtoken'
	}).done(function (r) {
		fetchingFbToken = false;
		queryCentralToken(r, cb, wiki);
	}).fail(errCb);
}

function testCORS(done, wiki) {
	mw.loader.using(['mediawiki.user', 'mediawiki.api', 'mediawiki.ForeignApi']).done(function () {
		if (CORSsupported)
			return done();
		doCORSreq({
			meta: 'tokens|userinfo'
		}, wiki || 'www.mediawiki.org', function (data, textStatus) {
			if (!data.query || !data.query.userinfo.id) {
				CORSsupported = 'CORS supported but not logged-in';
				mw.log(CORSsupported, data, textStatus);
			} else { 
				CORSsupported = 'OK';
			}
			done(data);
		}, function (jqXHR, textStatus, errorThrown) {
			CORSsupported = 'CORS not supported: ' + textStatus + ' \nError: ' + errorThrown;
			done();
		});
	});
}

var doCORSreq = function (params, wiki, cb, errCb, method) {
	$.support.cors = true;
	var api = new mw.ForeignApi('//' + wiki + '/w/api.php'); // mw.util.wikiScript("api")
	method = method === 'POST' ? 'post' : 'get';
	api[method](params).done(function (r) {
			cb(r, wiki);
		}).fail(errCb);
};

function getFbToken(cb, wiki) {
	if (fbToken) return cb(fbToken);
	if (fetchingFbToken) return;
	
	var para = { meta: 'tokens' };
	var h = mw.hook('commons.libglobalreplace.fbToken.fetched').add(cb);
	var errCb = function (r) {
		centralToken.centralauthtoken = 0;
		fbToken = '+\\';
		throw new Error("Error fetching csrftoken from Wikidata. ");
	};
	fetchingFbToken = true;

	if (centralToken.centralauthtoken) {
		para.centralauthtoken = centralToken.centralauthtoken;
		centralToken.centralauthtoken = 0;
	}
	if (!para.centralauthtoken) {
		CORSsupported = false;
		// Test logged-in
		return testCORS(function (r) {
			// If the user is suddenly reported to be logged-out try again.
			if (CORSsupported !== 'OK')
				return getCentralAuth(cb, errCb, wiki);
			
			fbToken = r.query.tokens.csrftoken;
			h.fire( fbToken );
		}, wiki);
	}

	doCORSreq(para, wiki, function (r) {
		fbToken = r.query.tokens.csrftoken;
		getCentralAuth(cb, errCb, wiki); // Need new authtoken
	}, errCb);
}

var updateReplaceStatus = function ($prog) {
	// If we are using CommonsDelinker (CD), it will mark this progress object
	// as resolved as soon as the requst was placed in the queue; 
	// Don't know whether we should stop replacement under user account
	// when we request CD to do our job; but see no pressing need to
	if (!$prog.remaining && !$prog.usingCD) {
		$prog.resolve("All usages replaced");
		// Kill the timer: Everything worked in time!
		if ($prog.CDtimeout) clearTimeout($prog.CDtimeout);
	}
	$prog.notify("Replacing usage: " +
		Math.round(($prog.total - $prog.remaining) * 100 / $prog.total) +
		"% (" + ($prog.total - $prog.remaining) + "/" + $prog.total +
		") \nDo not close this window until the task is completed.");
};
var decrementAndUpdate = function ($prog) {
	$prog.remaining--;
	updateReplaceStatus($prog);
};
var incrementAndUpdate = function ($prog) {
	$prog.remaining++;
	updateReplaceStatus($prog);
};
var checkPage = function ($prog, pg, wiki, cb) {
	if (!pg.revisions) {
		$prog.notify("No page text for " + pg.title + " – " + wiki + " – private wiki or out of date?");
		if (cb && $.isFunction(cb))
			cb();
		return false;
	} else {
		return true;
	}
};
var compareTexts = function ($prog, oldT, newT, title, wiki) {
	if (oldT === newT) {
		$prog.notify("No changes at " + title + " – " + wiki + " – template use?");
		decrementAndUpdate($prog);
		return false;
	} else {
		return true;
	}
};

function noUnlinkFromNamespace( pg, $prog ) {
	return ( pg.ns % 2 ) || // Skip talk pages
		( pg.ns < 0 ) || // Paranoia
		( $prog.notOnNs && $.inArray( pg.ns, $prog.notOnNs ) >= 0 ); // Skip optional namespaces
}

/**
 *  Replace usage at Wikimedia Commons.
 **/
var localReplace = function (re, localUsage, of, nf, sr, fr, $prog) {
	
	function isBadPage( pg ) {
		return ( pg.ns === 6 && $.inArray( pg.title.replace( /^File\:/, '' ), [of, nf] ) !== -1 ) || // Self-reference
			( pg.ns === 2 && /^User:\w+Bot\b/.test( pg.title ) ) || // Bot subpage on Commons
			( pg.ns === 4 && /(Deletion[_ ]requests\/[^\n]*|Undeletion[_ ]requests\/[^\n]*)\b/.test( pg.title ) ); // DR and UDR on Commons
	}
	
	$.each(localUsage, function (id, pg) {
		// Check page exists
		if (!checkPage($prog, pg, 'Commons') || isBadPage(pg) || noUnlinkFromNamespace(pg, $prog)) {
			decrementAndUpdate($prog);
			return mw.log('LocalReplace skipped for', pg.title);
		}

		var isEditable = true,
			summary = sr + ' [[File:' + of + ']] → [[File:' + nf + ']] ' + fr,
			edit;

		$.each(pg.protection, function (i, pr) {
			if ('edit' === pr.type) {
				if ($.inArray(pr.level, mw.config.get('wgUserGroups')) === -1)
					isEditable = false;
				return false;
			}
		});

		if (isEditable) {
			var oldText = pg.revisions[0]['*'],
				nwe1 = mw.libs.wikiDOM.nowikiEscaper(pg.revisions[0]['*']),
				newText = nwe1.secureReplace(re, '$1' + nf).getText();

			if (!compareTexts($prog, oldText, newText, pg.title, "Commons")) return;

			edit = {
				cb: function () {
					decrementAndUpdate($prog);
				},
				errCb: function () {
					decrementAndUpdate($prog);
					$prog.notify("Unable to update " + pg.title + " \nUsing CommonsDelinker");
					commonsDelinker(of, nf, sr, fr, $prog);
				},
				title: pg.title,
				text: newText,
				editType: 'text',
				watchlist: 'nochange',
				minor: true,
				summary: summary,
				basetimestamp: pg.revisions[0].timestamp
			};
		} else {
			// If page is protected, post a request to the talk page
			edit = {
				cb: function () {
					decrementAndUpdate($prog);
				},
				errCb: function () {
					decrementAndUpdate($prog);
				},
				title: mw.libs.commons.getTalkPageFromTitle(pg.title),
				text: "== Please replace [[:File:" + of + "]] ==\n{{edit request}}\nThis page is protected while posting this message. " + "Please replace <code>[[:File:" + of + "]]</code> with <code>[[:File:" + nf + "]]</code> because " + sr + " " + fr + "\nThank you. " + "<small>Message added by [[:c:GR|global replace]]</small> -- ~~~~",
				editType: 'appendtext',
				watchlist: 'nochange',
				minor: true,
				summary: summary
			};
		}
		mw.loader.using(['ext.gadget.libAPI', 'mediawiki.user'], function () {
			if (!mw.user.isAnon())
				edit.assert = 'user';
			mw.libs.commons.api.editPage(edit);
		});
	});
};
/**
 *  Replace usage in other wikis.
 *  It's not uncommon that edits fail due to title blacklist, abuse filter,
 *  captcha, server timeouts, protected pages etc. but in this case
 *  we kindly ask CommonsDelinker whether it will do the remaining ones for us.
 **/
var globalReplace = function (re, globalUsage, of, nf, sr, fr, $prog) {
	var guWiki = {};
	// First we have to compile a list of pages per wiki
	$.each(globalUsage, function (i, gu) {
		if (!(gu.wiki in guWiki)) {
			guWiki[gu.wiki] = [gu.title];
		} else {
			guWiki[gu.wiki].push(gu.title);
		}
	});

	var gotPagesContents = function (result, wiki) {
		$prog.notify("Got page contents for " + wiki + ". Updating them now.");

		var editNow = function (edit) {
			if (!mw.user.isAnon())
				edit.assert = 'user';
			doCORSreq(edit, wiki, function (r) {
				mw.log('editNow', r);
				if (r.error || (r.edit && (r.edit.spamblacklist || 'Success' !== r.edit.result))) {
					// ERROR
					_onErr(r);
				} else {
					// SUCCESS
					decrementAndUpdate($prog);
				}
			}, _onErr, 'POST' );
		};

		var summary = '([[c:GR|GR]]) ' + sr.replace(/\[\[(.+)\]\]/, '[[c:$1]]') +
			' [[File:' + of + ']] → [[File:' + nf + ']] ' +
			fr.replace(/\[\[(.+?)\]\]/g, '[[c:$1]]');

		var edit = {
			action: 'edit',
			summary: summary,
			minor: true,
			nocreate: true,
			watchlist: 'nochange'
		};

		var wdEdit = {
			action: 'wbsetclaimvalue',
			snaktype: 'value',
			summary: summary
		};

		var _onErr = function (r) {
			getPageContentsFailed('', wiki, JSON.stringify(r) + " Unable to update page at ");
		};
		
		edittoken = result.query.tokens.csrftoken;

		// TODO: Work around protection
		$.each(result.query.pages, function (id, pg) {

			if (!checkPage($prog, pg, wiki, function () {
					// Perhaps it's a private wiki and CommonsDelinker has access?
					commonsDelinker(of, nf, sr, fr, $prog);
				}) || noUnlinkFromNamespace(pg, $prog)) {
				decrementAndUpdate($prog);
				return;
			}

			var replacementCount = 0,
				newText,
				oldText = pg.revisions[0]['*'];

			if (wiki === 'www.wikidata.org' && pg.contentmodel === 'wikibase-item') {
				try {
					newText = JSON.parse(oldText);
					$.each(newText.claims, function (propId, propClaims) {
						$.each(propClaims, function (idx, claim) {
							if (claim.type !== 'statement') return;
							if (!claim.mainsnak || !claim.mainsnak.datavalue ||
								typeof claim.mainsnak.datavalue.value !== 'string') return;
							if (sanitizeFileName(claim.mainsnak.datavalue.value) === sanitizeFileName(of)) {
								replacementCount++;
								if (replacementCount > 1)
									incrementAndUpdate($prog);
								getFbToken(function (token) {
									$.extend(wdEdit, {
										claim: claim.id,
										baserevid: pg.lastrevid,
										value: JSON.stringify(nf),
										token: token
									});
									if (centralToken.centralauthtoken) {
										wdEdit.centralauthtoken = centralToken.centralauthtoken;
										centralToken.centralauthtoken = 0;
									}
									editNow(wdEdit);
								}, 'www.wikidata.org');
							}
						});
					});
					if (!replacementCount) {
						return getPageContentsFailed('', wiki, "Nothing suitable for replacement found on " + pg.title + " on ");
					}
				} catch (noMatterWhat) {
					return getPageContentsFailed('', wiki, noMatterWhat + " Issue replacing usage on entry " + pg.title + " on ");
				}
			} else {
				var editNowCB = function(token) {
					if (!token || /^\+\\+$/.test(token))
						return getPageContentsFailed('', wiki, 'No token for ');
					newText = mw.libs.wikiDOM.nowikiEscaper(oldText).secureReplace(re, '$1' + nf).getText();
					if (!compareTexts($prog, oldText, newText, pg.title, wiki)) return;
					$.extend(edit, {
						title: pg.title,
						starttimestamp: result.curtimestamp,
						basetimestamp: pg.revisions[0].timestamp,
						text: newText,
						token: token
					});
					if (centralToken.centralauthtoken) {
						edit.centralauthtoken = centralToken.centralauthtoken;
						centralToken.centralauthtoken = 0;
					}
					editNow(edit);
				};
				
				if (!edittoken || /^\+\\+$/.test(edittoken)) {
					// Try get fallback token
					return getFbToken(editNowCB, wiki);
				}
				editNowCB(edittoken);
			}
		});
	};
	
	function getPageContentsFailed(err, wiki, text) {
		err += err ? " \n" : " ";
		$prog.notify((text || "Unable to get information from ") + wiki + err + "\nUsing CommonsDelinker");
		decrementAndUpdate($prog);
		commonsDelinker(of, nf, sr, fr, $prog);
		return false;
	}

	// Then send out the queries to the Wikis
	$.each(guWiki, function (wiki, titles) {
		var runReplacements = function () {
			doCORSreq({
				prop: 'info|revisions',
				curtimestamp: 1,
				meta: 'tokens',
				rvprop: 'content|timestamp',
				titles: titles.join('|').replace(/_/g, ' ')
			}, wiki, gotPagesContents, getPageContentsFailed);
		};

		// Now, it's possible that the wiki has a local file with the new name,
		// a so-called "shadow".
		// In this case the replacement is most likely undesired.
		var gotLocalImages = function (r) {
			if (r && r.query && r.query.allimages && r.query.allimages.length) {
				// Skip this wiki
				$prog.notify("Skipping " + wiki + " because there is a shadow file with the same target name.");
				$prog.remaining -= titles.length;
				updateReplaceStatus($prog);
			} else {
				runReplacements();
			}
		};

		doCORSreq({
			list: 'allimages',
			aifrom: nf,
			aito: nf
		}, wiki, gotLocalImages, runReplacements);
	});
};

/**
 *  Asks CommonsDelinker to replace a file.
 **/
var commonsDelinker = function (of, nf, sr, fr, $prog) {
	// Don't ask CommonsDelinker multiple times 
	// to replace the same file
	if ($prog.usingCD) return;

	if ($prog.dontUseCD)
		return $prog.reject("Unable replacing all usages. Usually CD would now have been instructed but you wished not to do so.");

	// Tell other processes that we're now using the delinker
	// So they don't stop us by resolving the progress
	$prog.usingCD = true;

	mw.libs.globalReplaceDelinker(of, nf, sr + ' ' + fr, function () {
		$prog.resolve("CommonsDelinker has been instructed to replace " + of + " with " + nf);
	}, function (t) {
		$prog.reject("Error while asking CommonsDelinker to replace " + of + " with " + nf + " Reason: " + t);
	});
};

var sanitizeFileName = function (fn) {
	return $.trim(fn.replace(/_/g, ' ')).replace(/^(?:File|Image)\:/, '');
};

function queryCentralToken(token, cb, wiki) {
	if (token.centralauthtoken)
		centralToken = token.centralauthtoken;
	getFbToken(cb, wiki);
}

/**
 * @param {string} of Old file name. The old file name will be replaced with the new file name.
 * @param {string} nf New file name.
 * @param {string} sr Short reason like "file renamed". Will be prefixed to the edit summary.
 * @param {string} fr Full reason like "file renamed because it was offending". Will be appended to the edit summary.
 * @param {$.Deferred} $prog Deferred object reflecting the current progress.
 **/
var replace = function (of, nf, sr, fr, $prog) {
	var pending = 0,
		localResult,
		globalResult;

	of = sanitizeFileName(of);
	nf = sanitizeFileName(nf);

	var _queryLocal = function (result) {
		pending--;
		if (result) localResult = result;
		if (pending > 0) return;
		_selectMethod();
	};
	var _queryGlobal = function (result) {
		pending--;
		if (result) globalResult = result;
		if (pending > 0) return;
		_selectMethod();
	};
	var _selectMethod = function () {
		var globalUsage = _firstItem(globalResult.query.pages).globalusage,
			globalUsageCount = globalUsage.length,
			localUsage = localResult.query ? localResult.query.pages : {},
			usageCount = Object.keys(localUsage).length + globalUsageCount;

		$prog.remaining = usageCount;
		$prog.total = usageCount;
		mw.log(CORSsupported);
		if (!usageCount) {
			$prog.resolve("File was not in use. Nothing replaced.");
		} else if ((usageCount >= usageThreshold || (CORSsupported !== 'OK' && globalUsageCount)) && !$prog.dontUseCD) {
			$prog.notify("Instructing CommonsDelinker to replace this file");
			commonsDelinker(of, nf, sr, fr, $prog);
		} else {
			if (usageCount - globalUsageCount) localReplace(getFileRegEx(of, '(?:[\\n\\[\\=\\>\\|]|[\\n\\[\\=\\>\\|][Ff]ile\\:)\\s*'), localUsage, of, nf, sr, fr, $prog);
			if (globalUsageCount) globalReplace(getFileRegEx(of), globalUsage, of, nf, sr, fr, $prog);
			$prog.notify("Replacing usage immediately using your user account. Do not close this window until the process completed.");
		}
		// Finally, set a timeout that will instruct CommonsDelinker if it takes too long
		$prog.CDtimeout = setTimeout(function () {
			commonsDelinker(of, nf, sr, fr, $prog);
		}, 50000);
	};

	$prog.notify("Query usage and selecting replace-method");
	pending++;
	queryGET({
		generator: 'imageusage',
		giufilterredir: 'nonredirects',
		giulimit: usageThreshold,
		prop: 'info|revisions',
		inprop: 'protection',
		rvprop: 'content|timestamp',
		giuredirect: 1,
		giutitle: 'File:' + of
	}, _queryLocal);
	pending++;
	queryGET({
		prop: 'globalusage',
		guprop: '',
		gulimit: usageThreshold,
		gufilterlocal: 1,
		titles: 'File:' + of
	}, _queryGlobal);
	pending++;
	testCORS(function () {
		pending--;
		if (pending > 0)
			return;
		_selectMethod();
	});
};

// Expose globally
/**
 * @param {string} oldFile Old file name. The old file name will be replaced with the new file name.
 *                     Can be in any format (both "File:Abc def.png" and "Abc_def.png" work)
 * @param {string} newFile New file name.
 *                     Can be in any format (both "File:Abc def.png" and "Abc_def.png" work)
 * @param {string} shortReason Short reason like "file renamed". Will be prefixed to the edit summary.
 * @param {string} fullReason Full reason like "file renamed because it was offending". Will be appended to the edit summary.
 * @param {boolean} dontUseDelinker Prevents usage of CommonsDelinker (only provided for debugging/scripting)
 * @param {array} notOnNamespaces Skip optional namespacenumbers
 * @return {$.Deferred} $prog jQuery deferred-object reflecting the current progress. See http://api.jquery.com/category/deferred-object/ for more info.
 * @examle See this gadget's introduction.
 **/
mw.libs.globalReplace = function (oldFile, newFile, shortReason, fullReason, dontUseDelinker, notOnNamespaces) {
	var $progress = $.Deferred();
	$progress.pendingQueries = 0;
	$progress.dontUseCD = dontUseDelinker;
	$progress.notOnNs = $.isArray(notOnNamespaces) ? notOnNamespaces : false;
	var args = Array.prototype.slice.call(arguments, 0);
	// Delete optional dontUseDelinker and notOnNamespaces
	if (args.length > 4) args.splice(4);
	// Add progress
	args.push($progress);
	replace.apply(this, args);
	return $progress;
};
mw.libs.globalReplaceDelinker = function (oldFile, newFile, reason, cb, errCb) {
	oldFile = sanitizeFileName(oldFile);
	newFile = sanitizeFileName(newFile);

	reason = reason.replace(/\{/g, '&#123;').replace(/\}/g, '&#125;').replace(/\=/g, '&#61;');
	var edit = {
		cb: cb,
		errCb: errCb,
		title: 'User:CommonsDelinker/commands',
		text: '\n{{universal replace|' + oldFile + '|' + newFile + '|reason=' + reason + '}}',
		editType: 'appendtext',
		watchlist: 'nochange',
		summary: 'universal replace: [[File:' + oldFile + ']] → [[File:' + newFile + ']]'
	};
	if ($.inArray('sysop', mw.config.get('wgUserGroups')) === -1) {
		edit.title = 'User:CommonsDelinker/commands/filemovers';
	}
	mw.loader.using(['ext.gadget.libAPI'], function () {
		mw.libs.commons.api.editPage(edit);
	});
};

}(jQuery, mediaWiki));
//</nowiki>