User:DYKUpdateBot/Code

Below is the code for DYKUpdateBot. Many thanks to the developers of the JavaWikiBotFramework (JWBF), which made this possible. The bot runs on revision 178 of the JWBF and version 2.5 of Apache's Commons Lang library.

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Scanner;

import org.apache.commons.lang.StringEscapeUtils;
import org.jdom.Document;
import org.jdom.Element;

import net.sourceforge.jwbf.actions.mw.MediaWiki;
import net.sourceforge.jwbf.bots.MediaWikiBot;
import net.sourceforge.jwbf.contentRep.mw.SimpleArticle;

public class DYKUpdateBot extends EnWikiBot {

	private static final String TDYKLoc = "Template:Did you know";
	private static final String QueueRootLoc = "Template:Did you know/Queue/";
	private static final String TimeLoc = "Template:Did you know/Next update/Time";
	private static final String NextUpdateQueueLoc = "Template:Did you know/Queue/Next";
	private static final String ClearTemplate = "{{User:DYKUpdateBot/REMOVE THIS LINE}}";
	private static final String TimeBetweenUpdatesLoc = "User:DYKUpdateBot/Time Between Updates";
	private static final String ArchiveLoc = "Wikipedia:Recent additions";
	private static final String WTDYKLoc = "Wikipedia talk:Did you know";
	private static final String ErrorOutputLoc = "User:DYKUpdateBot/Errors";
	private static final String DriftLoc = "User:DYKUpdateBot/ResyncDrift";
	private static final String BaseCommonsAPIURL = "https://commons.wikimedia.org/w/";
	private static final int TimeBetweenEdits = 5; // in seconds
	private static final int TimeBetweenStatusChecks = 600; // in seconds
	private static final int NumQueues = 7;
	private static final int NumExceptionsBeforeAttemptedReset = 55;
	private static final String[] shipTemplates = {"ship", "sclass", "Jsub", 
		"barge", "GTS", "HSC", "MS", "MV", "PS", "SS", "tugboat", "HMAS", 
		"HMCS", "HMNZS", "HMS", "RMS", "USAT", "USCGC", "USNS", "USRC", "USS", 
		"SMS", "SMU", "GS", "HNLMS", "HNoMS" };
	private StringBuilder errorLog;
	private int nextQueue;
	
	public DYKUpdateBot(int timeBetweenEdits, int numExceptionsBeforeAttemptedReset, 
			String purgeLoc, String userName, String password) {
		super(timeBetweenEdits, numExceptionsBeforeAttemptedReset, purgeLoc, userName,
				password);
	}
	
	/**
	 * Loops every TimeBetweenStatusChecks seconds until it's time to update DYK
	 */
	public void run() {
		boolean dykResetExceptionThrown = false;
		do {
			dykResetExceptionThrown = false;
			try {
				login();
				nextQueue = findNextQueueNumber();
				errorLog = new StringBuilder();
				lastDelId = getLastDelId();
				while (isOn()) {
					checkifLoggedIn();
					log(new Date().toString()); // output the date and time
					// figure out when next update is
					SimpleArticle dykTimePage = readContent(TimeLoc);
					String dykTime = dykTimePage.getText().trim();
					if (dykTime.lastIndexOf("\n") != -1) { // if there are multiple lines, get the last line
						dykTime = dykTime.substring(dykTime.lastIndexOf("\n")).trim();
					}
					GregorianCalendar nextUpdateTime = new GregorianCalendar(BotLocale);
					// first set it to the last update time
					try {
						nextUpdateTime.setTime(APITimestampFormat.parse(dykTime));
					} catch (ParseException e) {
						logError("Time at [[" + TimeLoc + "]] is not formatted correctly");
						postErrors();
						sleep(TimeBetweenStatusChecks * 1000);
						continue;
					}
					// then get the number of seconds between updates, and add it
					int timeBetweenUpdates;
					try {
						timeBetweenUpdates = Integer.parseInt(
								readContent(TimeBetweenUpdatesLoc).getText().trim());
					} catch (Exception e) {
						logError("Time between updates at [[" + TimeBetweenUpdatesLoc + 
								"]] is not formatted correctly");
						postErrors();
						sleep(TimeBetweenStatusChecks * 1000);
						continue;
					}
					// add the correct number of seconds to show the time for the next update
					nextUpdateTime.add(Calendar.SECOND, timeBetweenUpdates);
					// figure out what the current time is
					GregorianCalendar currentTime = new GregorianCalendar(BotLocale);
	
					// update DYK if it's time
					long secondsUntilUpdate = (nextUpdateTime.getTimeInMillis() - 
							currentTime.getTimeInMillis())/1000;
					log("Seconds left until next update: " + secondsUntilUpdate);
					GregorianCalendar nextNextUpdateTime = new GregorianCalendar(BotLocale);
					// calendar for checking if image is protected the whole time it's on the Main Page
					nextNextUpdateTime.setTimeInMillis(nextUpdateTime.getTimeInMillis());
					nextNextUpdateTime.add(Calendar.SECOND, timeBetweenUpdates);
					if (secondsUntilUpdate <= 0) {
						updateDYK(dykTimePage, timeBetweenUpdates, nextNextUpdateTime);
					} else if (secondsUntilUpdate < 7200) {
						checkFormatting(secondsUntilUpdate, nextNextUpdateTime);
					}
					postErrors();
					if (secondsUntilUpdate < TimeBetweenStatusChecks && secondsUntilUpdate > 0) {
						currentTime.setTime(new Date());
						sleep(Math.abs(nextUpdateTime.getTimeInMillis() - currentTime.getTimeInMillis()));
					} else {
						sleep(TimeBetweenStatusChecks * 1000);
					}
				}
			} catch (DYKResetException e) {
				log("Reset exception caught, resetting...");
				dykResetExceptionThrown = true;
			} catch (Exception e) {
				e.printStackTrace(System.out);
				log("Exception occurred; exiting at " + new Date().toString());
			}
		} while (dykResetExceptionThrown);
	}
	
	/**
	 * Checks if all pages are formatted correctly for the next update
	 * If something's wrong, the bot will post to WT:DYK 2 hours before the update
	 * Most of this code is copied from updateDYK()
	 * @param number of seconds until the next update
	 * @param time when the update after next will go live; 
	 * 		aka when the set for the next update will be taken off
	 */
	private void checkFormatting(long secondsUntilUpdate, GregorianCalendar nextNextUpdateTime) {
		// figure out which queue is next
		nextQueue = findNextQueueNumber();
		if (nextQueue == 0)	return;			// couldn't parse
		String wikilinkToQueue = "[[" + QueueRootLoc + nextQueue + "|Queue " + nextQueue + "]]";
		
		// get the wikitext of the queue
		String queueText = removeUnnecessarySpaces(readContent(QueueRootLoc + nextQueue).getText());
		
		// make sure the queue has {{DYKbotdo}}
		if (!queueText.contains("{{DYKbotdo")) {
			logError(wikilinkToQueue + " is not tagged with {{tl|DYKbotdo}}");
			if (secondsUntilUpdate < 7200) {
				// post to WT:DYK if less than two hours left
				try {
					// get the text of the message and update it
					Scanner in = new Scanner(new File("almostLate.txt"));
					StringBuilder errorBuilder = new StringBuilder();
					while (in.hasNext()) {
						errorBuilder.append(in.nextLine()).append("\n");
					}
					in.close();
					String errorMessage = errorBuilder.toString().trim();
					while (errorMessage.contains("queueNum")) {
						errorMessage = errorMessage.replace("queueNum", "" + nextQueue);
					}
					if (errorMessage.contains("hoursLeft")) {
						errorMessage = errorMessage.replace("hoursLeft", "two hours");
					}
					String setIdentifier = APITimestampFormat.format(nextNextUpdateTime.getTime());
					if (errorMessage.contains("uniqueSetIdentifier")) {
						// if Template:Did you know/Next update/Time changes, the 
						// set identifier will also change
						errorMessage = errorMessage.replace("uniqueSetIdentifier", setIdentifier);
					}
					do {
						try {
							SimpleArticle WTDYK = readContent(WTDYKLoc);
							// edit WT:DYK if an alert isn't already posted for this set
							if (!WTDYK.getText().contains(setIdentifier)) {
								WTDYK.addText("\n\n" + errorMessage);
								WTDYK.setEditSummary("DYK is almost late");
								writeContent(WTDYK);
							}
							return;
						} catch (EditConflictException e) {
							log("Edit conflict caught");
							// will try again because of while(true)
						}
					} while (true);
				} catch (DYKResetException e) {
					throw e;
				} catch (Exception e) {
					logError("Error occurred while posting 'dyk is late' message");
				}
			}
			return; // don't continue checking for formatting errors, as the queue may be empty
		}

		// make sure the queue has <!--Hooks--> and <!--HooksEnd-->
		int indexOfHooksinQueue = queueText.indexOf("<!--Hooks-->");
		int indexOfHooksEndinQueue = queueText.indexOf("<!--HooksEnd-->", indexOfHooksinQueue);
		if (indexOfHooksinQueue == -1 || indexOfHooksEndinQueue == -1) {
			logError(wikilinkToQueue + " is missing a <nowiki><!--Hooks--> or <!--HooksEnd--></nowiki>");
			return; // can't find hooks, bail out
		}
		String newHooks = queueText.substring(indexOfHooksinQueue + 12, indexOfHooksEndinQueue);
		
		// make sure image doesn't have |right and is set to 100x100px
		String newHooksLowerCase = newHooks.toLowerCase();
		if (newHooksLowerCase.contains("[[file:") || 
				newHooksLowerCase.contains("[[image:")) { // image file
			int startIndex = Math.max(newHooksLowerCase.lastIndexOf("[[file:") + 7, 
					newHooksLowerCase.lastIndexOf("[[image:") + 8);
			int endIndex = startIndex;
			for (int i=1; newHooks.indexOf("]]", endIndex + 2) != -1; i++) {
				endIndex = newHooks.indexOf("]]", endIndex + 2);
				if (newHooks.substring(startIndex, endIndex).split("\\[\\[").length == i) {
					break;
				}
			}
			String imageWikitext = newHooks.substring(startIndex, endIndex);
			if (imageWikitext.contains("|right")) {
				logError("Warning: File formatting contains |right in " + wikilinkToQueue);
			}
			if (!imageWikitext.contains("100x100px")) {
				logError("Warning: File size is not set to 100x100px in " + wikilinkToQueue);
			}
		}
		
		// make sure all curly braces are matched
		if (queueText.split("\\{\\{").length != queueText.split("\\}\\}").length) {
			logError("Unmatched left <nowiki>(\"{{\") and right (\"}}\")</nowiki> curly braces in " + wikilinkToQueue);
		}
		
		// make sure file is protected
		DYKFile incomingFile = findFile(newHooks);
		if (incomingFile != null) checkIfProtected(incomingFile.getFilename(), nextNextUpdateTime, true);
		
		// fetch T:DYK
		String dykMainText = readContent(TDYKLoc).getText();
		
		// make sure T:DYK has <!--Hooks--> and <!--HooksEnd-->
		int indexOfHooksonTDYK = dykMainText.indexOf("<!--Hooks-->");
		int indexOfHooksEndonTDYK = dykMainText.indexOf("<!--HooksEnd-->", indexOfHooksonTDYK);
		if (indexOfHooksonTDYK == -1 || indexOfHooksEndonTDYK == -1) {
			logError("[[" + TDYKLoc + "]] is missing a <nowiki><!--Hooks--> or <!--HooksEnd--></nowiki>");
		}
	}
	
	/**
	 * Updates DYK
	 * @param the page indicating the time of the last update
	 * @param time when the update after next will go live; 
	 * 		aka when the set for the next update will be taken off
	 */
	private void updateDYK(SimpleArticle dykTimePage, final int timeBetweenUpdates,
			GregorianCalendar nextNextUpdateTime) {
		// figure out which queue to update from
		nextQueue = findNextQueueNumber();
		if (nextQueue == 0)	return;	// couldn't parse
		
		// get the wikitext of the queue
		SimpleArticle queue = new SimpleArticle(readContent(QueueRootLoc + nextQueue));
		String queueText = removeUnnecessarySpaces(queue.getText());
		
		// make sure the queue has {{DYKbotdo}}
		int dykbotdoIndex = queueText.indexOf("{{DYKbotdo");
		String wikilinkToQueue = "[[" + QueueRootLoc + nextQueue + "|Queue " + nextQueue + "]]";
		if (dykbotdoIndex == -1) {
			logError(wikilinkToQueue + " is not tagged with {{tl|DYKbotdo}}");
			return;
		}
		String dykbotdo = queueText.substring(dykbotdoIndex, queueText.indexOf("\n", dykbotdoIndex)).trim();
		// make sure the queue has <!--Hooks--> and <!--HooksEnd-->, then find hooks
		int indexOfHooksinQueue = queueText.indexOf("<!--Hooks-->");
		int indexOfHooksEndinQueue = queueText.indexOf("<!--HooksEnd-->", indexOfHooksinQueue);
		if (indexOfHooksinQueue == -1 || indexOfHooksEndinQueue == -1) {
			logError(wikilinkToQueue + " is missing a <nowiki><!--Hooks--> or <!--HooksEnd--></nowiki>");
			return;
		}
		queueText = checkIfEachHookOnNewLine(queueText, indexOfHooksinQueue, indexOfHooksEndinQueue);
		indexOfHooksEndinQueue = queueText.indexOf("<!--HooksEnd-->", indexOfHooksinQueue); // this may have changed from above line
		String newHooks = queueText.substring(indexOfHooksinQueue + 12, indexOfHooksEndinQueue);
		// make sure all curly braces are matched
		if (queueText.split("\\{\\{").length != queueText.split("\\}\\}").length) {
			logError("Unmatched left <nowiki>(\"{{\") and right (\"}}\")</nowiki> curly braces in " + wikilinkToQueue);
			return;
		}
		// make sure the image/file is protected
		DYKFile incomingFile = findFile(newHooks);
		if (incomingFile != null && !checkIfProtected(incomingFile.getFilename(), nextNextUpdateTime, true)) {
			return;
		}
		
		// fetch T:DYK
		SimpleArticle dykMain = new SimpleArticle(readContent(TDYKLoc));
		String dykMainText = dykMain.getText();
		
		// make sure T:DYK has <!--Hooks--> and <!--HooksEnd-->, then find hooks
		int indexOfHooksonTDYK = dykMainText.indexOf("<!--Hooks-->");
		int indexOfHooksEndonTDYK = dykMainText.indexOf("<!--HooksEnd-->", indexOfHooksonTDYK);
		if (indexOfHooksonTDYK == -1 || indexOfHooksEndonTDYK == -1) {
			logError("[[" + TDYKLoc + "]] is missing a <nowiki><!--Hooks--> or <!--HooksEnd--></nowiki>");
			return;
		}
		
		// replace old hooks with new hooks
		String oldHooks = dykMainText.substring(indexOfHooksonTDYK + 12, indexOfHooksEndonTDYK).trim();
		dykMainText = dykMainText.substring(0, indexOfHooksonTDYK + 12) + newHooks +
				dykMainText.substring(indexOfHooksEndonTDYK, dykMainText.length());
		GregorianCalendar time = new GregorianCalendar(BotLocale);
		
		// edit T:DYK
		dykMain.setText(dykMainText);
		dykMain.setEditSummary("Bot automatically updating DYK template with hooks copied from " + 
				"[[" + QueueRootLoc + nextQueue + "|" + "queue " + nextQueue + "]]");
		try {
			dykMain.setEditTimestamp(OverrideEditConflicts);
		} catch (ParseException e) {}	// impossible
		writeContent(dykMain);
		
		// purge the main page
		purge("Main Page", true);
		
		// reset DYK time
		String dykTimePageText = dykTimePage.getText();
		String dykTime = dykTimePageText.trim();
		if (dykTime.lastIndexOf("\n") != -1) { // if there are multiple lines, get the last line
			dykTime = dykTime.substring(dykTime.lastIndexOf("\n")).trim();
		}
		String timeEditSummary = "Resetting the clock";
		GregorianCalendar writeTime = new GregorianCalendar(BotLocale);
		writeTime.setTimeInMillis(time.getTimeInMillis());
		writeTime.set(Calendar.SECOND, 0);
		writeTime.set(Calendar.MILLISECOND, 0);
		int drift = calculateDrift(writeTime, timeBetweenUpdates);
		if (drift != 0) {
			writeTime.add(Calendar.MINUTE, drift);
			timeEditSummary += ", with drift";
		}
		String wikiTimeString = APITimestampFormat.format(new Date(writeTime.getTimeInMillis()));
		dykTimePage.setText(dykTimePageText.substring(0, dykTimePageText.indexOf(dykTime)) + 
				wikiTimeString);
		dykTimePage.setEditSummary(timeEditSummary);
		try {
			dykTimePage.setEditTimestamp(OverrideEditConflicts);
		} catch (ParseException e) {}	// impossible
		writeContent(dykTimePage);
		
		// find old file and associated tags
		DYKFile file = findFile(oldHooks);
		checkFileTags(file);
		
		// archive old hooks
		archive(oldHooks, time, file);
		
		// remove any commented-out wikitext from queueText
		while (queueText.indexOf("<!--") != -1) {
			int endCommentIndex = queueText.indexOf("-->") + 3;
			if (endCommentIndex == -1) {
				endCommentIndex = queueText.length();
			}
			queueText = queueText.substring(0, queueText.indexOf("<!--")) + 
					queueText.substring(endCommentIndex);
		}
		
		// parse the credits
		LinkedList<DYKCredit> credits = parseCredits(queueText, newHooks);
		
		// tag article talk pages
		tagArticles(time, credits);
		
		// tag user talk pages
		giveUserCredits(credits, dykbotdo);
		
		// clear queue
		queue.setText(ClearTemplate);
		queue.setEditSummary("Update is done, removing the hooks");
		try {
			queue.setEditTimestamp(OverrideEditConflicts);
		} catch (ParseException e) {}	// impossible
		writeContent(queue);
		
		// update next queue number
		int updatedNextQueue = (nextQueue % NumQueues) + 1;
		SimpleArticle nextQueuePage = new SimpleArticle("" + updatedNextQueue, NextUpdateQueueLoc);
		nextQueuePage.setEditSummary("Next queue is [[" + QueueRootLoc + updatedNextQueue + "|" + 
				"queue " + updatedNextQueue + "]]");
		try {
			nextQueuePage.setEditTimestamp(OverrideEditConflicts);
		} catch (ParseException e) {}	// impossible
		writeContent(nextQueuePage);
		
		// delete/unprotect and tag outgoing file
		//boolean fileDeleted = deleteFile(file); // Jan 2017 - delete and unprotect don't work,
		//if (!fileDeleted) unprotectFile(file);  // likely due to authentication issues
		tagFile(file, time);

		nextQueue = updatedNextQueue;
	}
	
	/**
	 * Reads the next queue number from NextUpdateQueueLoc
	 * @return next queue number, or 0 if there was an error parsing
	 */
	private int findNextQueueNumber() {
		SimpleArticle nextQueuePage = new SimpleArticle(readContent(NextUpdateQueueLoc));
		int nextQueue = 0;
		try {
			nextQueue = Integer.parseInt(nextQueuePage.getText());
		} catch (NumberFormatException e) {
			logError("Could not parse [[" + NextUpdateQueueLoc + "]]; check if it's a number 1-" + NumQueues);
		}
		return nextQueue;
	}
	
	/**
	 * 
	 * @param updateTime the calendar corresponding to the time of the current update
	 * @param timeBetweenUpdates in seconds
	 * @return drift in minutes; negative is advance, positive is delay
	 */
	private int calculateDrift(GregorianCalendar updateTime, final int timeBetweenUpdates) {
		final long millisecondsPerMinute = 60 * 1000;
		final long millisecondsPerDay = 24 * 60 * millisecondsPerMinute; //86400000
		long leastDifferenceFrom0000 = Long.MAX_VALUE;
		HashSet<Long> differences = new HashSet<Long>();
		GregorianCalendar updateIter = new GregorianCalendar(BotLocale);
		updateIter.setTimeInMillis(updateTime.getTimeInMillis());
		while (true) {
			long currentDifferenceFrom0000 = updateIter.getTimeInMillis()%millisecondsPerDay;
			if (currentDifferenceFrom0000 > millisecondsPerDay/2) {
				currentDifferenceFrom0000 = -(millisecondsPerDay - currentDifferenceFrom0000);
			}
			if (Math.abs(leastDifferenceFrom0000) > Math.abs(currentDifferenceFrom0000)) {
				leastDifferenceFrom0000 = currentDifferenceFrom0000;
			}
			if (differences.contains(currentDifferenceFrom0000) || differences.size() >= 24) {
				break;
			}
			differences.add(currentDifferenceFrom0000);
			updateIter.add(Calendar.SECOND, timeBetweenUpdates);
		}
		
		String driftText = readContent(DriftLoc).getText();
		int maxAdvance = 0;
		int maxDelay = 0;
		try {
			String[] driftLines = driftText.split("\n");
			maxAdvance = Integer.parseInt(driftLines[0].split(":")[1].trim()); // in minutes
			maxDelay = Integer.parseInt(driftLines[1].split(":")[1].trim());; //in minutes
		} catch (Exception e) {
			log("Couldn't parse drift");
			return 0;
		}
		
		if (leastDifferenceFrom0000 > 0) {
			return -Math.min(maxAdvance, (int)(leastDifferenceFrom0000/millisecondsPerMinute));
		} else if (leastDifferenceFrom0000 < 0) {
			return Math.min(maxDelay, (int)(-leastDifferenceFrom0000/millisecondsPerMinute));			
		} else {
			return 0;
		}
	}
	
	/**
	 * Archives the latest set to ArchiveLoc
	 * @param the hooks to be archived
	 * @param a Calendar object containing the time that DYK was updated
	 */
	private void archive(String hooks, Calendar updateTime, DYKFile file) {
		do {
			try {
				if (file != null) {
					// if the file was cropped, point to the original file in the archives
					String originalFile = file.getCroppedFrom();
					if (originalFile != null) {
						int fileStartIndex = hooks.indexOf(file.getFilename());
						hooks = hooks.substring(0, fileStartIndex) + originalFile + hooks.substring(fileStartIndex + file.getFilename().length());
					}
				}
				
				SimpleArticle archivePage = new SimpleArticle(readContent(ArchiveLoc));
				String timeHeading = new SimpleDateFormat("'*'''''''''''HH:mm, d MMMM yyyy '(UTC)'''''''''''", BotLocale).
						format(updateTime.getTime());
				String sectionHeading = new SimpleDateFormat("'==='d MMMM yyyy'==='", BotLocale).format(updateTime.getTime());
				String archiveText = archivePage.getText();
				// check if there is a section heading already for today
				int thisDateIndex = archiveText.indexOf(sectionHeading);
				if (thisDateIndex == -1) { // if there isn't, create a new section heading and add the new set
					int firstSectionIndex = archiveText.indexOf("===", archiveText.indexOf("<!--BOTPOINTER-->"));
					if (firstSectionIndex == -1) { // if no archive sections exist (ie at the very beginning of a month)
						firstSectionIndex = archiveText.indexOf("\n", archiveText.indexOf("<!--BOTPOINTER-->")) + 1;
					}
					archiveText = archiveText.substring(0, firstSectionIndex) + 
							sectionHeading + "\n" + timeHeading + "\n" + hooks + "\n\n" + 
							archiveText.substring(firstSectionIndex);
				} else { // otherwise add the set under the section heading for today
					int writeIndex = thisDateIndex + sectionHeading.length();
					archiveText = archiveText.substring(0, writeIndex) + "\n" + timeHeading + "\n" +
							hooks + "\n" + archiveText.substring(writeIndex);
				}
				archivePage.setText(archiveText);
				archivePage.setEditSummary("Archiving latest set");
				writeContent(archivePage);
				return;
			} catch (EditConflictException e) {
				log("Edit conflict caught");
				// will try again because of while(true)
			} catch (DYKResetException e) {
				throw e;
			} catch (Exception e) {
				logError("Error occurred while archiving");
				return;
			}
		} while(true);
	}
	
	/**
	 * Parses the credits; associates each article title with the user to be credited
	 * and the hook
	 * @param the wikitext of the queue
	 * @param the hooks in the queue
	 * @return parsed credits
	 */
	private LinkedList<DYKCredit> parseCredits(String queueText, String hooks) {
		LinkedList<DYKCredit> credits = new LinkedList<DYKCredit>();
		// unescape all html encoding in the hooks; for example, "M&amp;M" will become "M&M"
		hooks = StringEscapeUtils.unescapeHtml(hooks);
		// find all credit templates and parse article titles, users, and hooks
		int dykMakeIndex = queueText.indexOf("{{DYKmake");
		int dykNomIndex = queueText.indexOf("{{DYKnom");
		while (dykMakeIndex != -1 || dykNomIndex != -1) {
			int nextCreditIndex;
			if (dykMakeIndex == -1) {
				nextCreditIndex = dykNomIndex;
			} else if (dykNomIndex == -1) {
				nextCreditIndex = dykMakeIndex;
			} else {
				nextCreditIndex = Math.min(dykMakeIndex, dykNomIndex);
			}
			
			int closeTemplateIndex = queueText.indexOf("}}", nextCreditIndex);
			int closeTemplatesEncountered = 1;
			while (queueText.substring(nextCreditIndex + 2, closeTemplateIndex)
					.split("\\{\\{").length > closeTemplatesEncountered) {
				closeTemplateIndex = queueText.indexOf("}}", closeTemplateIndex+2);
				++closeTemplatesEncountered;
			}
			String creditTemplate = queueText.substring(nextCreditIndex, closeTemplateIndex + 2);
			boolean dykMake = (nextCreditIndex == dykMakeIndex);

			// these next two lines are the "increment" part of the while loop
			dykMakeIndex = queueText.indexOf("{{DYKmake", nextCreditIndex + 1);
			dykNomIndex = queueText.indexOf("{{DYKnom", nextCreditIndex + 1);
			// end increment
			
			LinkedList<String> creditTemplatePieces = new LinkedList<String>(Arrays.asList(
					creditTemplate.substring(2, creditTemplate.length() - 2).split("\\|")));
			int numContinuing = 0;
			for (int i=0; i < creditTemplatePieces.size(); ) {
				boolean continuation = numContinuing > 0;
				int numOpenTemplates = creditTemplatePieces.get(i).split("\\{\\{").length - 1;
				int numCloseTemplates = creditTemplatePieces.get(i).split("\\}\\}").length - 1;
				
				numContinuing = numContinuing + numOpenTemplates - numCloseTemplates;
				if (continuation) {
					creditTemplatePieces.set(i - 1, creditTemplatePieces.get(i - 1) + "|" + creditTemplatePieces.get(i));
					creditTemplatePieces.remove(i);
				} else {
					++i;
				}
			}
			
			String title = null;
			String user = null;
			String subpage = null;
			boolean firstPiece = true;
			boolean invalidCreditTemplate = false;
			int unnamedParamsSeen = 0;
			for (String piece : creditTemplatePieces) {
				String trimmedPiece = piece.trim();
				if (firstPiece) {
					if (!(trimmedPiece.equals("DYKmake") || trimmedPiece.equals("DYKnom"))) {
						invalidCreditTemplate = true;
						break;
					}
					firstPiece = false;
				} else {
					int firstEqualsIndex = piece.indexOf('=');
					if (firstEqualsIndex != -1) {
						String paramName = piece.substring(0, firstEqualsIndex).trim();
						String paramValue = piece.substring(firstEqualsIndex + 1).trim();
						paramValue = StringEscapeUtils.unescapeHtml(paramValue);
						
						if (paramName.equals("1")) title = paramValue;
						else if (paramName.equals("2")) user = paramValue;
						else if (paramName.equals("subpage")) subpage = paramValue;
						else {
							logError("Invalid credit template: <nowiki>" + creditTemplate + "</nowiki>");
							invalidCreditTemplate = true;
							break;
						}
					} else {
						trimmedPiece = StringEscapeUtils.unescapeHtml(trimmedPiece);
						if (unnamedParamsSeen == 0) { // first unnamed param is title
							title = trimmedPiece;
						} else if (unnamedParamsSeen == 1) { // second is user
							user = trimmedPiece;
						} else {
							logError("Invalid credit template: <nowiki>" + creditTemplate + "</nowiki>");
							invalidCreditTemplate = true;
							break;							
						}
						++unnamedParamsSeen;
					}
				}
			}
			if (title == null || user == null) {
				logError("Invalid credit template: <nowiki>" + creditTemplate + "</nowiki>");
				invalidCreditTemplate = true;				
			}
			if (invalidCreditTemplate) continue;
			
			// check for common formatting errors
			if (title.startsWith("[[")) {
				title = title.substring(2);
			}
			if (title.endsWith("]]")) {
				title = title.substring(0, title.length() - 2);
			}
			if (title.equals("Example") || title.isEmpty()) {
				continue;
			}
			
			boolean errorInArticleTitle = false;
			String hook = null;
 
			// make sure the title corresponds to a real article
			title = title.substring(0, 1).toUpperCase() + title.substring(1); // capitalize first letter
			SimpleArticle article = new SimpleArticle(readContent(title));
			if (article.getText().isEmpty()) { // if the article's been deleted, or otherwise nonexistent
				logError("Article [[" + title + "]] does not exist");
				errorInArticleTitle = true;
			} else {
				hook = findHook(hooks, title);
			}
			if (!errorInArticleTitle) {
				String redirectTo = checkForPageRedirect(article.getText());
				if (redirectTo != null) { 
					article = new SimpleArticle(readContent(redirectTo));
					if (article.getText().isEmpty()) {
						logError("Article [[" + title + "]] is a redirect to a deleted article");
						errorInArticleTitle = true;
					} else if (hook == null) { // if there was no matching hook before, try again
						hook = findHook(hooks, article.getLabel());
					}
				}
			}
			if (!errorInArticleTitle && hook == null) {
				// if we couldn't find the hook before, let's try other options
				// check for redirects to the given page
				LinkedList<String> otherPossibleTitles = findRedirectsToPage(article.getLabel(), 50);
				// check for odd characters (like &nbsp;)
				String normalizedTitle = normalizeTitle(article.getLabel());
				if (!article.getLabel().equals(normalizedTitle)) {
					otherPossibleTitles.add(normalizedTitle);
				}
				for (String possibility : otherPossibleTitles) {
					hook = findHook(hooks, possibility);
					if (hook != null)	break;
				}
				if (hook == null) {
					hook = findHook(hooks.replaceAll(StringEscapeUtils.unescapeHtml("&nbsp;"), " "),
							article.getLabel());
				}
			}
			if (!errorInArticleTitle && hook == null) {
				logError("Couldn't find hook for [[" + title + "]]");
			}
			
			if (user.contains("}}")) user = expandTemplates(user);
			String userTalkPage = validateUserTalkPage(user); // make sure this is a valid user talk page
			
			credits.add(new DYKCredit(article.getLabel(), userTalkPage, hook, errorInArticleTitle, dykMake, subpage));
		}
		return credits;
	}
	
	/**
	 * Finds the hook of the title given in the hooks given
	 * @param hooks in the set
	 * @param title of the article
	 * @return the article's hook, or null if none found
	 */
	private String findHook(String hooks, String title) {
		String hook = null;
		// convert to lower case and underscores to spaces
		String normalizedTitle = title.replaceAll("_", " ").toLowerCase();
		int titleIndex = hooks.toLowerCase().indexOf(normalizedTitle);
		while (titleIndex != -1 && (hook == null || hook.contains("px|") || 
				hook.contains("100x100px") || hook.toLowerCase().contains("{{dyk listen")
				|| hook.toLowerCase().contains("{{main page image"))) { 
			// "px" parts are in case the image caption or filename has the title
			int startOfHook = hooks.lastIndexOf("\n", titleIndex);
			if (startOfHook == -1) startOfHook = 0;
			int endOfHook = hooks.indexOf("\n", titleIndex);
			if (endOfHook == -1) endOfHook = hooks.length();
			hook = hooks.substring(startOfHook, endOfHook).trim();
			titleIndex = hooks.toLowerCase().indexOf(normalizedTitle, 
					titleIndex + normalizedTitle.length());
		}
		if (hook == null || hook.contains("px|") || hook.contains("100x100px")
				|| hook.toLowerCase().contains("{{dyk listen") 
				|| hook.toLowerCase().contains("{{main page image")) {
			hook = findShipHook(hooks, title);
			if (hook == null)	return null;
		}
		// hook formatting
		while (hook.substring(hook.length() - 4).equalsIgnoreCase("<br>")) {
			// http://en.wikipedia.org/w/index.php?title=Template:Did_you_know&oldid=2521104
			hook = hook.substring(0, hook.length() - 4).trim();
		}
		if (hook.substring(hook.length() - 5).equalsIgnoreCase("</li>")) {
			// http://en.wikipedia.org/w/index.php?title=Template:Did_you_know&oldid=9218861
			hook = hook.substring(0, hook.length() - 5).trim();
		}
		if (hook.substring(0, 4).equalsIgnoreCase("<li>")) {
			hook = hook.substring(4, hook.length()).trim();
		}
		if (hook.charAt(0) == '*') {
			hook = hook.substring(1);
		}
		if (hook.substring(0, 7).equals("{{*mp}}")) {
			hook = hook.substring(7);
		}
		if (hook.contains("{{*mp}}")) {
			log("Hook for [[" + title + "]] has an extra {{*mp}}; hook mashup?");
		}
		return hook;
	}
	
	/**
	 * Finds the hook of the title given if a ship template is used (like {{SS}})
	 * @param hooks in the set
	 * @param title of the article
	 * @return the article's hook, or null if none found
	 */
	private String findShipHook(String hooks, String title) {
		int i = 3;
		// figure out which template matches the title
		for (; i < shipTemplates.length; ++i) {
			if (title.toLowerCase().startsWith(shipTemplates[i].toLowerCase())) break;
		}
		if (i == shipTemplates.length) {
			if (title.toLowerCase().contains(" class ") && hooks.toLowerCase().contains("{{sclass")) {
				// looks there's a possible match with {{sclass}} or {{sclass2}}
				i = 1;
			} else if (title.toLowerCase().startsWith("japanese submarine") && hooks.toLowerCase().contains("{{jsub")) {
				// match with {{Jsub}}
				i = 2;
			} else if (hooks.toLowerCase().contains("{{ship")) {
				// if none of the specific templates match, maybe {{ship}} will
				i = 0;
			} else {
				return null;
			}
		}
		for (int templateIndex = hooks.toLowerCase().indexOf("{{" + shipTemplates[i].toLowerCase()); 
				templateIndex != -1; 
				templateIndex = hooks.toLowerCase().indexOf("{{" + shipTemplates[i].toLowerCase(), templateIndex + 2)) {
			// find the ship template
			int endIndex = templateIndex;
			for (int j=2; hooks.indexOf("}}", endIndex + 2) != -1; j++) {
				endIndex = hooks.indexOf("}}", endIndex + 2);
				if (hooks.substring(templateIndex, endIndex).split("\\{\\{").length == j) {
					break;
				}
			}
			String template = hooks.substring(templateIndex + 2, endIndex);
			// parse the ship template and assemble it into a title
			String[] cutup = template.split("\\|");
			int base = 0;
			if (i < 2) base = 1;
			String titleFromTemplate;
			if (i != 1) {
				titleFromTemplate = cutup[base].trim() + " " + cutup[base+1].trim();
				if (cutup.length >= base+3 && !cutup[base+2].isEmpty() && 
						!cutup[base+2].trim().equals("3=2")) {
					titleFromTemplate += " (" + cutup[base+2].trim() + ")";
				}
			} else { // {{sclass}} and {{sclass2}}
				titleFromTemplate = cutup[base].trim() + " class " + cutup[base+1].trim();
			}
			if (i == 2) {
				titleFromTemplate = "Japanese submarine " + titleFromTemplate.substring(5);
			}
			// if the title from the credits and the title assembled from the template
			// match, we've found the correct hook
			if (titleFromTemplate.equalsIgnoreCase(title)) {
				return findHook(hooks, template);
			}
		}
		return null;
	}

	/**
	 * Tag article talk pages
	 * If {{ArticleHistory}} exists on the talk page, the bot will add the DYK credit there instead of 
	 * adding a new {{DYK talk}}
	 * @param a Calendar containing the time that DYK was last updated
	 * @param the credits (contains article title, username, and hook)
	 */
	private void tagArticles(Calendar time, LinkedList<DYKCredit> credits) {
		// make the start of a DYK talk tag without the hook
		String tag = new SimpleDateFormat("'{{DYK talk|'d MMMM'|'yyyy", BotLocale).format(time.getTime());
		String editSummaryTimestamp = new SimpleDateFormat("d MMMM yyyy", BotLocale).format(time.getTime());
		HashSet<String> taggedArticles = new HashSet<String>();

		// tag articles
		for (DYKCredit credit: credits) {
			if (credit.errorInArticleTitle)	continue;
			if (taggedArticles.contains(credit.articleTitle)) continue;
			boolean editConflicted = false;
			do {
				editConflicted = false;
				try {
					// build up the tag
					String tagWithHook = tag;
					if (credit.hook != null) {
						tagWithHook += "|entry=" + credit.hook;
					}
					if (credit.nompage != null) {
						tagWithHook += "|nompage=" + credit.nompage;
					}
					tagWithHook += "}}";

					// get the talk page
					SimpleArticle talkPage = new SimpleArticle(readContent("Talk:" + credit.articleTitle));
					String talkContent = talkPage.getText();
					if (talkContent.isEmpty())	talkPage.setEditTimestamp(OverrideEditConflicts);
					String talkContentLowerCase = talkContent.toLowerCase();
					if (talkContentLowerCase.contains("{{articlehistory")) { 
						// if it has {{ArticleHistory}}
						int articleHistoryIndex = talkContentLowerCase.indexOf("{{articlehistory");
						String articleHistory = talkContent.substring(articleHistoryIndex,
								talkContent.indexOf("}}", articleHistoryIndex));
						String articleHistoryNew = new String(articleHistory);								
						int currentStatusIndex = articleHistoryNew.indexOf("currentstatus");
						while (currentStatusIndex != -1 &&
								!articleHistoryNew.substring(0, currentStatusIndex).trim().endsWith("|")) {
							currentStatusIndex = articleHistoryNew.indexOf("currentstatus", currentStatusIndex + 13);
						}
						String currentStatusString = "|currentstatus"; //default
						if (currentStatusIndex != -1) {
							int temp = currentStatusIndex;
							currentStatusIndex = articleHistoryNew.lastIndexOf("|", currentStatusIndex);
							currentStatusString = articleHistoryNew.substring(currentStatusIndex, temp + 13);
						}
	
						int dykdateIndex = articleHistoryNew.indexOf("dykdate");
						while (dykdateIndex != -1 && 
								!articleHistoryNew.substring(0, dykdateIndex).trim().endsWith("|")) {
							dykdateIndex = articleHistoryNew.indexOf("dykdate", dykdateIndex + 7);
						}
						int pipeAfterIndex = articleHistoryNew.indexOf("|", dykdateIndex + 7);
						if (pipeAfterIndex == -1) {
							pipeAfterIndex = articleHistoryNew.length();
						}
						boolean blankDYKdate = articleHistoryNew.substring(dykdateIndex + 7, 
								pipeAfterIndex).trim().equals("=");
						String dykDateParam = editSummaryTimestamp;
						String dykEntryAndNom = "";
						if (credit.hook != null) dykEntryAndNom = "\n|dykentry=" + credit.hook;
						if (credit.nompage != null) dykEntryAndNom = "\n|dyknom=" + credit.nompage;
	
						if (currentStatusIndex == -1 && dykdateIndex == -1) {
							// if there's no currentStatus or dykdate
							logError("Could not tag [[" + credit.articleTitle +
									"]] by {{tl|ArticleHistory}}; please tag article manually");
						} else if (dykdateIndex == -1 || blankDYKdate) {
							if (dykdateIndex == -1) {
								articleHistoryNew = articleHistoryNew.replace(currentStatusString, "|dykdate=" +
										dykDateParam + dykEntryAndNom + "\n" + currentStatusString);
							} else if (blankDYKdate) {
								String dykdateOld = articleHistoryNew.substring(dykdateIndex, 
										articleHistoryNew.indexOf("=", dykdateIndex) + 1);
								
								// remove old |dykentry if it exists
								int dykentryIndex = articleHistoryNew.indexOf("dykentry", dykdateIndex);
								if (dykentryIndex != -1) {
									int onePastPipeAfterEntryIndex = articleHistoryNew.indexOf("|", dykentryIndex);
									if (onePastPipeAfterEntryIndex == -1) {
										onePastPipeAfterEntryIndex = articleHistoryNew.length();
									} else {
										++onePastPipeAfterEntryIndex;
									}
									articleHistoryNew = articleHistoryNew.substring(0, dykentryIndex) + 
											articleHistoryNew.substring(onePastPipeAfterEntryIndex);
								}
								
								// remove old |dyknom if it exists
								int dyknomIndex = articleHistoryNew.indexOf("dyknom", dykdateIndex);
								if (dyknomIndex != -1) {
									int onePastPipeAfterNomIndex = articleHistoryNew.indexOf("|", dyknomIndex);
									if (onePastPipeAfterNomIndex == -1) {
										onePastPipeAfterNomIndex = articleHistoryNew.length();
									} else {
										++onePastPipeAfterNomIndex;
									}
									articleHistoryNew = articleHistoryNew.substring(0, dyknomIndex) + 
											articleHistoryNew.substring(onePastPipeAfterNomIndex);
								}
								
								articleHistoryNew = articleHistoryNew.replace(dykdateOld, dykdateOld +
										dykDateParam + dykEntryAndNom);
							}
							talkContent = talkContent.replace(articleHistory, articleHistoryNew);
							talkPage.setText(talkContent.trim());
							talkPage.setEditSummary("Article appeared on [[WP:Did you know|DYK]] on " +
									editSummaryTimestamp + ", adding to " +
									"{{[[Template:ArticleHistory|ArticleHistory]]}}");
							writeContent(talkPage);
						} else {
							log("ArticleHistory up to date for article " + credit.articleTitle);
						}
					} else { // if it doesn't have ArticleHistory, add a new tag
						int indexOfFirstSection = talkContent.indexOf("==");
						if (indexOfFirstSection == -1) indexOfFirstSection = talkContent.length();
						String zeroSection = talkContent.substring(0, indexOfFirstSection);
						String theRest = talkContent.substring(indexOfFirstSection);
						int lastTemplateIndex = findLastTemplateIndex(zeroSection);
						String zeroSectionA = zeroSection.substring(0, lastTemplateIndex);
						String zeroSectionB = zeroSection.substring(lastTemplateIndex);
						talkContent = zeroSectionA.trim() + "\n" + tagWithHook + "\n\n" + zeroSectionB + theRest;
						talkPage.setText(talkContent.trim());
						talkPage.setEditSummary("Article appeared on [[WP:Did you know|DYK]] on " +
								editSummaryTimestamp + ", adding {{[[Template:DYK talk|DYK talk]]}}");
						writeContent(talkPage);
					}
					taggedArticles.add(credit.articleTitle);
				} catch (EditConflictException e) {
					log("Edit conflict caught");
					editConflicted = true;
				} catch (DYKResetException e) {
					throw e;
				} catch (Exception e) {
					logError("Error occurred when attempting to tag [[" + credit.articleTitle + "]]");
				}
			} while (editConflicted);
		}
	}
	
	/**
	 * Tag user talk pages
	 * @param the credits (contains article title, username, and hook)
	 * @param the {{DYKbotdo}} template
	 */
	private void giveUserCredits(LinkedList<DYKCredit> credits, String dykbotdo) {
		for (DYKCredit credit : credits) {
			if (credit.userTalkPage == null) continue;
			boolean editConflicted = false;
			do {
				editConflicted = false;
				try {				
					// tag user talk page
					SimpleArticle userTalk = readContent(credit.userTalkPage);
					if (userTalk.getText().isEmpty())	userTalk.setEditTimestamp(OverrideEditConflicts);
					userTalk.addText("\n\n==DYK for " + credit.articleTitle + "==");
					String creditTemplate;
					if (credit.dykMake) { // if it's {{DYKmake}}
						creditTemplate = "\n{{subst:Template:DYKmake/DYKmakecredit";
					} else { // if it's {{DYKNom}}
						creditTemplate = "\n{{subst:Template:DYKnom/DYKnomcredit";
					}
					creditTemplate += " |article=" + credit.articleTitle;
					if (credit.hook != null) creditTemplate += " |hook=" + credit.hook;
					if (credit.nompage != null) creditTemplate += " |nompage=" + credit.nompage;
					creditTemplate += " |optional= }} ";
					userTalk.addText(creditTemplate);
					int dykBotDoPipeIndex = dykbotdo.indexOf("|");
					if (dykBotDoPipeIndex == -1) {
						userTalk.addText("~~~~");
					} else {
						userTalk.addText(dykbotdo.substring(dykBotDoPipeIndex + 1, 
								dykbotdo.lastIndexOf("}}")));
						userTalk.addText(" ~~~~~");
					}
					// form edit summary
					String adminUsername = findUserLink(dykbotdo);
					String editSummary = "Giving DYK credit for [[" + credit.articleTitle + "]]";
					if (adminUsername != null) {
						editSummary += " on behalf of [[User:" + adminUsername + "|" + 
								adminUsername + "]]";
					}
					userTalk.setEditSummary(editSummary);
					// edit talk page
					writeContent(userTalk);
				} catch (EditConflictException e) {
					editConflicted = true;
					log("Edit conflict caught");
				} catch (DYKResetException e) {
					throw e;
				} catch (Exception e) {
					logError("Error occurred while distributing user credits");
				}
			} while (editConflicted);
		}
	}
	
	/**
	 * Checks if a user exists or has been renamed
	 * If the user talk page redirects to another user talk, this method will return the target username
	 * Otherwise, if the username is not registered and not an IP address, null is returned
	 * @param username to check
	 * @return a valid username, or null if none
	 */
	private String validateUserTalkPage(String username) {
		// example credits aren't valid
		if (username.equals("Editor") || username.equals("Nominator") || username.isEmpty()) {
			return null;
		}
		String userTalkPage = "User talk:" + username;
		
		// check if the talk page redirects to another page (if the user's been renamed)
		SimpleArticle talkPage = new SimpleArticle(readContent(userTalkPage));
		String redirectTo = checkForPageRedirect(talkPage.getText());
		if (redirectTo != null) {
			int userTalkIndex = redirectTo.toLowerCase().indexOf("user talk:");
			if (userTalkIndex != -1) {
				userTalkPage = redirectTo.substring(userTalkIndex);
				username = userTalkPage.substring(10);

				// support redirects to talk page archives
				// for example User talk:Djembayz -> User talk:Djembayz/Archive July 2014
				int slashIndex = username.indexOf("/");
				if (slashIndex != -1) username = username.substring(0, slashIndex);
			}
		}
				
		// check if the username is registered
		String apiURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query&list=users&ususers=" + 
				MediaWiki.encode(username);
		Document doc = fetchUsingSAXBuilder(apiURL);
		Element userInfo = doc.getRootElement().getChild("query", ns).getChild("users", ns).getChild("user", ns);
		if (userInfo.getAttribute("missing") == null && userInfo.getAttribute("invalid") == null
				&& !username.contains("|")) {
			return userTalkPage;
		}
		
		// check if the user made edits (for IP addresses)
		String apiURL2 = BaseEnWikiAPIURL + "api.php?format=xml&action=query&list=usercontribs&uclimit=1&ucprop=ids&ucuser=" + 
				MediaWiki.encode(username);
		Document doc2 = fetchUsingSAXBuilder(apiURL2);
		Element userContribs = doc2.getRootElement().getChild("query").getChild("usercontribs");
		if ((userContribs.getChildren().size() > 0) && !username.contains("|")) {
			return userTalkPage;
		}
		
		// the username isn't registered or technically impossible
		logError("The username '" + username + "' is invalid");
		return null;
	}
	
	/**
	 * Finds the link to the admins userpage in {{DYKbotdo}}
	 * @param the {{DYKbotdo}} tag
	 * @return the admin's username
	 */
	private String findUserLink(String dykbotdo) {
		try {
			if (dykbotdo.contains("User:") || dykbotdo.contains("User talk:")) {
				int userLinkIndex = Math.max(dykbotdo.indexOf("User:"), dykbotdo.indexOf("User talk:"));
				return dykbotdo.substring(dykbotdo.indexOf(":", userLinkIndex) + 1, 
						dykbotdo.indexOf("|", userLinkIndex));
			}
		} catch (Exception e) {
			return null;
		}
		return null;
	}
	
	/**
	 * Finds the DYK sound/video/image from hooks wikitext
	 * If a sound or video file (.ogg) is used without the proper template, 
	 * the bot will assume it's a video; these exceptions should be checked manually
	 */
	private DYKFile findFile(String hooks) {
		String hooksLowerCase = hooks.toLowerCase();
		if (hooksLowerCase.contains("{{dyk listen")) { // sound file
			int startIndex = hooks.indexOf("|", hooksLowerCase.indexOf("{{dyk listen")) + 1;
			int fileEndIndex = hooks.indexOf("|", startIndex);
			String filename = hooks.substring(startIndex, fileEndIndex);
			return new DYKFile(filename, "sound");
		} else if (hooksLowerCase.contains("{{tall image")) {
			int startIndex = hooks.indexOf("|", hooksLowerCase.indexOf("{{tall image")) + 1;
			int fileEndIndex = hooks.indexOf("|", startIndex);
			String filename = hooks.substring(startIndex, fileEndIndex);
			return new DYKFile(filename, "image");			
		} else if (hooksLowerCase.contains("{{main page image")) {
			// test cases:
			// {{main page image|image=Carrot soup.jpg|caption=A cream of carrot soup with bread|width=120x133}}
			// {{main page image |image=Carrot soup.jpg|caption=A cream of carrot soup with bread|width=120x133}}
			// {{main page image | image=Carrot soup.jpg|caption=A cream of carrot soup with bread|width=120x133}}
			// {{main page image | image = Carrot soup.jpg |caption=A cream of carrot soup with bread|width=120x133}}
			// {{main page image|image=image:Carrot soup.jpg|caption=A cream of carrot soup with bread|width=120x133}}
			// {{main page image|File:Carrot soup.jpg}}
			// {{main page image|Carrot soup.jpg}}
			// {{main page image|Carrot soup.jpg|A cream of carrot soup with bread}}
			int fileStartIndex = hooksLowerCase.indexOf("|", hooksLowerCase.indexOf("{{main page image")) + 1;
			int fileEndIndex = Math.min(hooks.indexOf("|", fileStartIndex), hooks.indexOf("}}", fileStartIndex));
			String filename = hooks.substring(fileStartIndex, fileEndIndex).trim();
			int equalsIndex = filename.indexOf('=');
			if (equalsIndex != -1) {
				String paramNameLowerCase = filename.substring(0, equalsIndex).trim();
				if (paramNameLowerCase.equals("image")) {
					filename = filename.substring(equalsIndex + 1).trim();
				}
			}
			int colonIndex = filename.indexOf(':');
			if (colonIndex != -1) {
				String prefixLowerCase = filename.substring(0, colonIndex).toLowerCase().trim();
				if (prefixLowerCase.equals("image") || prefixLowerCase.equals("file")) {
					filename = filename.substring(colonIndex + 1).trim();
				}
			}
			return new DYKFile(filename, "image");
		} else if (hooksLowerCase.contains("[[file:") || 
				hooksLowerCase.contains("[[image:")) { // image file
			int startIndex = Math.max(hooksLowerCase.lastIndexOf("[[file:") + 7, 
					hooksLowerCase.lastIndexOf("[[image:") + 8);
			int midIndex = hooks.indexOf("|", startIndex);
			int endIndex = startIndex;
			for (int i=1; hooks.indexOf("]]", endIndex + 2) != -1; i++) {
				endIndex = hooks.indexOf("]]", endIndex + 2);
				if (hooks.substring(startIndex, endIndex).split("\\[\\[").length == i) {
					break;
				}
			}
			int rollIndex = hooks.lastIndexOf("|", endIndex);
			while (hooks.lastIndexOf("[[", rollIndex) > (startIndex - 7)) {
				rollIndex = hooks.lastIndexOf("|", rollIndex - 1);
			}
			String type = "image";
			String filename = hooks.substring(startIndex, midIndex).trim();
			if (filename.substring(filename.length() - 4).equals(".ogg")) {
				// http://en.wikipedia.org/w/index.php?diff=next&oldid=273311345
				type = "video";
				logError("Check if [[:File:" + filename + "]] is a sound or video file");
			}
			return new DYKFile(filename, type);
		}
		logError("Can't find an image, sound, or video file");
		return null;
	}
	
	/**
	 * Store information about tags on the file, like {{c-uploaded}}
	 * @param the DYKFile to check
	 */
	private void checkFileTags(DYKFile file) {
		if (file == null) return;
		SimpleArticle filePage = new SimpleArticle(readContent("File:" + file.getFilename()));
		String fileText = filePage.getText();
		file.setCuploaded(fileText.contains("{{c-uploaded}}") || fileText.contains("{{C-uploaded}}"));
		
		int mCroppedIndex = Math.max(fileText.indexOf("{{m-cropped"), fileText.indexOf("{{M-cropped"));
		if (mCroppedIndex != -1) {
			String croppedFrom = fileText.substring(fileText.indexOf('|', mCroppedIndex) + 1, 
					fileText.indexOf("}}", mCroppedIndex)).trim();
			if (croppedFrom.toLowerCase().startsWith("file:")) {
				croppedFrom = croppedFrom.substring(5).trim();
			}
			if (croppedFrom.toLowerCase().startsWith("image:")) {
				croppedFrom = croppedFrom.substring(6).trim();				
			}
			file.setCroppedFrom(croppedFrom);
		}
	}
	
	/**
	 * Checks if the file specified is protected either locally or on Commons
	 * The bot will detect both cascade-protection and normal protection
	 * If you pass in a salted file (for example Capture.JPG), the function will return false
	 * @param fileName without "File:" in front, for example "Andrey Alexandrovich Popov.jpg"
	 * @param time when the image will leave the Main Page
	 * @return true if the file is fully protected, false otherwise (but see above note on salting)
	 */
	@SuppressWarnings("unchecked")
	private boolean checkIfProtected(String fileName, GregorianCalendar nextNextUpdateTime,
			boolean logging) {
		String imageInfoURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query" +
				"&prop=imageinfo&iilimit=1&iiprop=&titles=File:" + MediaWiki.encode(fileName);
		Document imageInfo = fetchUsingSAXBuilder(imageInfoURL);
		Element pageInfo = imageInfo.getRootElement().getChild("query", ns).getChild("pages", ns).getChild("page", ns);
		String rootAPIurl;
		if (pageInfo.getAttributeValue("imagerepository").equals("shared")) { // at Commons
			rootAPIurl = BaseCommonsAPIURL + "api.php";
		} else if (pageInfo.getAttributeValue("imagerepository").equals("local")) { // at Enwiki
			rootAPIurl = BaseEnWikiAPIURL + "api.php";			
		} else { // the file doesn't exist; this should never happen
			if (logging) logError("[[:File:" + fileName + "]] does not exist");
			return false;
		}
		String protectionInfoURL = rootAPIurl + "?format=xml&action=query&prop=info" + 
				"&inprop=protection&titles=File:" + MediaWiki.encode(fileName);
		Document protectionInfo = fetchUsingSAXBuilder(protectionInfoURL);
		List<Element> protectionNodes = protectionInfo.getRootElement().getChild("query", ns).getChild("pages", ns)
				.getChild("page", ns).getChild("protection", ns).getChildren("pr", ns);
		if (protectionNodes.isEmpty()) { // isn't protected or (checked above) doesn't exist
			String logMessage = "[[:File:" + fileName + "]] is not protected";
			if (rootAPIurl.contains("commons")) {
				logMessage += "; either 1) Upload the file to en.wiki, or 2) protect the file at Commons";
			}
			if (logging) logError(logMessage);
			return false; 
		}
		boolean notFullyProtected = false;
		boolean protectionExpireEarly = false;
		for (Element protectionNode : protectionNodes) {
			if (!(protectionNode.getAttributeValue("type").equals("edit") &&
					protectionNode.getAttributeValue("level").equals("sysop"))) {
				notFullyProtected = true;
				continue;
			}
			String protectionExpiryTime = protectionNode.getAttributeValue("expiry");
			if (protectionExpiryTime.equals("infinity") || nextNextUpdateTime == null) {
				return true;
			}
			try {
				if (convertWikiTimestamp(protectionExpiryTime).before(nextNextUpdateTime)) {
					protectionExpireEarly = true;
					continue;
				} else {
					return true; // protection doesn't expire early, so we're good
				}
			} catch (ParseException e) {} // impossible
		}
		if (protectionExpireEarly) {
			if (logging) {
				logError("The protection for [[:File:" + fileName + "]] " +
					"will expire while or before it's on the Main Page");
			}
			return false;			
		}
		if (notFullyProtected) {
			if (logging) logError("[[:File:" + fileName + "]] is not fully protected");
			return false;			
		}
		return false; // unreachable code
	}
	
	/**
	 * Checks if the file should be deleted, then deletes it
	 * The file will be deleted if it's a cropped version made just for DYK
	 * Otherwise, the file won't be deleted if:
	 *  1. It doesn't exist at Commons and/or Enwiki under the same filename
	 *  2. It isn't tagged with {{c-uploaded}}
	 *  3. The first revision in the file's history is before the first upload
	 * @param the file to be deleted
	 */
	@SuppressWarnings("unchecked")
	private boolean deleteFile(DYKFile file) {
		if (file == null) return false;
		String filename = file.getFilename();
		try {
			if (file.getCroppedFrom() == null) { // always delete if this is a cropped image
				if (!file.getCuploaded()) {
					// if it's not tagged with c-uploaded on enwiki, don't delete
					return false;
				}
				
				// if it doesn't exist at Commons, don't delete
				MediaWikiBot commonsBot = new MediaWikiBot(BaseCommonsAPIURL);
				if (readContent(commonsBot, "File:" + filename).getText().isEmpty()) {
					logError("[[:File:" + filename + "]] is tagged with c-uploaded but does not exist at Commons");
					return false;
				}
				
				// figure out when the image was uploaded
				int revs = 10;
				String imageInfoURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query" +
						"&iiprop=timestamp&prop=imageinfo&iilimit=" + revs + "&titles=File:" + 
						MediaWiki.encode(filename);
				Document imageInfo = fetchUsingSAXBuilder(imageInfoURL);
				Element pageInfo = imageInfo.getRootElement().getChild("query", ns).getChild("pages", ns).getChild("page", ns);
				if (pageInfo.getAttributeValue("imagerepository").equals("shared")) {
					return false; // no information on enwiki's copy
				}
				List<Element> timestamps = pageInfo.getChild("imageinfo", ns).getChildren("ii", ns);
				if (timestamps.size() == revs) log("Fetching " + revs + "/" + revs + " revisions");
				Calendar uploadTime = convertWikiTimestamp(timestamps.get
						(timestamps.size() - 1).getAttributeValue("timestamp"));
				
				// figure out the date of the first revision
				String revInfoURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query" +
						"&prop=revisions&rvlimit=1&rvdir=newer&rvprop=timestamp" +
						"&titles=File:" + MediaWiki.encode(filename);
				Document revisionInfo = fetchUsingSAXBuilder(revInfoURL);
				Calendar firstRevTime = convertWikiTimestamp(revisionInfo.getRootElement()
						.getChild("query", ns).getChild("pages", ns).getChild("page", ns).getChild("revisions", ns)
						.getChild("rev", ns).getAttributeValue("timestamp"));
				
				if (firstRevTime.before(uploadTime)) {
					// if the first revision was before the upload, don't delete
					return false;
				}
			}
			// otherwise, delete
			String deleteReason = "{{[[Template:c-uploaded|c-uploaded]]}} file off the " +
					"[[T:DYK|DYK]] section of the Main Page";
			deleteContent("File:" + filename, deleteReason);
			return true;
		} catch (DYKResetException e) {
			throw e;
		} catch (Exception e) {
			logError("Error occurred while deleting [[:File:" + filename + "]]");
			return false;
		}
	}
	
	/**
	 * Unprotects a file if:
	 *  1. It exists on English Wikipedia and is fully protected 
	 *  2. The string "Main Page" is in the reason for the most recent protection
	 */
	private void unprotectFile(DYKFile file) {
		if (file == null) return;
		String filename = file.getFilename();	
		SimpleArticle filePage = new SimpleArticle(readContent("File:" + filename));
		if (filePage.getText().isEmpty()) {
			return; // don't continue if the file isn't on enwiki
		}
		if (!checkIfProtected(filename, null, false)) {
			return; // don't continue if the file isn't fully protected
		}
		
		String protectionLogURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query" + 
				"&list=logevents&letype=protect&leprop=parsedcomment" +
				"&letitle=File:" + MediaWiki.encode(filename);
		Document protectionLog = fetchUsingSAXBuilder(protectionLogURL);
		Element protLogItem = protectionLog.getRootElement().getChild("query", ns)
				.getChild("logevents", ns).getChild("item", ns);
		if (protLogItem == null) {
			return; // don't continue if the file wasn't manually protected
		}
		String protReason = protLogItem.getAttributeValue("parsedcomment");
		if (!protReason.contains("Main Page")) {
			return; // don't continue if the file wasn't protected for DYK
		}
		
		unprotectContent("File:" + filename, "File off the [[T:DYK|DYK]] section of the Main Page");
	}

	/**
	 * Checks if the file exists at Commons or English Wikipedia, 
	 * then tags the file on English Wikipedia if it does exist
	 */
	private void tagFile(DYKFile file, Calendar time) {
		if (file == null) return;
		String filename = file.getCroppedFrom(); // tag the original file if the image was cropped
		if (filename == null) filename = file.getFilename();
		do {
			try {
				SimpleArticle filePage = new SimpleArticle(readContent("File:" + filename));
				MediaWikiBot commonsBot = new MediaWikiBot(BaseCommonsAPIURL);
				if (!filePage.getText().isEmpty() || 
						!readContent(commonsBot, "File:" + filename).getText().isEmpty()) {
					if (filePage.getText().contains("{{DYKfile")) {
						log("The file " + filename + " has already been tagged");
						return;
					}
					String fileTag = "{{DYKfile|" +
							new SimpleDateFormat("d MMMM'|'yyyy", BotLocale).format(time.getTime()) + 
							"|type=" + file.getType() + "}}"; //create DYKfile tag
					String fileContent = filePage.getText();
					if (fileContent.isEmpty())	filePage.setEditTimestamp(OverrideEditConflicts);
					int indexOfFirstSection = fileContent.indexOf("==");
					if (indexOfFirstSection == -1) indexOfFirstSection = fileContent.length();
					fileContent = fileContent.substring(0, indexOfFirstSection).trim() + 
							"\n" + fileTag + "\n" + fileContent.substring(indexOfFirstSection).trim();
					filePage.setText(fileContent);
					filePage.setEditSummary("File appeared on [[WP:Did you know|DYK]] on " +
							new SimpleDateFormat("d MMMM yyyy", BotLocale).format(time.getTime()));
					writeContent(filePage);
				} else {
					logError("[[:File:" + filename + "]] does not exist at Commons or English Wikipedia");
				}
				return;
			} catch (EditConflictException e) {
				log("Edit conflict caught");
				// will try again because of while(true)
			} catch (DYKResetException e) {
				throw e;
			} catch (Exception e) {
				logError("Error occurred while tagging [[:File:" + filename + "]]");
				return;
			}
		} while (true);
	}
	
	/**
	 * Makes sure that each hook is on its own line
	 * @param queue wikitext
	 * @param index of <!--Hooks--> in the queue
	 * @param index of <!--HooksEnd--> in the queue
	 * @return queue wikitext with each hook on its own line
	 */
	private String checkIfEachHookOnNewLine(String queueText, int indexOfHooksinQueue, 
			int indexOfHooksEndinQueue) {
		for (int hookIndex = queueText.indexOf("{{*mp}}", indexOfHooksinQueue); 
				hookIndex != -1 && hookIndex < indexOfHooksEndinQueue;
				hookIndex = queueText.indexOf("{{*mp}}", hookIndex + 7)) {
			if (hookIndex != 0 && queueText.charAt(hookIndex - 1) != '\n') {
				log("Multiple hooks detected on one line, fixing");
				queueText = queueText.substring(0, hookIndex) + "\n" + queueText.substring(hookIndex);
				indexOfHooksEndinQueue++;
			}
		}
		return queueText;
	}
	
	/**
	 * Checks if the DYK has been reset manually.
	 * If so, bot attempts to reset itself by throwing an exception.
	 * The exception propagates up to the run() method.
	 */
	protected void checkIfReset() {
		if (findNextQueueNumber() != nextQueue) {
			log("DYK next queue number has been changed manually, attempting reset");
			throw new DYKResetException();
		}
	}
	
	/**
	 * Finds the first line after the template cluster on an article talk page
	 * Used to add a new DYK talk template after other templates and before conversations
	 */
	private int findLastTemplateIndex(String text) {
		String[] lines = text.split("\n");
		int openingBrackets = 0;
		int closingBrackets = 0;
		int returnIndex = 0;
		for (String line : lines) {
			int openIndex = 0;
			while (line.indexOf("{{", openIndex) != -1) {
				openingBrackets++;
				openIndex = line.indexOf("{{", openIndex) + 2;
			}
			int closeIndex = 0;
			while (line.indexOf("}}", closeIndex) != -1) {
				closingBrackets++;
				closeIndex = line.indexOf("}}", closeIndex) + 2;
			}
			if (line.trim().length() >= 2 && openingBrackets == closingBrackets
					&& ((openIndex == 0 && closeIndex == 0) || 
					line.matches("^[\\s]*\\{\\{[\\s]*[Tt]alk[\\s]*\\:.*"))) {
				return returnIndex;
			}
			returnIndex += line.length() + 1;
		}
		return text.length();
	}
	
	/**
	 * Logs a message into the error log
	 * The error log is then posted to ErrorOutputLoc at the end of every run by postErrors()
	 */
	private void logError(String message) {
		errorLog.append(message).append("\n\n");
		System.out.println("Error: " + message);
	}
	
	/**
	 * At the end of each run, errors will be posted to the page specified in ErrorOutputLoc
	 * Also, the page will be cleared after a clean run
	 */
	private void postErrors() {
		SimpleArticle errorsPage = new SimpleArticle(readContent(ErrorOutputLoc));
		String errors = errorLog.toString().trim();
		errorLog = new StringBuilder(); // clear local buffer
		if (errorsPage.getText().trim().equals(errors)) {
			// if the errors are already on the page, don't post again
			return;
		}
		errorsPage.setText(errors);
		if (errors.isEmpty()) {
			errorsPage.setEditSummary("No errors; clear");
		} else {
			errorsPage.setEditSummary("Posting latest errors");
		}
		try {
			errorsPage.setEditTimestamp(OverrideEditConflicts);
		} catch (ParseException e) {}	// impossible
		writeContent(errorsPage);
	}
	
	/**
	 * Replaces multiple spaces with a single space in the given string
	 * @param the text with unnecessary spaces
	 * @return text without unnecessary spaces
	 */
	private String removeUnnecessarySpaces(String text) {
		String[] words = text.split(" ");
		StringBuilder textWithoutExtraSpaces = new StringBuilder();
		for (String word : words) {
			if (!word.isEmpty()) textWithoutExtraSpaces.append(word).append(" ");
		}
		if (textWithoutExtraSpaces.length() > 0) {
			textWithoutExtraSpaces.deleteCharAt(textWithoutExtraSpaces.length() - 1);
		}
		return textWithoutExtraSpaces.toString();
	}
	
	/**
	 * Checks if a local text file is set to "on"
	 * @return true if it's on, false otherwise
	 */
	protected boolean isOn() {
		do {
			try {
				BufferedReader reader = new BufferedReader(new FileReader("UpdateBotSwitch.txt"));
				String status = reader.readLine();
				reader.close();
				return status.equalsIgnoreCase("on");
			} catch (Exception e) {
				log("File read exception caught");
				sleep(5000);
			}
		} while (true);
	}
	
	public static void main(String[] args) {
		DYKUpdateBot.initializeLoggers();
		DYKUpdateBot updateBot = new DYKUpdateBot(TimeBetweenEdits, NumExceptionsBeforeAttemptedReset,
				NextUpdateQueueLoc, UserInfo.getUser(), UserInfo.getPassword());
		synchronized (updateBot) {
			updateBot.run();
		}
	}
	
	class DYKCredit {
		String articleTitle;
		String userTalkPage;
		String hook;
		boolean errorInArticleTitle;
		boolean dykMake;
		String nompage;
		
		DYKCredit(String articleTitle, String userTalkPage, String hook, boolean errorInArticleTitle,
				boolean dykMake, String nompage) {
			this.articleTitle = articleTitle;
			this.userTalkPage = userTalkPage;
			this.hook = hook;
			this.errorInArticleTitle = errorInArticleTitle;
			this.dykMake = dykMake;
			if (nompage != null) this.nompage = "Template:Did you know nominations/" + nompage;
		}
	}
}

import java.lang.management.ManagementFactory;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Locale;
import java.util.TimeZone;

import net.sourceforge.jwbf.actions.mw.MediaWiki;
import net.sourceforge.jwbf.actions.mw.editing.GetRevision;
import net.sourceforge.jwbf.actions.mw.util.ActionException;
import net.sourceforge.jwbf.bots.MediaWikiBot;
import net.sourceforge.jwbf.bots.util.LoginData;
import net.sourceforge.jwbf.contentRep.mw.Article;
import net.sourceforge.jwbf.contentRep.mw.SimpleArticle;

import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.Namespace;
import org.jdom.input.SAXBuilder;

public abstract class EnWikiBot {
	static {
		TimeZone.setDefault(TimeZone.getTimeZone("Coordinated Universal Time"));
	}
	public static final Namespace ns = Namespace.NO_NAMESPACE;
	protected static final Locale BotLocale = Locale.forLanguageTag("en-US");
	protected static final SimpleDateFormat APITimestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", BotLocale);
	protected static final String OverrideEditConflicts = "9999-12-31T23:59:59Z";
	protected static final String BaseEnWikiAPIURL = "https://en.wikipedia.org/w/";
	final String purgeLoc;
	protected final int timeBetweenEdits;
	protected final int numExceptionsBeforeAttemptedReset;
	protected final String userName;
	private final String password;
	protected long lastRevId;
	protected long lastDelId;
	private MediaWikiBot enBot;
	
	abstract protected boolean isOn();
	abstract protected void checkIfReset();
	
	public EnWikiBot(int timeBetweenEdits, int numExceptionsBeforeAttemptedReset,
			String purgeLoc, String userName, String password) {
		this.timeBetweenEdits = timeBetweenEdits;
		this.numExceptionsBeforeAttemptedReset = numExceptionsBeforeAttemptedReset;
		this.purgeLoc = purgeLoc;
		this.userName = userName;
		this.password = password;
		String processInfo = ManagementFactory.getRuntimeMXBean().getName();
		log("PID: " + processInfo.substring(0, processInfo.indexOf('@')));
		log(Locale.getDefault().toLanguageTag());
		try {
			enBot = new MediaWikiBot(BaseEnWikiAPIURL);
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
		lastRevId = getLastRevId();
	}
	
	/**
	 * Gets the revision ID of the last edit made by the bot
	 * This function is used to make sure that the bot really has edited when it thinks it has
	 * This function is affected by server lag
	 * @return last revision ID
	 */
	protected long getLastRevId() {
		String apiURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query&list=usercontribs" +
				"&uclimit=1&ucprop=ids&ucuser=" + userName;
		Document doc = fetchUsingSAXBuilder(apiURL);
		Element editInfo = doc.getRootElement().getChild("query", ns).getChild("usercontribs", ns).getChild("item", ns);
		return Long.parseLong(editInfo.getAttributeValue("revid"));
	}
	
	/**
	 * Gets the revision ID of the last edit made by the bot at the given page
	 * This function is used to make sure that the bot really has edited when it thinks it has
	 * This function is not affected by server lag
	 * @return last revision ID
	 */
	protected long getLastRevId(String title) {
		title = MediaWiki.encode(title);
		String apiURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query&prop=revisions" +
				"&rvprop=ids&rvlimit=1&rvuser=" + userName + "&titles=" + title;
		Document doc = fetchUsingSAXBuilder(apiURL);
		Element pageInfo = doc.getRootElement().getChild("query", ns).getChild("pages", ns).getChild("page", ns);
		if (pageInfo.getChildren().size() == 0) return 0; // the page has never been edited by the bot
		Element editInfo = pageInfo.getChild("revisions", ns).getChild("rev", ns);
		return Long.parseLong(editInfo.getAttributeValue("revid"));
	}
	
	/**
	 * Finds the redirects to the given page
	 * @param article title
	 * @param limit of the number of redirects to fetch
	 * @return list of pages that redirect to the given page
	 */
	@SuppressWarnings("unchecked")
	protected LinkedList<String> findRedirectsToPage(String title, int limit) {
		LinkedList<String> redirects = new LinkedList<String>();
		String getRedirectsURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query" +
				"&list=backlinks&blfilterredir=redirects&blnamespace=0&bllimit=" + limit + "&bltitle=" + 
				MediaWiki.encode(title);
		Document redirectsInfo = fetchUsingSAXBuilder(getRedirectsURL);
		Element backlinks = redirectsInfo.getRootElement().getChild("query", ns).getChild("backlinks", ns);
		Iterator<Element> redirectIter = backlinks.getDescendants();
		while (redirectIter.hasNext()) {
			redirects.add(redirectIter.next().getAttributeValue("title"));
		}
		return redirects;
	}
	
	/**
	 * Checks if the current page is a redirect by parsing the page
	 * @param text on the page
	 * @return null if the page isn't a redirect, name of the "redirect to" page if it is
	 */
	protected String checkForPageRedirect(String pageText)
	{
		String redirectTo = null;
		if (pageText.toLowerCase().trim().startsWith("#redirect")) {
			int linkStartIndex = pageText.indexOf("[[") + 2;
			int linkPipeIndex = pageText.indexOf("|", linkStartIndex);
			int linkEndIndex = pageText.indexOf("]]", linkStartIndex);
			if (linkStartIndex < linkPipeIndex && linkPipeIndex < linkEndIndex) {
				linkEndIndex = linkPipeIndex;
			}
			redirectTo = pageText.substring(linkStartIndex, linkEndIndex);
			if (redirectTo.indexOf("#") != -1) {
				redirectTo = redirectTo.substring(0, redirectTo.indexOf("#"));
			}
		}
		return redirectTo;
	}
	
	/**
	 * Converts a wiki timestamp (like "2009-01-17T23:45:32Z") to a Java Calendar
	 * @param wikiTimestamp in wiki format
	 * @return Calendar set to the specified time in UTC
	 * @throws ParseException 
	 */
	public Calendar convertWikiTimestamp(String wikiTimestamp) throws ParseException {
		GregorianCalendar time = new GregorianCalendar(BotLocale);
		time.setTime(APITimestampFormat.parse(wikiTimestamp));
		return time;
	}

	/**
	 * Fetches the URL using SAXBuilder
	 * If an exception is thrown, the bot will wait at least 5 seconds before attempting again
	 * @param url you want to fetch (should be formatted in XML)
	 * @return the XML tree in the form of a Document
	 */
	protected Document fetchUsingSAXBuilder(String url) {
		int exceptionCounter = 0;
		do {
			try {
				return new SAXBuilder().build(url);
			} catch (Exception e) {
				exceptionCounter++;
				log("SAXbuilder exception caught, #" + exceptionCounter);
				if (exceptionCounter > numExceptionsBeforeAttemptedReset) {
					checkIfReset();
				}
				// wait at least 5 seconds and at most an hour before attempting another read
				sleep(Math.min(5000 + (1000 * exceptionCounter), 3600000));
			}
		} while (true);
	}
	
	/**
	 * Checks if the bot is logged in, and logs in if not
	 * There's no easy way to tell if the bot's logged in, so the bot null edits its userpage
	 * and checks if its username shows up in the correct variables in the returned HTML
	 * Wikipedia automatically logs out a user one month after login
	 */
	protected void checkifLoggedIn() {
		SimpleArticle userpage = readContent("User:" + userName);
		userpage.setEditSummary("");
		try {
			String userpageHTML = enBot.performAction(new PostModifyContentWithEditConflicts(userpage));
			if (!userpageHTML.contains("\"wgUserName\":\"" + userName + "\"")) {
				// the bot got logged off somehow
				log("Logging in");
				login();
			}
		} catch (Exception e) {
			log("Caught exception during null edit on login check");
		}
	}
	
	/**
	 * Logs in to the wiki
	 */
	protected void login() {
		int exceptionCounter = 0;
		LoginData login = new LoginData();
		do {
			try {
				enBot.performAction(new PostLoginNew(userName, password, login));
				return;
			} catch (Exception e) {
				exceptionCounter++;
				log("Exception caught while logging in");
				if (exceptionCounter > numExceptionsBeforeAttemptedReset) {
					checkIfReset();
				}
				// wait at least 5 seconds and at most an hour before attempting another login
				sleep(Math.min(5000 + (1000 * exceptionCounter), 3600000));
			}
		} while (true);
	}
	
	/**
	 * Purges the given page
	 */
	protected String purge(String page, boolean sleep) {
		int loopCounter = 0;
		do {
			try {
				String xmlReply = enBot.performAction(new PostPurge(page));
				if (xmlReply == null) throw new ActionException();
				if (sleep) sleep(timeBetweenEdits * 1000);
				return xmlReply;
			} catch (Exception e) {
				loopCounter++;
				log("Purge exception caught, #" + loopCounter);
				if (loopCounter > numExceptionsBeforeAttemptedReset) {
					checkIfReset();
				}
				// wait at least 5 seconds and at most an hour before attempting another purge
				sleep(Math.min(5000 + (1000 * loopCounter), 3600000));
			}
		} while (true);
	}
	
	/**
	 * See documentation for readContent(MediaWikiBot, String) below
	 */
	protected Article readContent(String pageName) {
		return readContent(enBot, pageName);
	}
	
	/**
	 * Reads a Wikipedia page
	 * If an exception is thrown (most likely because of server connection issues), the bot will wait
	 * at least 5 seconds until attempting again
	 * The time between attempts increases by 1 second each attempt, up to a maximum of 1 hour
	 * @param bot that specifies which wiki you're reading from
	 * @param page to read
	 * @return the article
	 */
	protected Article readContent(MediaWikiBot bot, String pageName) {
		int loopCounter = 0;
		do {
			try {
				return bot.readContent(pageName, GetRevision.CONTENT | GetRevision.TIMESTAMP);
			} catch (Exception e) {
				loopCounter++;
				log("Read exception caught, #" + loopCounter);
				if (loopCounter > numExceptionsBeforeAttemptedReset) {
					checkIfReset();
				}
				// wait at least 5 seconds and at most an hour before attempting another read
				sleep(Math.min(5000 + (1000 * loopCounter), 3600000));
			}
		} while (true);
	}
	
	/**
	 * See documentation for writeContent(MediaWikiBot, SimpleArticle) below
	 */
	protected void writeContent(SimpleArticle page) {
		writeContent(enBot, page);
	}
	
	/**
	 * Edits a Wikipedia page
	 * If an exception is thrown (most likely because of server connection issues), the bot will wait
	 * at least 5 seconds until attempting again
	 * The time between attempts increases by 1 second each attempt, up to a maximum of 1 hour
	 * @param logged-in bot
	 * @param page you want to edit
	 */
	protected void writeContent(MediaWikiBot bot, SimpleArticle page) {
		int loopCounter = 0;
		String normalizedTitle = normalizeTitle(page.getLabel());
		if (!normalizedTitle.equals(page.getLabel())) {
			log("Title normalized from " + page.getLabel() + " to " + normalizedTitle);
			page.setLabel(normalizedTitle);
		}
		do {
			try {
				bot.performAction(new PostModifyContentWithEditConflicts(page));
				log("Editing " + page.getLabel());
				sleep(timeBetweenEdits * 1000);
				long latestRevId = getLastRevId(page.getLabel());
				if (latestRevId <= lastRevId) { // the edit didn't go through
					log("Edit didn't process correctly, attempting again");
					throw new ActionException();
				} else {
					lastRevId = latestRevId;
				}
				return;
			} catch (EditConflictException e) {
				if (loopCounter > 0) {
					log("Newer page available, but skipping to avoid " +
							"double-editing (check for edit conflicts)");
					lastRevId = getLastRevId(page.getLabel());
					return;
				} else {
					throw e;
				}
			} catch (Exception e) {
				// wait at least 5 seconds and at most an hour
				sleep(Math.min(5000 + (1000 * loopCounter), 3600000));
				long latestRevId = getLastRevId(page.getLabel());
				if (latestRevId > lastRevId) { // the edit did go through
					log("Edit processed correctly, continuing");
					lastRevId = latestRevId;
					sleep(timeBetweenEdits * 1000);
					return;
				} // else
				loopCounter++;
				log("Write exception caught, #" + loopCounter);
				if (loopCounter > numExceptionsBeforeAttemptedReset) {
					checkIfReset();
				}
				checkifLoggedIn(); //make sure we're logged in
			}
		} while (true);
	}
	

	/**
	 * Deletes a Wikipedia page
	 * If an exception is thrown (most likely because of server connection issues), the bot will wait
	 * at least 5 seconds before attempting again
	 * The time between attempts increases by 1 second each attempt, up to a maximum of 10 attempts
	 * @param page to delete
	 * @param reason for deletion
	 */
	protected void deleteContent(String pageName, String reason) {
		boolean errorThrown = false;
		int loopCounter = 0;
		do {
			try {
				enBot.performAction(new PostDeleteWithReason(pageName, 
						reason, enBot.getSiteinfo(), enBot.getUserinfo()));
				log("Deleting " + pageName);
				errorThrown = false;
				sleep(timeBetweenEdits * 1000);
				long latestDelId = getLastDelId();
				if (latestDelId <= lastDelId) { // the delete didn't go through
					log("Delete didn't process correctly, attempting again");
					throw new ActionException();
				} else {
					lastDelId = latestDelId;
				}
				return;
			} catch (Exception e) {
				sleep(timeBetweenEdits * 1000);
				long latestDelId = getLastDelId();
				if (latestDelId > lastDelId) { // the delete did go through
					log("Delete processed correctly, continuing");
					lastDelId = latestDelId;
					errorThrown = false;
					return;
				} // else
				errorThrown = true;
				loopCounter++;
				log("Delete exception caught, #" + loopCounter);
				checkifLoggedIn(); //make sure we're logged in
				// wait at least 5 seconds and at most an hour before attempting another delete
				sleep(Math.min(5000 + (1000 * loopCounter), 3600000));
			}
		} while (errorThrown && loopCounter < 10);
	}

	/**
	 * Unprotects a Wikipedia page
	 * If an exception is thrown (most likely because of server connection issues), the bot will wait
	 * at least 5 seconds before attempting again
	 * The time between attempts increases by 1 second each attempt, up to a maximum of 10 attempts
	 * @param page to unprotect
	 * @param reason for unprotection
	 */
	protected void unprotectContent(String pageName, String reason) {
		boolean errorThrown = false;
		int loopCounter = 0;
		do {
			try {
				enBot.performAction(new PostUnprotectWithReason(pageName, reason));
				log("Unprotecting " + pageName);
				errorThrown = false;
				sleep(timeBetweenEdits * 1000);
				return;
			} catch (Exception e) {
				errorThrown = true;
				loopCounter++;
				log("Unprotect exception caught, #" + loopCounter);
				checkifLoggedIn(); //make sure we're logged in
				// wait at least 5 seconds and at most an hour before attempting another delete
				sleep(Math.min(5000 + (1000 * loopCounter), 3600000));
			}
		} while (errorThrown && loopCounter < 10);
	}
	
	/**
	 * Gets the log ID of the last delete action by the bot
	 * This function is used to make sure that the bot really has deleted when it thinks it has
	 * @return last deletion log ID
	 */
	protected long getLastDelId() {
		String apiURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query&list=logevents" +
			"&letype=delete&leprop=ids&lelimit=1&leuser=" + userName;
		Document doc = fetchUsingSAXBuilder(apiURL);
		Element itemInfo = doc.getRootElement().getChild("query", ns).getChild("logevents", ns).getChild("item", ns);
		return Long.parseLong(itemInfo.getAttributeValue("logid"));
	}
	
	/**
	 * Normalizes a page title so Mediawiki will like it
	 * @param title to be normalized, e.g. "1922&ndash;23 Nelson F.C. season"
	 * @return normalized title, e.g. "1922–23 Nelson F.C. season"
	 */
	protected String normalizeTitle(String pageName) {
		pageName = pageName.replaceAll("&amp;", "&");
		String apiURL = BaseEnWikiAPIURL + "api.php?format=xml&action=query&titles=" +
				MediaWiki.encode(pageName);
		Document doc = fetchUsingSAXBuilder(apiURL);
		Element normalized = doc.getRootElement().getChild("query", ns).getChild("normalized", ns);
		if (normalized == null) return pageName;
		return normalized.getChild("n", ns).getAttributeValue("to");
	}

	/**
	 * Expands templates in the given wikitext
	 * @param the wikitext to be expanded
	 * @return expanded wikitext
	 */
	protected String expandTemplates(String wikitext) {
		String apiURL = BaseEnWikiAPIURL + "api.php?format=xml&action=expandtemplates&text=" +
			MediaWiki.encode(wikitext);
		Document doc = fetchUsingSAXBuilder(apiURL);
		return doc.getRootElement().getChildText("expandtemplates", ns);
	}
	
	/**
	 * Pause for the time given
	 * @param time to wait, in milliseconds
	 */
	protected void sleep(long milliseconds) {
		do {
			try {
				this.wait(milliseconds);
				return;
			} catch (InterruptedException e1) {
				log ("Interrupted exception caught");
			}
		} while (true);
	}
	
	protected void log(String message) {
		System.out.println(message);
	}
	
	/**
	 * Initializes the various loggers that the JWBF uses
	 */
	protected static void initializeLoggers() {
		BasicConfigurator.configure();
		Logger.getLogger("org.apache.commons.httpclient").setLevel(Level.FATAL);
		Logger.getLogger("httpclient.wire").setLevel(Level.FATAL);
		Logger.getLogger("net.sourceforge.jwbf").setLevel(Level.FATAL);
		Logger.getLogger(PostModifyContentWithEditConflicts.class).setLevel(Level.FATAL);
	}
}

import java.util.Date;

public class DYKFile {
	private final String filename;
	private final String type;
	private Date dykDate;
	private final String rolloverText;
	private boolean cuploaded;
	private String croppedFrom;
	
	// for backwards-compatibility with other packages
	public DYKFile(String filename, String type, String rolloverText) {
		this.filename = filename;
		this.type = type;
		this.rolloverText = rolloverText;
		this.cuploaded = false;
	}
	
	public DYKFile(String filename, String type) {
		this.filename = filename;
		this.type = type;
		this.rolloverText = null;
		this.cuploaded = false;
	}
	
	public String getFilename() {
		return filename;
	}
	
	public String getType() {
		return type;
	}
	
	public void setDYKDate(Date dykDate) {
		this.dykDate = dykDate;
	}
	
	public Date getDYKDate() {
		return dykDate;
	}
	
	public void setCuploaded(boolean cuploaded) {
		this.cuploaded = cuploaded;
	}
	
	public boolean getCuploaded() {
		return cuploaded;
	}
	
	public void setCroppedFrom(String filename) {
		croppedFrom = filename;
	}
	
	public String getCroppedFrom() {
		return croppedFrom;
	}
	
	public String toStatsString() {
		if (type.equals("sound")) {
			return "{{DYK Listen|" + filename + "|" + rolloverText + "}}";
		} else if (type.equals("video")) {
			return "{{DYK Watch|" + filename + "|" + rolloverText + "}}";
		} else {
			return "[[File:" + filename + "|100x100px|" + rolloverText + "]]";
		}
	}
	
	public String toString() {
		return null; // unused code
	}
}

import net.sourceforge.jwbf.actions.Post;
import net.sourceforge.jwbf.actions.mw.HttpAction;
import net.sourceforge.jwbf.actions.mw.util.MWAction;
import net.sourceforge.jwbf.actions.mw.util.ProcessException;

public class PostPurge extends MWAction {
	private final Post msg;
	
	public PostPurge(final String title) {
		super();
		Post pm = new Post("/api.php?action=purge&format=xml");
		pm.addParam("titles", title);
		msg = pm;
	}
	
	public String processAllReturningText(final String s) throws ProcessException {
		return s;
	}
	
	public HttpAction getNextMessage() {
		return msg;
	}

}

public class DYKResetException extends RuntimeException {
	private static final long serialVersionUID = 6465485908664532508L;
}

public class EditConflictException extends RuntimeException {
	private static final long serialVersionUID = 7595756569739191727L;
}

import org.apache.log4j.Logger;

import net.sourceforge.jwbf.actions.Post;
import net.sourceforge.jwbf.actions.mw.HttpAction;
import net.sourceforge.jwbf.actions.mw.MediaWiki;
import net.sourceforge.jwbf.actions.mw.editing.PostDelete;
import net.sourceforge.jwbf.actions.mw.util.ProcessException;
import net.sourceforge.jwbf.contentRep.mw.Siteinfo;
import net.sourceforge.jwbf.contentRep.mw.Userinfo;

public class PostDeleteWithReason extends PostDelete {
	protected static final Logger LOG = Logger.getLogger(PostDelete.class);
	protected final String reason;
	protected final String title;


	public PostDeleteWithReason(String title, String reason, Siteinfo si, Userinfo ui)
			throws ProcessException {
		super(title, si, ui);
		this.reason = reason;
		this.title = title;
	}
	
	/**
	 * This method is copied from PostDelete, with the reason added into the URL
	 */
	@Override
	protected HttpAction getSecondRequest() {
		HttpAction msg = null;
		if (getToken() == null || getToken().length() == 0) {
			throw new IllegalArgumentException(
					"The argument 'token' must not be \""
					+ String.valueOf(getToken()) + "\"");
		}
		if (LOG.isTraceEnabled()) {
			LOG.trace("enter PostDelete.generateDeleteRequest(String)");
		}

		String uS = "/api.php" + "?action=delete" + "&title=" + MediaWiki.encode(title) + 
				"&token=" + MediaWiki.encode(getToken()) + 
				"&reason=" + MediaWiki.encode(reason) + "&format=xml";
		if (LOG.isDebugEnabled()) {
			LOG.debug("delete url: \"" + uS + "\"");
		}
		Post pm = new Post(uS);
		msg = pm;

		return msg;
	}
}

//this is almost a straight copy & paste of revision 260 of JWBF's PostLogin
/*
 * Copyright 2007 Thomas Stock.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 * Contributors:
 * Philipp Kohl
 * Carlos Valenzuela
 */

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;

import net.sourceforge.jwbf.actions.mw.login.PostLogin;
import net.sourceforge.jwbf.actions.mw.util.ProcessException;
import net.sourceforge.jwbf.actions.mw.util.MWAction;

import org.apache.log4j.Logger;
import org.jdom.DataConversionException;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.xml.sax.InputSource;
/**
 *
 * @author Thomas Stock
 */
public class PostLoginNew extends MWAction {
	private final Logger log = Logger.getLogger(PostLogin.class);
	private net.sourceforge.jwbf.actions.Post msg;


	private final String success = "Success";
	private final String wrongPass = "WrongPass";
	private final String notExists = "NotExists";
	private final String needToken = "NeedToken";
	private net.sourceforge.jwbf.bots.util.LoginData login = null;
    private boolean reTry = false;
    private boolean reTryLimit = true;
    private final String username;
    private final String pw;

	/**
	 *
	 * @param username the
	 * @param pw password
	 * @param domain a
	 * @param login a
	 */
	public PostLoginNew(final String username, final String pw, net.sourceforge.jwbf.bots.util.LoginData login) {
		super();
		this.login = login;
		this.username = username;
		this.pw = pw;
		msg = getLoginMsg(username, pw, null);

	}

    private net.sourceforge.jwbf.actions.Post getLoginMsg(final String username, final String pw,
            final String token) {
        net.sourceforge.jwbf.actions.Post pm = new net.sourceforge.jwbf.actions.Post("/api.php?action=login&format=xml");
        pm.addParam("lgname", username);
        pm.addParam("lgpassword", pw);
        if (token != null) {
            pm.addParam("lgtoken", token);
        }
        return pm;
    }

	/**
	 * {@inheritDoc}
	 */
	@Override
	public String processAllReturningText(final String s) throws ProcessException {
		SAXBuilder builder = new SAXBuilder();
		Element root = null;
		try {
			Reader i = new StringReader(s);
			Document doc = builder.build(new InputSource(i));

			root = doc.getRootElement();
			findContent(root);
		} catch (JDOMException e) {
			log.error(e.getClass().getName() + e.getLocalizedMessage());
		} catch (IOException e) {
			log.error(e.getClass().getName() + e.getLocalizedMessage());
		} catch (NullPointerException e) {
			log.error(e.getClass().getName() + e.getLocalizedMessage());
			throw new ProcessException("No regular content was found, check your api\n::" + s);
		} catch (Exception e) {
			log.error(e.getClass().getName() + e.getLocalizedMessage());
			throw new ProcessException(e.getLocalizedMessage());
		}


		return s;
	}
	/**
	 *
	 * @param startElement the, where the search begins
	 * @throws ProcessException if problems with login
	 */
	private void findContent(final Element startElement) throws ProcessException {

		Element loginEl = startElement.getChild("login", EnWikiBot.ns);
		String result = loginEl.getAttributeValue("result");
		if (result.equalsIgnoreCase(success)) {
			try {
				login.setup(loginEl.getAttribute("lguserid").getIntValue()
						, loginEl.getAttributeValue("lgusername"), "0", true);
			} catch (DataConversionException e) {
				e.printStackTrace();
			}
		} else if (result.equalsIgnoreCase(needToken) && reTryLimit ) {
			msg = getLoginMsg(username, pw, loginEl.getAttributeValue("token"));
			reTry = true;
			reTryLimit = false;
		} else if (result.equalsIgnoreCase(wrongPass)) {
			throw new ProcessException("Wrong Password");
		} else if (result.equalsIgnoreCase(notExists)) {
			throw new ProcessException("No sutch User");
		}

	}
	/**
	 * {@inheritDoc}
	 */
	public net.sourceforge.jwbf.actions.mw.HttpAction getNextMessage() {
		return msg;
	}

    /* (non-Javadoc)
     * @see net.sourceforge.jwbf.mediawiki.actions.util.MWAction#hasMoreMessages()
     */
    @Override
    public boolean hasMoreMessages() {
        boolean temp = super.hasMoreMessages() || reTry;
        reTry  = false;
        return temp;
    }
}

// this is JWBF's PostModifyContent (rev 178) modified for edit conflicts and new(er) edit token requirements
/*
 * Copyright 2007 Thomas Stock.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * 
 * Contributors:
 * 
 */
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Hashtable;

import net.sourceforge.jwbf.actions.Post;
import net.sourceforge.jwbf.actions.mw.HttpAction;
import net.sourceforge.jwbf.actions.mw.MediaWiki;
import net.sourceforge.jwbf.actions.mw.util.MWAction;
import net.sourceforge.jwbf.actions.mw.util.ProcessException;
import net.sourceforge.jwbf.contentRep.mw.ContentAccessable;
import net.sourceforge.jwbf.contentRep.mw.SimpleArticle;

import org.apache.log4j.Logger;

/**
 * 
 * 
 * Writes an article.
 * 
 * 
 * TODO no api use.
 * @author Thomas Stock
 * @supportedBy MediaWiki 1.9.x, 1.10.x, 1.11.x, 1.12.x, 1.13.x, 1.14.x
 * 
 */
public class PostModifyContentWithEditConflicts extends MWAction {

	protected static final SimpleDateFormat WpTimestampFormat = new SimpleDateFormat("yyyyMMddHHmmss");
	private int numMessagesSent = 0;
	private final ContentAccessable article;
	private static final Logger LOG = Logger.getLogger(PostModifyContentWithEditConflicts.class);
	private Hashtable<String, String> table = new Hashtable<String, String>();

	/**
	 * 
	 * @param a
	 *            the
	 */
	public PostModifyContentWithEditConflicts(final ContentAccessable a) {
		this.article = a;
	}

	
	public HttpAction getNextMessage() {
		++numMessagesSent;
		
		Post postMessage = new Post("/index.php?title=" + MediaWiki.encode(article.getLabel()) + "&action=submit");
		
		if (numMessagesSent == 1) {
			return postMessage; // send off first request to grab edit token from the response
		}

		postMessage.addParam("wpSave", "Save");

		postMessage.addParam("wpUltimateParam", table.get("wpUltimateParam"));

		postMessage.addParam("wpUnicodeCheck", table.get("wpUnicodeCheck"));

		postMessage.addParam("wpStarttime", table.get("wpStarttime"));

		postMessage.addParam("wpEditToken", table.get("wpEditToken"));
		
		try {
			if (WpTimestampFormat.parse(table.get("wpEdittime")).getTime() >
					((SimpleArticle) article).getEditTimestamp().getTime()) {
				throw new EditConflictException();
			}
		} catch (ParseException e) {}	// impossible
		postMessage.addParam("wpEdittime", table.get("wpEdittime"));
		
		postMessage.addParam("wpTextbox1", article.getText());

		String editSummaryText = article.getEditSummary();
		if (editSummaryText != null && editSummaryText.length() > 200) {
			editSummaryText = editSummaryText.substring(0, 200);
		}

		postMessage.addParam("wpSummary", editSummaryText);
		if (article.isMinorEdit()) {
			postMessage.addParam("wpMinoredit", "1");
		}

		LOG.info("WRITE: " + article.getLabel());
		
		return postMessage;
	}

	@Override
	public boolean hasMoreMessages() {
		return numMessagesSent < 2;
	}

	@Override
	public String processReturningText(String returnedHTML, HttpAction action)
			throws ProcessException {
		if (numMessagesSent == 1) {
			parseWpValues(returnedHTML);
			LOG.debug(table);
		}
		return returnedHTML;
	}

	/**
	 * 
	 * @param text
	 *            where to search
	 * @param table
	 *            table with required values
	 */
	private void parseWpValues(final String text) {
		String[] tParts = text.split("\n");
		// System.out.println(tParts.length);
		for (int i = 0; i < tParts.length; i++) {
			if (tParts[i].indexOf("wpEditToken") > 0) {
				// \<input type='hidden' value=\"(.*?)\" name=\"wpEditToken\"
				int begin = tParts[i].indexOf("value") + 7;
				int end = tParts[i].indexOf("name") - 2;
				// System.out.println(line.substring(begin, end));
				// System.out.println("read wp token:" + tParts[i]);
				table.put("wpEditToken", tParts[i].substring(begin, end));

			} else if (tParts[i].indexOf("wpEdittime") > 0) {
				// value="(\d+)" name=["\']wpEdittime["\']
				int begin = tParts[i].indexOf("value") + 7;
				int end = tParts[i].indexOf("name") - 2;
				// System.out.println( "read wp edit: " +
				// tParts[i].substring(begin, end));

				table.put("wpEdittime", tParts[i].substring(begin, end));

			} else if (tParts[i].indexOf("wpStarttime") > 0) {
				// value="(\d+)" name=["\']wpStarttime["\']
				int begin = tParts[i].indexOf("value") + 7;
				int end = tParts[i].indexOf("name") - 2;
				// System.out.println("read wp start:" + tParts[i]);

				table.put("wpStarttime", tParts[i].substring(begin, end));

			} else if (tParts[i].indexOf("wpUnicodeCheck") > 0) {
				// \<input type='hidden' value=\"(.*?)\" name=\"wpUnicodeCheck\"
				int begin = tParts[i].indexOf("value") + 7;
				int end = tParts[i].indexOf("name", begin) - 2;
				// System.out.println(line.substring(begin, end));
				// System.out.println("read wp token:" + tParts[i]);
				table.put("wpUnicodeCheck", tParts[i].substring(begin, end));
			} else if (tParts[i].indexOf("wpUltimateParam") > 0) {
				// \<input type='hidden' value=\"(.*?)\" name=\"wpUltimateParam\"
				int begin = tParts[i].indexOf("value") + 7;
				int end = tParts[i].indexOf("name", begin) - 2;
				// System.out.println(line.substring(begin, end));
				// System.out.println("read wp token:" + tParts[i]);
				table.put("wpUltimateParam", tParts[i].substring(begin, end));
			} 
		}

	}

}

import java.io.IOException;
import java.io.StringReader;

import org.jdom.Document;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.xml.sax.InputSource;

import net.sourceforge.jwbf.actions.Get;
import net.sourceforge.jwbf.actions.Post;
import net.sourceforge.jwbf.actions.mw.HttpAction;
import net.sourceforge.jwbf.actions.mw.MediaWiki;
import net.sourceforge.jwbf.actions.mw.util.MWAction;

public class PostUnprotectWithReason extends MWAction {
	private final String title;
	private final String reason;
	private final Get tokenRequest;
	private String token;
	private boolean inHandshake = true;
	private boolean finished = false;
	
	public PostUnprotectWithReason(String title, String reason) throws JDOMException, IOException {
		this.title = title;
		this.reason = reason;
		if (title == null || title.length() == 0) {
		  throw new IllegalArgumentException("The argument 'title' must not be null or empty");
		}
		
		// URL to fetch a protect token from the API
		String url = "/api.php?format=xml&action=query&prop=info&titles=" +
				MediaWiki.encode(title) + "&intoken=protect";
		tokenRequest = new Get(url);
	}
	
	@Override
	public String processReturningText(String s, HttpAction response) {
		if (response.getRequest().equals(tokenRequest.getRequest())) {
			Document tokenPage;
			try {
				tokenPage = new SAXBuilder().build(new InputSource(new StringReader(s)));
				token = tokenPage.getRootElement().getChild("query", EnWikiBot.ns).getChild("pages", EnWikiBot.ns)
						.getChild("page", EnWikiBot.ns).getAttributeValue("protecttoken");
			} catch (JDOMException e) {
				throw new UnprotectException();
			} catch (IOException e) {
				throw new UnprotectException();
			}
		}
		return "";
	}
	
	protected HttpAction getSecondRequest() {
		HttpAction unprotectRequest = null;
		if (token == null || token.length() == 0) {
			throw new IllegalArgumentException(
					"The argument 'token' must not be \""
					+ token + "\"");
		}
		String bar = MediaWiki.encode("|");

		String unprotectURL = "/api.php?format=xml&action=protect" +
				"&protections=edit=all" + bar + "move=all" + bar + "upload=all" + 
				"&title=" + MediaWiki.encode(title) + 
				"&token=" + MediaWiki.encode(token) + 
				"&reason=" + MediaWiki.encode(reason);
		unprotectRequest = new Post(unprotectURL);

		return unprotectRequest;
	}

	@Override
	public HttpAction getNextMessage() {
		if (inHandshake) {
			inHandshake = false;
			return tokenRequest;
		} else {
			finished = true;
			return getSecondRequest();
		}
	}
	
	@Override
	public boolean hasMoreMessages() {
		return !finished;
	}
	
	public class UnprotectException extends RuntimeException {
		private static final long serialVersionUID = 1L;
	}
}