User:Magicpiano/NRBot/NRISOnly.js
< User:Magicpiano | NRBot
Code that you insert on this page could contain malicious content capable of compromising your account. If you import a script from another page with "importScript", "mw.loader.load", "iusc", or "lusc", take note that this causes you to dynamically load a remote script, which could be changed by others. Editors are responsible for all edits and actions they perform, including by scripts. User scripts are not centrally supported and may malfunction or become inoperable due to software changes. A guide to help you find broken scripts is available. If you are unsure whether code you are adding to this page is safe, you can ask at the appropriate village pump.
This code will be executed when previewing this page.
This code will be executed when previewing this page.
Documentation for this user script can be added at User:Magicpiano/NRBot/NRISOnly.
/* jshint maxerr: 3000 , esversion: 6 */
var wikitext = 'error';
var NRISOnlyStructure=[]; // NRISOnlyStructure[table][row]["Titles"][item].StatName to get info
var TotalToQuery=0;
var TotalQueried=0;
var BotName= "NationalRegisterBot";
var BotUser = "User:"+BotName;
var OutputBase = BotUser+"/AllNRHPPages";
var NRISOnlyFile = BotUser+"/NRISOnly";
var SubstubsFile = BotUser+"/Substubs";
var StatusBase = BotUser;
var ErrorCount=0;
var WarningCount=[["",0]]; // 0=status, 1=count
var InitialTime=0;
var ProgressDivTimer=0;// timer for updating ProgressDiv
var DefaultQueryPause=1; // number of milliseconds to wait between each API query; increased by code if rate limit reached
// Users authorized to run this script
var AuthorizedUsers = [BotName, "Dudemanfellabra", "Magicpiano", "TheCatalyst31"];
var CurrentMaintainer = "Magicpiano"; // User who is currently maintaining the script
var CurrentOperator = ""; // User who is running this instance (will be set from dropdown list)
var CurrentOperatorEditSummary = ""; // "(operated by $CurrentOperator )"
function CheckPermission() {
var username = mw.user.getName();
console.log("CheckPermission",username);
for (var i=0; i< AuthorizedUsers.length; i++) {
if (username == AuthorizedUsers[i]) {
NRISOnlyButtons();
return;
}
}
alert("You are not authorized to run the NRISOnly script. Contact "+CurrentMaintainer+" for permission if you think you should have it.");
}
function NRISOnlyButtons() {
var pageName=mw.config.get('wgPageName');
//look to see if button is already here
var button = document.getElementById('nrisonlybutton');
var needsButton = false;
if (button === null) {
// no such element
needsButton = true;
} else if (!button.hasAttributes || !button.hasAttribute("class")) {
needsButton = true;
} else if (button.getAttribute("id") !== "nrisonlybutton") {
needsButton = true;
}
if (!needsButton) return;
button=document.createElement("input");
button.setAttribute("type", "button");
button.setAttribute("id", "nrisonlybutton");
button.id = 'nrisonlybutton';
var operatorfield=document.createElement("SELECT");
operatorfield.setAttribute("id","nrisonlyoperator");
operatorfield.setAttribute("type","select");
var nochoice=document.createElement("OPTION");
nochoice.setAttribute("type","option");
nochoice.text="Please select your username";
operatorfield.add(nochoice);
var username = mw.user.getName();
var initial_selection = 0; //default to "Please select"
var count=0;
for (var i=1; i< AuthorizedUsers.length; i++) {
if (AuthorizedUsers[i] != BotName) {
count++;
var opchoice=document.createElement("OPTION");
opchoice.setAttribute("type","option");
opchoice.label=AuthorizedUsers[i];
opchoice.text=AuthorizedUsers[i];
if (username != BotName) {
opchoice.disabled = (username != AuthorizedUsers[i]);
if (!opchoice.disabled) {
initial_selection = count; // forcing to current user
}
} else {
opchoice.disabled=false;
}
operatorfield.add(opchoice);
}
}
operatorfield.selectedIndex=initial_selection;
var content=document.getElementById('mw-content-text');
if (location.href.indexOf('action')==-1) {
if (pageName=="Wikipedia:WikiProject_National_Register_of_Historic_Places/Progress") {
button.setAttribute("value", "Update list for "+BotName);
button.setAttribute("onclick", "NRISOnlyClicked('update')");
} else if (pageName==OutputBase+"/Duplications") {
button.setAttribute("value", "Gather duplicate stats");
button.setAttribute("onclick", "NRISOnlyClicked('duplicate')");
} else if (pageName==NRISOnlyFile) {
button.setAttribute("value", "Tag these articles");
button.setAttribute("onclick", "NRISOnlyClicked('tag')");
} else {
return;
}
content.parentNode.insertBefore(button, content);
content.parentNode.insertBefore(operatorfield, content);
}
}
function NRISOnlyClicked(action) { // after button is clicked, disable it and perform action
var operatorfield = document.getElementById('nrisonlyoperator');
if (operatorfield.selectedIndex===0) {
alert("Please select your username");
return;
}
CurrentOperator = operatorfield.value;
CurrentOperatorEditSummary = " (operated by [[User:"+CurrentOperator+"|"+CurrentOperator+"]])";
var nrisonlybutton = document.getElementById('nrisonlybutton');
nrisonlybutton.disabled = true;
var ProgressDiv = document.createElement("div");
ProgressDiv.setAttribute("id", "ProgressDiv");
ProgressDiv.setAttribute("style", "width:600px; border:1px solid black; padding:5px; background:#ffffff");
nrisonlybutton.parentNode.insertBefore(ProgressDiv, nrisonlybutton);
if (action=="update") {
ProgressDiv.innerHTML = "Initializing...";
getNRISOnlyProgressPageWikitext(mw.config.get('wgPageName')); // after wikitext fetched, SetupNRISOnlyTables() is called
} else if (action=="duplicate") {
CheckDuplicatesNew(); // TODO
} else if (action=="tag") {
TagNRISOnly();
}
}
// create array of table structure to be populated later
function SetupNRISOnlyTables() {
var table=document.getElementsByClassName('wikitable sortable');
// extract list names from Progress page
for (var i=1; i<table.length; i++) { // skip national table
var tr=table[i].getElementsByTagName("tr");
NRISOnlyStructure[i-1]=[];
var skipOffset=1; // number of duplicate/skippable rows encountered so far
for (var j=1; j<tr.length-2; j++) { // skip title row, statewide duplicates, and totals row
var td=tr[j].getElementsByTagName("td"); // fill in existing data in case error
var link=td[1].getElementsByTagName("a");
if (link.length!==0 && link[0].href.search("#")==-1) {
NRISOnlyStructure[i-1][j-skipOffset]={};
NRISOnlyStructure[i-1][j-skipOffset].Titles=[]; // Placeholder for article names
link=decodeURI(link[0].href).split("/");
link=link[link.length-1].replace(/_/g," ");
NRISOnlyStructure[i-1][j-skipOffset].Link=link;
NRISOnlyStructure[i-1][j-skipOffset].ID=td[0].innerHTML.substr(0,5); // for finding empty counties
NRISOnlyStructure[i-1][j-skipOffset].CatsQueried=0; // for querying later
NRISOnlyStructure[i-1][j-skipOffset].TotalToCheck=0;
NRISOnlyStructure[i-1][j-skipOffset].TotalChecked=0;
} else { // must be a duplicate row
skipOffset++;
}
}
}
for (i=0; i<NRISOnlyStructure.length; i++) { // count total number of lists to check
TotalToQuery+=NRISOnlyStructure[i].length;
}
var ProgressDiv=document.getElementById("ProgressDiv");
ProgressDiv.innerHTML+=" Done!<br>";
var ProgressSpan=document.createElement("span");
ProgressSpan.setAttribute("id", "ProgressSpan");
ProgressDiv.appendChild(ProgressSpan);
ProgressSpan.innerHTML = "Querying articles in each list... 0 (0%) of "+TotalToQuery+" lists complete.";
var TimeSpan=document.createElement("span");
TimeSpan.setAttribute("id", "TimeSpan");
ProgressDiv.appendChild(TimeSpan);
TimeSpan.innerHTML = "";
var EditSpan=document.createElement("span");
EditSpan.setAttribute("id", "EditSpan");
ProgressDiv.appendChild(EditSpan);
EditSpan.innerHTML = "";
InitialTime=new Date(); // record starting time
UpdateNRISOnlyProgressDiv();
LoadNRISOnlyList(0,0); // begin querying first page
}
// load next list to query
function LoadNRISOnlyList(currentTable,currentRow) {
// check if we need to go to the next table
if (currentRow>NRISOnlyStructure[currentTable].length-1) {
currentRow=0;
currentTable++;
}
// check if there are no more tables
if (currentTable>NRISOnlyStructure.length-1) return;
setTimeout(function(){ // short delay to prevent API overload
getNRISOnlyListWikitext(currentTable,currentRow);
LoadNRISOnlyList(currentTable,currentRow+1);
}, DefaultQueryPause);
return;
}
function NRISOnlyWikitextFetched(ajaxResponse,status,currentTable,currentRow) {
if (status!="success") {
NewNRISOnlyWarning("Wikitext "+ajaxResponse.errorThrown);
setTimeout(function(){ // try again after delay if rate limit reached
getNRISOnlyListWikitext(currentTable,currentRow);
}, 250);
return;
}
// won't get here unless successful
var responseText=JSON.parse(ajaxResponse.responseText);
var pagetext=responseText.query.pages[responseText.query.pageids[0]].revisions[0]["*"];
var regex;
var StartIndex;
var tabletext;
if (responseText.query.redirects) { // if redirect, find section
var SectionName="Undefined";
for (var r in responseText.query.redirects) {
if (typeof responseText.query.redirects[r].tofragment!="undefined") SectionName=responseText.query.redirects[r].tofragment.replace(/.27/g,"'");
}
regex = new RegExp("=[ ]*(\\[\\[(.*?\\|)?[ ]*)?"+SectionName+"([ ]*\\]\\])?[ ]*=", "g");
var sectionheader=pagetext.match(regex);
if (sectionheader === null || sectionheader === undefined) { // if no section found, check if one of known empty counties
// last checked: 2023-04-03
var EmptyCounties=[//"01061", // Geneva County AL
"02270", // Kusilvak Census Area AK (old)
"02158", // also Kusilvak Census Area AK (recent)
"08014", // Broomfield County CO
"12067", // Lafayette County FL
"20081", // Haskell County KS
"20175", // Seward County KS
"20187", // Stanton County KS
"20189", // Stevens County KS
"26051", // Gladwin County MI
"26079", // Kalkaska County MI
"26119", // Montmorency County MI
"26129", // Ogemaw County MI
"26133", // Osceola County MI
"31009", // Blaine County NE
"31113", // Logan County NE
"31117", // McPherson County NE
"38085", // Sioux County ND
"48017", // Bailey County TX
"48023", // Baylor County TX
"48033", // Borden County TX
"48069", // Castro County TX
"48079", // Cochran County TX
"48103", // Crane County TX
"48107", // Crosby County TX
"48119", // Delta County TX
"48131", // Duval County TX
"48155", // Foard County TX
"48165", // Gaines County TX
"48207", // Haskell County TX
"48219", // Hockley County TX
"48247", // Jim Hogg County TX
"48269", // King County TX
"48279", // Lamb County TX
"48341", // Moore County TX
"48389", // Reeves County TX
"48415", // Scurry County TX
"48421", // Sherman County TX
"48433", // Stonewall County TX
"48437", // Swisher County TX
"48445", // Terry County TX
"48461", // Upton County TX
"48475", // Ward County TX
"48501", // Yoakum County TX
"51735" // Poquoson VA
];
var ID = NRISOnlyStructure[currentTable][currentRow].ID;
var errorcode = 0;
for (var k=0; k<EmptyCounties.length; k++) {
if (ID==EmptyCounties[k]) {errorcode=-1}
}
if (errorcode!==0) { // must be an empty county
TotalQueried++;
if (TotalQueried==TotalToQuery) DoneQueryingNRIS();
return;
}
// if we're here, must have been a redirect with no section, and not a known empty county
sectionheader=pagetext.match(/{{NRHP header/g); // then look for tables without a section
if (sectionheader===null||sectionheader.length>1) { // if still can't find a table or find multiple tables, fatal error
NRISOnlyFatalError(0,currentTable,currentRow);
return;
}
}
StartIndex=pagetext.indexOf(sectionheader[0]);
var sectiontext=pagetext.substr(StartIndex,pagetext.indexOf("\n==",StartIndex)-StartIndex); // only look at relevant section
StartIndex=sectiontext.indexOf("{{NRHP header");
if (StartIndex==-1) {
if (sectiontext.indexOf("{{NRHP row")!=-1) {
NRISOnlyFatalError(2,currentTable,currentRow); // incorrectly formatted table
return;
} else { // must be an empty county
TotalQueried++;
if (TotalQueried==TotalToQuery) DoneQueryingNRIS();
return;
}
}
tabletext=sectiontext.substr(StartIndex,sectiontext.indexOf("\n|}",StartIndex)-StartIndex);
} else { // if not a redirect, default to first table on page
StartIndex=pagetext.indexOf("{{NRHP header");
if (StartIndex==-1) {
NRISOnlyFatalError(1,currentTable,currentRow); // no list found
return;
}
tabletext=pagetext.substr(StartIndex,pagetext.indexOf("\n|}",StartIndex)-StartIndex);
}
// now that tabletext has only relevant table, extract rows
var Rows=[];
var str = "{{";
var start=0;
var commentstart=0;
while (true) {
commentstart=tabletext.indexOf("<!--",start);
start=tabletext.indexOf(str,start);
if (start==-1) break;
while (commentstart<start&&commentstart!=-1) { // skip any commented out rows
start=tabletext.indexOf("-->",commentstart);
commentstart=tabletext.indexOf("<!--",start);
start=tabletext.indexOf(str,start);
}
if (start==-1) break;
var open=1;
var index=start+str.length;
while (open!==0 && index<tabletext.length) { // make sure to find correct matching close brackets for row template
if (tabletext.substr(index,2)=="}}") {
open--;
index++;
} else if (tabletext.substr(index,2)=="{{") {
open++;
index++;
}
index++;
}
var template=tabletext.substr(start,index-start);
regex = new RegExp("{{[\\s]*NRHP row(\\s)*\\|", "g");
if (template.match(regex)!==null) Rows.push(template); // make sure it's the row template and not some other one
start++;
}
for (i=0; i<Rows.length; i++) { // get rid of false positives inside nowiki or pre tags
regex=new RegExp("<[ ]*?(nowiki|pre)[ ]*?>((?!<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>)(.|\\n))*?"+Rows[i].replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")+"(.|\\n)*?<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>", "g");
if (tabletext.match(regex)!==null) {Rows.splice(i,1); i--;}
}
for (var i=0; i<Rows.length; i++) { // extract titles for querying
var ThisRow=Rows[i];
var article=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?.*?[\n|\|]/g);
var blank=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?[\n|\|]/g) ; // default to name param if article
if (article===null||blank!==null) article=ThisRow.match(/\|[ ]*?name[ ]*?=[ ]*?.*?[\n|\|]/g); // blank or missing
// strip param name, final line break
article=article[0].replace(/\|[ ]*?(article|name)[ ]*?=[ ]*?/g,"").replace(/[\n|\|]/g,"").replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,"").trim();
article=decodeURIComponent(article.split("#")[0].trim()); // corrections for weird titles
var refnum=ThisRow.match(/[0-9]{8,9}/g); // also extract refnums
if (refnum!==null) {
if (refnum[0]=="98000562"||refnum[0]=="98000563") refnum[0]="98000562,98000563"; // hard-code troublesome
if (refnum[0]=="01000363"||refnum[0]=="01000364") refnum[0]="01000363,01000364"; // multistate duplicates
if (refnum[0]=="02000529"||refnum[0]=="02000530") refnum[0]="02000529,02000530"; // ...
refnum=refnum[0];
} else {
refnum="missing";
}
var temp=NRISOnlyStructure[currentTable][currentRow].Titles;
temp[temp.length]={"name":article, "refnum":refnum, "exists":true,
"isDab":false, "isTaggedNRISOnly":false, "isNRISOnly":false, "proseSize":0, "numberRefs":0};
NRISOnlyStructure[currentTable][currentRow].Titles=temp;
}
StartIndex=0;
LoadNextNRISOnlyListQuery(StartIndex,currentTable,currentRow);
return;
}
// ready next batch of articles to query
function LoadNextNRISOnlyListQuery(StartIndex,currentTable,currentRow) {
if (StartIndex==NRISOnlyStructure[currentTable][currentRow].Titles.length) { // all queries begun for this list
return;
}
// must have some more rows to query
var TempTitles;
if (NRISOnlyStructure[currentTable][currentRow].Titles.length-StartIndex>50) {
TempTitles=NRISOnlyStructure[currentTable][currentRow].Titles.slice(StartIndex,StartIndex+50);
} else {
TempTitles=NRISOnlyStructure[currentTable][currentRow].Titles.slice(StartIndex);
}
for (var i=0; i<TempTitles.length; i++) { // only extract article names
TempTitles[i]=TempTitles[i].name;
}
StartIndex+=TempTitles.length;
setTimeout(function(){ // short delay to prevent API overload
QueryNRISOnlyCats(TempTitles,StartIndex,currentTable,currentRow);
LoadNextNRISOnlyListQuery(StartIndex,currentTable,currentRow);
}, DefaultQueryPause);
return;
}
// query next batch of articles
function QueryNRISOnlyCats(TempTitles,StartIndex,currentTable,currentRow) {
var TitleList=TempTitles.join("|");
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'categories',
clcategories: 'Category:All disambiguation pages|Category:All articles sourced only to NRIS',
cllimit: 'max',
titles: TitleList,
redirects: 'true'
},
error: function(ArticlejsonObject,status,errorThrown) {ArticlejsonObject.errorThrown=errorThrown;},
complete: function(ArticlejsonObject,status) {
NRISOnlyCatsChecked(ArticlejsonObject,status,TempTitles,StartIndex,currentTable,currentRow);
}
});
return;
}
// parse API response for article query
function NRISOnlyCatsChecked(ArticlejsonObject,status,TempTitles,StartIndex,currentTable,currentRow) {
if (status!="success") {
NewNRISOnlyWarning("Articles "+ArticlejsonObject.errorThrown);
setTimeout(function(){ // try again after delay if rate limit reached
QueryNRISOnlyCats(TempTitles,StartIndex,currentTable,currentRow);
}, 250);
return;
}
// won't get here unless successful
NRISOnlyStructure[currentTable][currentRow].CatsQueried+=TempTitles.length;
var responseText=JSON.parse(ArticlejsonObject.responseText);
var i;
if (responseText.query.normalized) { // normalize any weird titles
for (var n in responseText.query.normalized) {
for (i=0; i<TempTitles.length; i++) {
if (TempTitles[i]==responseText.query.normalized[n].from) TempTitles[i]=responseText.query.normalized[n].to;
}
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) { // also update in main data array
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==responseText.query.normalized[n].from) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].name=responseText.query.normalized[n].to;
}
}
}
}
if (responseText.query.redirects) { // resolve any redirects also
for (var r in responseText.query.redirects) {
for (i=0; i<TempTitles.length; i++) {
if (TempTitles[i]==responseText.query.redirects[r].from) TempTitles[i]=responseText.query.redirects[r].to;
}
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) { // also update in main data array
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==responseText.query.redirects[r].from) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].name=responseText.query.redirects[r].to;
}
}
}
}
// now determine if dab/NRIS-only
for (var page in responseText.query.pages) {
var pagetitle=responseText.query.pages[page].title;
// mark as redlink
if (typeof responseText.query.pages[page].missing!="undefined") {
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) {
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==pagetitle) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].exists=false;
}
}
}
if (responseText.query.pages[page].categories) {
for (var category in responseText.query.pages[page].categories) {
// mark as link to dab
if (responseText.query.pages[page].categories[category].title=="Category:All disambiguation pages") {
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) {
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==pagetitle) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].exists=false; // dab=redlink
NRISOnlyStructure[currentTable][currentRow].Titles[i].isDab=true;
}
}
}
// mark already tagged NRIS-only
if (responseText.query.pages[page].categories[category].title.indexOf("sourced only to NRIS")!=-1) {
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) {
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==pagetitle) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].isTaggedNRISOnly=true;
}
}
}
}
}
}
if (NRISOnlyStructure[currentTable][currentRow].CatsQueried==NRISOnlyStructure[currentTable][currentRow].Titles.length) {
var Titles=[];
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) { // tally up articles to query
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].exists) {
Titles.push(NRISOnlyStructure[currentTable][currentRow].Titles[i].name);
NRISOnlyStructure[currentTable][currentRow].TotalToCheck++;
}
}
// now begin querying article content and checking if NRIS-only
StartIndex=0;
LoadNextNRISOnlyPageQuery(Titles,StartIndex,currentTable,currentRow);
}
}
// load each existing page to check if it should be tagged NRIS-only
function LoadNextNRISOnlyPageQuery(Titles,StartIndex,currentTable,currentRow) {
if (StartIndex==Titles.length) { // all queries begun for this list
if (Titles.length===0) { // some lists have no bluelinks
TotalQueried++;
if (TotalQueried==TotalToQuery) DoneQueryingNRIS();
}
return;
}
// must have some more rows to query
setTimeout(function(){ // short delay to prevent API overload
QueryNextNRISOnlyPage(Titles,StartIndex,currentTable,currentRow);
StartIndex++;
LoadNextNRISOnlyPageQuery(Titles,StartIndex,currentTable,currentRow);
}, DefaultQueryPause);
return;
}
// query the next page
function QueryNextNRISOnlyPage(Titles,StartIndex,currentTable,currentRow) {
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
titles: Titles[StartIndex],
indexpageids: true,
redirects: 'true'
},
error: function(ajaxResponse,status,errorThrown) {
ajaxResponse.errorThrown=errorThrown;
console.log("QueryNextNRISOnlyPage failed title="+Titles[StartIndex],status, errorThrown);
},
complete: function(ajaxResponse,status) {
NRISOnlyPageWikitextFetched(ajaxResponse,status,Titles,StartIndex,currentTable,currentRow);
}
});
return;
}
function getStateStructure() {
// NB it is important that the order of top-level entries in this list match the order of tables in WP:NRHPPROGRESS
var StateStructure=[
["Alabama",
["Jefferson County",["Jefferson County", "Birmingham"]],
["Mobile County",["Mobile County", "Mobile"]]],
["Alaska"],
["Arizona",
["Maricopa County",["Maricopa County", "Phoenix"]],
["Yavapai County",["Yavapai County", "Prescott"]]],
["Arkansas",
["Pulaski County",["Pulaski County", "Little Rock"]]],
["California",
["Los Angeles County",["Los Angeles County", "Los Angeles", "Pasadena"]]],
["Colorado",
["Denver County",["downtown Denver","northeast Denver","southeast Denver","west Denver"]]],
["Connecticut",
["Fairfield County",["Fairfield County","Bridgeport","Greenwich","Stamford"]],
["Hartford County",["Hartford County","Hartford","Southington","West Hartford","Windsor"]],
["Middlesex County",["Middlesex County","Middletown"]],
["New Haven County",["New Haven County","New Haven"]]],
["Delaware",
["New Castle County",["northern New Castle County","southern New Castle County","Wilmington"]]],
["D.C."],
["Florida",
["Miami-Dade County",["Miami-Dade County","Miami"]],
["Hillsborough County",["Hillsborough County","Tampa"]]],
["Georgia"],
["Hawaii",
["Honolulu County",["Oahu","Northwestern Hawaiian Islands"]],
["Maui County",["Maui","Kahoolawe","Lanai","Molokai"]]],
["Idaho"],
["Illinois",
["Cook County",["Cook County","Central Chicago","North Side Chicago","South Side Chicago","West Side Chicago","Evanston"]]],
["Indiana",
["Marion County",["Marion County","Center Township, Marion County"]]],
["Iowa",
["Polk County",["Polk County","Des Moines"]],
["Scott County",["Scott County","Downtown Davenport","east Davenport","west Davenport"]]],
["Kansas"],
["Kentucky",
["Jefferson County",["Jefferson County","Anchorage","Downtown Louisville","The Highlands, Louisville","Old Louisville","Portland, Louisville","Louisville's West End"]]],
["Louisiana"],
["Maine",
["Cumberland County",["Cumberland County","Portland"]]],
["Maryland",
["City of Baltimore",["Central Baltimore","East and Northeast Baltimore","North and Northwest Baltimore","South and Southeast Baltimore","West and Southwest Baltimore"]]],
["Massachusetts",
["Barnstable County",["Barnstable County","Barnstable"]],
["Bristol County",["Bristol County","Fall River","New Bedford","Taunton"]],
["Essex County",["Essex County","Andover","Gloucester","Ipswich","Lawrence","Lynn","Methuen","Salem"]],
["Hampden County",["Hampden County","Springfield"]],
["Middlesex County",["Middlesex County","Arlington","Cambridge","Concord","Framingham","Lexington","Lowell","Marlborough","Medford","Newton","Reading","Sherborn","Somerville","Stoneham","Wakefield","Waltham","Weston","Winchester"]],
["Norfolk County",["Norfolk County","Brookline","Milton","Quincy"]],
["Suffolk County",["Suffolk County","northern Boston","southern Boston"]],
["Worcester County",["Worcester County","northern Worcester County","Southbridge","Uxbridge","eastern Worcester","northwestern Worcester","southwestern Worcester"]]],
["Michigan",
["Wayne County",["Wayne County","Downtown and Midtown Detroit","Detroit"]]],
["Minnesota"],
["Mississippi"],
["Missouri",
["Jackson County",["Jackson County: Downtown Kansas City","Jackson County: Kansas City other"]],
["St. Louis",["Downtown and Downtown West St. Louis","St. Louis north and west of downtown","St. Louis south and west of downtown"]]],
["Montana"],
["Nebraska"],
["Nevada"],
["New Hampshire"],
["New Jersey",
["Bergen County",["Bergen County","Closter","Franklin Lakes","Ridgewood","Saddle River","Wyckoff"]]],
["New Mexico"],
["New York",
["Albany County",["Albany County","Albany"]],
["Dutchess County",["Dutchess County","Poughkeepsie","Rhinebeck"]],
["Erie County",["Erie County","Buffalo"]],
["Monroe County",["Monroe County","Rochester"]],
["Nassau County",["Glen Cove", "Hempstead (town)","North Hempstead (town)","Long Beach", "Oyster Bay (town)"]],
["New York County",["Manhattan below 14th Street","Manhattan from 14th to 59th Streets","Manhattan above 59th to 110th Streets","Manhattan above 110th Street","Manhattan on islands"]],
["Niagara County",["Niagara County","Niagara Falls"]],
["Onondaga County",["Onondaga County","Syracuse"]],
["Suffolk County",["Babylon (town)","Brookhaven (town)","East Hampton (town)","Huntington (town)","Islip (town)","Riverhead (town)","Shelter Island (town)","Smithtown (town)","Southampton (town)","Southold (town)"]],
["Westchester County",["northern Westchester County","southern Westchester County","New Rochelle","Peekskill","Yonkers"]]],
["North Carolina"],
["North Dakota"],
["Ohio",
["Cuyahoga County",["Cuyahoga County","Cleveland"]],
["Erie County",["Erie County","Sandusky"]],
["Franklin County",["Franklin County","Columbus"]],
["Hamilton County",["Hamilton County","downtown Cincinnati","eastern Cincinnati","western Cincinnati"]],
["Montgomery County",["Montgomery County","Dayton"]],
["Summit County",["Summit County","Akron"]]],
["Oklahoma"],
["Oregon",
["Multnomah County",["Multnomah County","North Portland","Northeast Portland","Northwest Portland","Southeast Portland","Southwest Portland"]]],
["Pennsylvania",
["Allegheny County",["Allegheny County","Pittsburgh"]],
["Chester County",["eastern Chester County","northern Chester County","southern Chester County"]],
["Lancaster County",["Lancaster County","Lancaster"]],
["Philadelphia",["Center City, Philadelphia","North Philadelphia","Northeast Philadelphia","Northwest Philadelphia","South Philadelphia","Southwest Philadelphia","West Philadelphia"]]],
["Rhode Island",
["Providence County",["Providence County","Pawtucket","Providence", "Woonsocket"]]],
["South Carolina",
["Charleston County",["Charleston County","Charleston"]],
["Greenville County",["Greenville County","Greenville"]],
["Richland County",["Richland County","Columbia"]],
["York County",["York County","Rock Hill"]]],
["South Dakota"],
["Tennessee"],
["Texas",
["Harris County", ["downtown Houston, Texas", "Houston Heights, Houston, Texas", "inner Harris County, Texas", "outer Harris County, Texas"]]
],
["Utah",
["Salt Lake County",["Salt Lake County","Salt Lake City"]],
["Washington County",["Washington County","Zion National Park"]]],
["Vermont"],
["Virginia"],
["Washington",
["King County",["King County","Seattle"]],
["Pierce County",["Pierce County","Tacoma"]],
["Spokane County",["Spokane County","Spokane"]]],
["West Virginia"],
["Wisconsin",
["Dane County",["Dane County","Madison"]],
["Milwaukee County",["Milwaukee County","Milwaukee"]]],
["Wyoming"],
["Puerto Rico"],
["Guam"],
["Virgin Islands"],
["Northern Mariana Islands"],
["American Samoa"],
["Federated States of Micronesia"],
["Palau"],
["Marshall Islands"],
["U.S. Minor Outlying Islands"]
];
return StateStructure;
}
// parse API response for page wikitext
function NRISOnlyPageWikitextFetched(ajaxResponse,status,Titles,StartIndex,currentTable,currentRow) {
if (status!="success") {
NewNRISOnlyWarning("Articles wikitext "+ajaxResponse.errorThrown);
setTimeout(function(){ // try again after delay if rate limit reached
QueryNextNRISOnlyPage(Titles,StartIndex,currentTable,currentRow);
}, 250);
return;
}
// won't get here unless successful
NRISOnlyStructure[currentTable][currentRow].TotalChecked++;
// now determine prose size, number of references, etc.
var responseText=JSON.parse(ajaxResponse.responseText);
for (var page in responseText.query.pages) {
var pagetext;
try {
pagetext = responseText.query.pages[page].revisions[0]['*'];
} catch (e) {
console.log('NRISOnlyPageWikitextFetched error reading responseText for page '+NRISOnlyStructure[currentTable][currentRow].Link,responseText.query.pages[page],e);
console.log('Current row',NRISOnlyStructure[currentTable][currentRow]);
throw e;
}
var prose=pagetext.replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,""); // strip comments
prose=prose.replace(/<ref[^e](.|[\r\n])*?([ ]*?\/|<\/ref[ ]*?)\>/gi,""); // strip refs
prose=prose.replace(/==[ ]*?External links[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,""); // strip external links section
prose=prose.replace(/==[ ]*?See also[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,""); // strip see also section
prose=prose.replace(/==[ ]*?(References|Notes)[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,""); // strip references section
// strip further reading section
prose=prose.replace(/==[ ]*?(Further|Additional) reading[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,"");
prose=prose.replace(/={2,5}.*?={2,5}/g,""); // strip section titles
// replace wikilinks with displayed text
prose=prose.replace(/\[\[(?![ ]*?Category:|[ ]*?Image:|[ ]*?File:)([^\]]*?\|)?(.*?)\]\]/gi,"$2");
prose=prose.replace(/\[[ ]*?http.*? (.*?)\]/g,"$1"); // replace inline external links with displayed text
prose=prose.replace(/'{2,5}(.*?)'{2,5}/g,"$1"); // replace bold/italic with displayed text
prose=prose.replace(/\[\[[ ]*?Category:.*?\]\]/g,""); // strip categories
prose=prose.replace(/\[\[[ ]*?(Image|File):.*?\]\]/g,""); // strip images
prose=prose.replace(/<[ ]*?gallery(.|\n)*?<[ ]*?\/[ ]*?gallery[ ]*?\>/gi,""); // strip galleries
while(true) { // strip templates
var str="{{";
var start=prose.indexOf(str);
if (start==-1) break;
var open=1;
var index=start+str.length;
while (open!==0 && index<prose.length) {
if (prose.substr(index,2)=="}}") {
open--;
index++;
} else if (prose.substr(index,2)=="{{") {
open++;
index++;
}
index++;
}
prose=prose.replace(prose.substr(start,index-start),"");
}
prose=prose.replace(/{\|(.|\n)*?\|}/g,""); // strip tables
prose=prose.replace(/ /g," "); // replace nbsp with regular space
prose=prose.replace(/<[ ]*?br.*?>/g, "\n"); // replace HTML line break with string line break
prose=prose.replace(/[ \n]+/g," "); // replace multiple spaces/linebreaks with single space
prose=prose.trim();
var i;
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) { // record prose size
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==Titles[StartIndex]) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].proseSize=prose.length;
}
}
var hasSpecialRefs=0;
if (pagetext.indexOf("{{GR")!=-1||pagetext.indexOf("{{sfn")!=-1||pagetext.indexOf("{{Sfn")!=-1) { // these count as refs
hasSpecialRefs=pagetext.match(/{{[ ]*(GR|sfn|Sfn)/g).length;
}
var Refs=pagetext.match(/<ref[^e](.|[\r\n])*?([ ]*?\/|<\/ref[ ]*?)\>/gi);
var comments=pagetext.match(/<\!\-\-(.|[\r\n])*?\-\-\>/g);
if (Refs===null) { // if no refs, skip page (default numberRefs=0)
if (hasSpecialRefs===0) continue;
Refs=[];
}
var l, m;
if (comments!==null) {
for (l=0; l<comments.length; l++) {
var CommentedRefs=comments[l].match(/<ref[^e](.|[\r\n])*?([ ]*?\/|<\/ref[ ]*?)\>/gi);
if (CommentedRefs===null) continue;
for (m=0; m<CommentedRefs.length; m++) {
for (var n=0; n<Refs.length; n++) {
if (Refs[n]==CommentedRefs[m]) {Refs.splice(n,1); n--;}
}
}
}
}
if (Refs.length===0) { // if all refs commented out, skip page (default numberRefs=0)
if (hasSpecialRefs===0) continue;
}
var citesNRIS=false;
for (l=0; l<Refs.length; l++) {
if (Refs[l].indexOf("{{NRISref")!=-1) citesNRIS=true; // check if NRISref is one of the refs
}
var namedRefs=[];
for (l=0; l<Refs.length; l++) { // extract names of refs
var nameOfRef=Refs[l].match(/name[ ]*?=.*?(\/|\>)/gi);
if (nameOfRef===null) {
continue;
} else {
nameOfRef = nameOfRef[0].replace(/("| )/g,'');
nameOfRef = nameOfRef.substr(5,nameOfRef.length-6);
}
namedRefs.push(nameOfRef);
}
var Duplicates=0;
for (l=0; l<namedRefs.length; l++) { // remove duplicate named refs
for (m=l+1; m<namedRefs.length; m++) {
if (namedRefs[m]==namedRefs[l]) {
Duplicates++;
namedRefs.splice(m,1);
m--;
}
}
}
var DistinctRefs = Refs.length-Duplicates;
if (hasSpecialRefs>0) DistinctRefs+=hasSpecialRefs;
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) { // record number of refs
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==Titles[StartIndex]) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].numberRefs=DistinctRefs;
}
}
if (DistinctRefs>1) continue; // not NRIS-only if more than one ref
if (citesNRIS&&DistinctRefs==1) { // if one ref and is NRIS-only, then tag
for (i=0; i<NRISOnlyStructure[currentTable][currentRow].Titles.length; i++) { // record number of refs
if (NRISOnlyStructure[currentTable][currentRow].Titles[i].name==Titles[StartIndex]) {
NRISOnlyStructure[currentTable][currentRow].Titles[i].isNRISOnly=true;
}
}
}
}
if (NRISOnlyStructure[currentTable][currentRow].TotalChecked==NRISOnlyStructure[currentTable][currentRow].TotalToCheck) {
TotalQueried++;
if (TotalQueried==TotalToQuery) DoneQueryingNRIS();
return;
}
}
// keep track of warnings encountered while querying API
function NewNRISOnlyWarning(warning) {
var NewWarning=true;
var i;
for (i=0; i<WarningCount.length; i++) { // check if already encountered error
if (warning==WarningCount[i][0]||WarningCount[i][0]==="") {WarningCount[i][0]=warning; WarningCount[i][1]++; NewWarning=false;}
}
if (NewWarning) WarningCount[WarningCount.length]=[warning,1]; // if new warning, make new entry
var test=0;
for (i=0; i<WarningCount.length; i++) {
test+=WarningCount[i][1];
}
if (test%50===0) DefaultQueryPause++; // for every 50 errors encountered, increase time between each query to throttle speed
}
// these errors require user input; can't just be ignored
function NRISOnlyFatalError(code,currentTable,currentRow) {
var errorArray = ['No county section found for ','No list found for ','Incorrectly formatted list for '];
var retry=confirm(errorArray[code]+NRISOnlyStructure[currentTable][currentRow].Link+"!\n\nCancel=Skip OK=Retry");
if (retry) {
getNRISOnlyListWikitext(currentTable,currentRow);
} else { // if chose to skip, add one to error count
TotalQueried++;
ErrorCount++;
if (TotalQueried==TotalToQuery) DoneQueryingNRIS();
}
return;
}
// update ProgressDiv to let user know what's going on
function UpdateNRISOnlyProgressDiv() {
var ProgressSpan=document.getElementById("ProgressSpan");
var TimeSpan=document.getElementById("TimeSpan");
var TimeRemainingStr;
var PercentQueried=Math.round(TotalQueried/TotalToQuery*1000)/10;
ProgressSpan.innerHTML = "Querying articles in each list... "+TotalQueried+" ("+PercentQueried+"%) of "+TotalToQuery+" lists complete.";
if (TotalQueried>100) {
var CurrentTime=new Date();
var SecondsElapsed = (CurrentTime-InitialTime)/1000;
var Average = SecondsElapsed/TotalQueried;
SecondsElapsed=Math.round(SecondsElapsed);
var MinutesElapsed = 0;
while (SecondsElapsed>=60) {
SecondsElapsed-=60;
MinutesElapsed++;
}
var SecondsRemaining = Math.round(Average*(TotalToQuery-TotalQueried));
var MinutesRemaining = 0;
while (SecondsRemaining>=60) {
SecondsRemaining-=60;
MinutesRemaining++;
}
TimeRemainingStr = "";
if (MinutesRemaining!==0) TimeRemainingStr=MinutesRemaining+" min ";
TimeRemainingStr+=SecondsRemaining+" sec";
var TimeElapsedStr = "";
if (MinutesElapsed!==0) TimeElapsedStr=MinutesElapsed+" min ";
TimeElapsedStr+=SecondsElapsed+" sec";
TimeRemainingStr+=" ("+TimeElapsedStr+" elapsed)";
} else {
TimeRemainingStr="Calculating...";
}
TimeSpan.innerHTML="<br>Estimated time remaining: "+TimeRemainingStr;
if (TotalQueried!=TotalToQuery) { // update ProgressDiv only at regular intervals to prevent CPU overload; stop once done
ProgressDivTimer=setTimeout(function(){
UpdateNRISOnlyProgressDiv();
}, 500);
}
return;
}
// get here once done querying everything
function DoneQueryingNRIS() {
clearTimeout(ProgressDivTimer);
var ProgressSpan=document.getElementById("ProgressSpan");
var TimeSpan=document.getElementById("TimeSpan");
var CurrentTime=new Date();
var SecondsElapsed = (CurrentTime-InitialTime)/1000;
SecondsElapsed=Math.round(SecondsElapsed);
var MinutesElapsed = 0;
while (SecondsElapsed>=60) {
SecondsElapsed-=60;
MinutesElapsed++;
}
var TimeStr=" Time elapsed: ";
if (MinutesElapsed!==0) TimeStr+=MinutesElapsed+" min ";
TimeStr+=SecondsElapsed+" sec";
ProgressSpan.innerHTML="Querying articles in each list... Done!"+TimeStr;
TimeSpan.innerHTML="<br>Compiling data (this could take a while)... ";
setTimeout(function() {CalculateNRISOnlyTotals()},100); // small delay for non-Firefox browsers to update screen
}
// now compile all the data
function CalculateNRISOnlyTotals() {
var StartTime=new Date(); // for calculating computation time
var AllTitles = []; // [name,refnum,list] - includes redlinks; used to find duplicates later (deprecating)
var Dabs = []; // [name,list]
var MissingRefnum = []; // [name,list]
var ExistingTitles = []; // name - only bluelinks
var ToBeTagged = []; // name
var ToBeUntagged = []; // name
var AllNRISOnly = []; // name - includes manually tagged articles
var OneRefNotNRIS = []; // name
var Unreferenced = []; // name
var ShortNRIS = []; // [name,length] - NRIS-only with <325 bytes prose
var ShortNotNRIS = []; // [name,length] - not NRIS-only with <325 bytes prose
var LongNRIS = []; // [name,length] - NRIS-only with >=325 bytes prose
var WorstNotNRIS = []; // [name,length] - 100 smallest non-NRIS-only
var WorstNRIS = []; // [name,length] - 100 smallest NRIS-only
var Duplications = []; // [name,refnum,[list1,..listn]] (deprecating)
var DupList = {}; // holds dup objects keyed by refnum (deprecating Duplications and AllTitles array)
var dup, curstatename;
for (var table=0; table<NRISOnlyStructure.length; table++) {
for (var row=0; row<NRISOnlyStructure[table].length; row++) {
for (var item=0; item<NRISOnlyStructure[table][row].Titles.length; item++) {
var thisItem=NRISOnlyStructure[table][row].Titles[item];
if (thisItem.refnum=="missing") {
MissingRefnum[MissingRefnum.length]=[thisItem.name,NRISOnlyStructure[table][row].Link];
} else {
// TODO deal with comma lists of refnums
if (Object.hasOwn(DupList,thisItem.refnum)) {
dup = DupList[thisItem.refnum];
} else {
dup = {
refnum: thisItem.refnum,
extrarefnums: [],
dupname: thisItem.name,
othernames: [],
loclist: []
};
DupList[thisItem.refnum] = dup;
if (thisItem.name != dup.dupname) {
dup.othernames.push(thisItem.name);
}
}
dup.loclist.push(NRISOnlyStructure[table][row].Link);
AllTitles[AllTitles.length]=[thisItem.name,thisItem.refnum,NRISOnlyStructure[table][row].Link];
}
if (thisItem.isDab) Dabs[Dabs.length]=[thisItem.name,NRISOnlyStructure[table][row].Link];
if (!thisItem.exists) continue; // if redlink, no longer care about it
// now no longer have to check for existence
ExistingTitles.push(thisItem.name);
// tagging
if (thisItem.isNRISOnly&&!thisItem.isTaggedNRISOnly) ToBeTagged.push(thisItem.name);
// only untag if more than one ref; sometimes tag is added manually to NRIS mirrors
if (!thisItem.isNRISOnly&&thisItem.isTaggedNRISOnly&&thisItem.numberRefs>1) ToBeUntagged.push(thisItem.name);
// referencing
if (thisItem.isNRISOnly||(!thisItem.isNRISOnly&&thisItem.isTaggedNRISOnly&&thisItem.numberRefs<=1)) {
// also include manually tagged in output
AllNRISOnly.push(thisItem.name);
}
if (!thisItem.isNRISOnly&&thisItem.numberRefs==1) OneRefNotNRIS.push(thisItem.name);
if (thisItem.numberRefs===0) Unreferenced.push(thisItem.name);
// prose length
if (thisItem.proseSize<325&thisItem.isNRISOnly) ShortNRIS[ShortNRIS.length]=[thisItem.name,thisItem.proseSize];
if (thisItem.proseSize<325&!thisItem.isNRISOnly) ShortNotNRIS[ShortNotNRIS.length]=[thisItem.name,thisItem.proseSize];
if (thisItem.proseSize>=325&thisItem.isNRISOnly) LongNRIS[LongNRIS.length]=[thisItem.name,thisItem.proseSize];
}
}
}
// remove duplicates
var i, j;
for (i=0;i<ExistingTitles.length; i++) {
for (j=i+1;j<ExistingTitles.length; j++) {
if (ExistingTitles[i]==ExistingTitles[j]) {ExistingTitles.splice(j,1);j--;}
}
}
for (i=0;i<ToBeTagged.length; i++) {
for (j=i+1;j<ToBeTagged.length; j++) {
if (ToBeTagged[i]==ToBeTagged[j]) {ToBeTagged.splice(j,1);j--;}
}
}
for (i=0;i<ToBeUntagged.length; i++) {
for (j=i+1;j<ToBeUntagged.length; j++) {
if (ToBeUntagged[i]==ToBeUntagged[j]) {ToBeUntagged.splice(j,1);j--;}
}
}
for (i=0;i<AllNRISOnly.length; i++) {
for (j=i+1;j<AllNRISOnly.length; j++) {
if (AllNRISOnly[i]==AllNRISOnly[j]) {AllNRISOnly.splice(j,1);j--;}
}
}
for (i=0;i<OneRefNotNRIS.length; i++) {
for (j=i+1;j<OneRefNotNRIS.length; j++) {
if (OneRefNotNRIS[i]==OneRefNotNRIS[j]) {OneRefNotNRIS.splice(j,1);j--;}
}
}
for (i=0;i<Unreferenced.length; i++) {
for (j=i+1;j<Unreferenced.length; j++) {
if (Unreferenced[i]==Unreferenced[j]) {Unreferenced.splice(j,1);j--;}
}
}
for (i=0;i<ShortNRIS.length; i++) {
for (j=i+1;j<ShortNRIS.length; j++) {
if (ShortNRIS[i][0]==ShortNRIS[j][0]) {ShortNRIS.splice(j,1);j--;}
}
}
for (i=0;i<ShortNotNRIS.length; i++) {
for (j=i+1;j<ShortNotNRIS.length; j++) {
if (ShortNotNRIS[i][0]==ShortNotNRIS[j][0]) {ShortNotNRIS.splice(j,1);j--;}
}
}
for (i=0;i<LongNRIS.length; i++) {
for (j=i+1;j<LongNRIS.length; j++) {
if (LongNRIS[i][0]==LongNRIS[j][0]) {LongNRIS.splice(j,1);j--;}
}
}
// sort by length
ShortNRIS.sort(function(a,b){if (a[1] < b[1]) return -1; if (a[1] > b[1]) return 1; return 0;});
ShortNotNRIS.sort(function(a,b){if (a[1] < b[1]) return -1; if (a[1] > b[1]) return 1; return 0;});
// extract worst
WorstNRIS=ShortNRIS.slice(0,100);
WorstNotNRIS=ShortNotNRIS.slice(0,100);
// now sort by name
ShortNRIS.sort(function(a,b){if (a[0] < b[0]) return -1; if (a[0] > b[0]) return 1; return 0;});
ShortNotNRIS.sort(function(a,b){if (a[0] < b[0]) return -1; if (a[0] > b[0]) return 1; return 0;});
// sort the other arrays by name
LongNRIS.sort(function(a,b){if (a[0] < b[0]) return -1; if (a[0] > b[0]) return 1; return 0;});
Dabs.sort(function(a,b){if (a[0] < b[0]) return -1; if (a[0] > b[0]) return 1; return 0;});
MissingRefnum.sort(function(a,b){if (a[0] < b[0]) return -1; if (a[0] > b[0]) return 1; return 0;});
// custom sort not needed for these
ExistingTitles.sort();
ToBeTagged.sort();
ToBeUntagged.sort();
AllNRISOnly.sort();
OneRefNotNRIS.sort();
Unreferenced.sort();
var refnum;
var DupCount = 0;
for (refnum in DupList) {
dup = DupList[refnum];
if (dup.loclist.length <= 1) {
delete DupList[refnum];
} else {
DupCount++;
}
}
for (i=0;i<AllTitles.length; i++) {
Duplications[Duplications.length]=[AllTitles[i][0],AllTitles[i][1],[AllTitles[i][2]]]; // [name,refnum,[list]]
for (j=i+1;j<AllTitles.length; j++) {
if (AllTitles[i][1]==AllTitles[j][1]) {
Duplications[Duplications.length-1][2].push(AllTitles[j][2]);
AllTitles.splice(j,1);
j--;
}
}
if (Duplications[Duplications.length-1][2].length==1) Duplications.pop(); // if no other lists added, not duplicate
}
// now dump data to subpages
var TimeSpan=document.getElementById("TimeSpan");
var EditSpan=document.getElementById("EditSpan");
var CurrentTime=new Date();
var SecondsElapsed = (CurrentTime-StartTime)/1000;
SecondsElapsed=Math.round(SecondsElapsed);
var MinutesElapsed = 0;
while (SecondsElapsed>=60) {
SecondsElapsed-=60;
MinutesElapsed++;
}
var TimeStr=" Time elapsed: ";
if (MinutesElapsed!==0) TimeStr+=MinutesElapsed+" min ";
TimeStr+=SecondsElapsed+" sec";
TimeSpan.innerHTML+="Done!"+TimeStr;
EditSpan.innerHTML="<br>Dumping data to subpages of ";
EditSpan.innerHTML+="<a href='http://en.wikipedia.org/wiki/"+OutputBase+"'>"+OutputBase+"</a>... 0 edits made.";
var d=new Date();
var months = ['January','February','March','April','May','June','July','August','September','October','November','December'];
var year=d.getYear();
if (year < 1000) year += 1900;
var DateStr = months[d.getMonth()]+" "+d.getDate()+", "+year;
var subpages=0;
var firstletter=ExistingTitles[0].substr(0,1);
var oldfirstletter;
var basewikitext = "";
var partialwikitext = "=="+firstletter+"==\n{{refbegin|3}}\n";
var k;
for (k=0; k<ExistingTitles.length; k++) {
oldfirstletter = firstletter;
firstletter = ExistingTitles[k].substr(0,1);
if (firstletter!=oldfirstletter) { // must be starting a new subpage, so edit the previous one
partialwikitext+='{{refend}}';
subpages++;
NRISOnlyDump({
title: OutputBase+'/'+oldfirstletter,
text: partialwikitext,
summary:'Generate list of all NRHP articles beginning with '+oldfirstletter+' linked from county lists as of '+DateStr+CurrentOperatorEditSummary
},EditSpan,subpages,"no");
basewikitext+='*[[/'+oldfirstletter+']]\n';
partialwikitext='=='+firstletter+'==\n{{refbegin|3}}\n# <li value="'+parseFloat(k+1)+'">[['+ExistingTitles[k]+']]\n';
} else {
partialwikitext+='# [['+ExistingTitles[k]+']]\n';
}
}
partialwikitext+="{{refend}}";
subpages++;
NRISOnlyDump({ // manually edit final subpage
title: OutputBase+'/'+firstletter,
text: partialwikitext,
summary:'Generate list of all NRHP articles beginning with '+firstletter+' linked from county lists as of '+DateStr
},EditSpan,subpages,"no");
basewikitext+='*[[/'+firstletter+']]\n<hr />*[[/Duplications]]';
subpages++;
NRISOnlyDump({
title: OutputBase,
text: basewikitext,
summary: 'Generate list of all NRHP articles linked from county lists as of '+DateStr+CurrentOperatorEditSummary
},EditSpan,subpages,"no");
// now dump list of duplicates
basewikitext = "{{TOC right}}\n<div id=\"duplicates-div\">\n";
var l;
if (DupCount>0) {
var oldstate="";
var state="";
var prevstate="";
var multiflag=false;
var unknownflag=false;
var firststateflag=true;
var wikitext;
console.log('DupList',DupCount, DupList);
var StateStructure = getStateStructure();
var stateno, dupno=0, dupinstate, cursstatename, processduphere;
var nStates, mostrecentstate, nameregex;
var abbrevloc;
var StateBreakout = {};
for (stateno in StateStructure) {
curstatename = StateStructure[stateno][0];
StateBreakout[curstatename] = {
curcount: 0, // used to count occurrence of this state in the current dup
totalcount: 0, // used to count total number of dup in this state
header: `==${curstatename}==\n<div data-name="dup-${curstatename}" data-type="dup-state">{{refbegin|2}}\n`,
footer: "{{refend}}</div>\n\n",
wikitext: '',
tempwiki: '',
dups: [],
};
}
StateBreakout["Multistate"] = {
curcount: 0,
totalcount: 0,
header: '==Multi-state==\n<div id="multistate" data-name="multistate" data-type="dup-multi">\n{{refbegin|2}}\n',
footer: "{{refend}}\n</div>\n",
wikitext: '',
tempwiki: '',
};
StateBreakout["Unknown"] = {
curcount: 0,
totalcount: 0,
header: '==Unknown==\n<div id="unknown" data-name="unknown" data-type="unknown">\n{{refbegin|2}}\n',
footer: "{{refend}}\n</div>\n",
wikitext: '',
tempwiki: '',
};
for (refnum in DupList) {
// reset some per-state counters before processing next dup
dup = DupList[refnum];
nStates = 0;
for (curstatename in StateBreakout) {
StateBreakout[curstatename].curcount = 0;
StateBreakout[curstatename].tempwiki = '';
}
multiflag = false;
unknownflag = false;
// iterate over dup's locations
// creating lines of this form for each state with more than one entry in the dup list,
// and for the multistate list if there is more than one state:
// # [[name]] (#refnum) <small>(entry1, entry2, ...)</small>
for (l in dup.loclist) {
state=getState(dup.loclist[l]);
StateBreakout[state].curcount++;
if (l>0) {
if (oldstate===""||state==="") {
console.log("Found entry with unknown state: oldstate='"+oldstate+"' state='"+state+"'", dup);
console.log("getState failed on: "+dup.loclist[l]);
unknownflag=true;
StateBreakout["Unknown"].curcount++;
abbrevloc = dup.loclist[l].replace(/National Register of Historic Places listings (i|o)n /g,"");
StateBreakout["Unknown"].tempwiki+=`<span data-type="duploc" data-refnum="${refnum}" data-location="${dup.loclist[l]}">[[${dup.loclist[l]}|${abbrevloc}]]</span>; `;
} else if (state!=oldstate) {
multiflag=true;
nStates++;
}
}
oldstate=state;
// add this location to tempwiki ("[[<entryurl>|<pretty>]]", where <pretty> is just the short name for the entry)
abbrevloc = dup.loclist[l].replace(/National Register of Historic Places listings (i|o)n /g,"");
StateBreakout[state].tempwiki+=`<span data-type="duploc" data-refnum="${refnum}" data-location="${dup.loclist[l]}">[[${dup.loclist[l]}|${abbrevloc}]]</span>; `;
}
for (curstatename in StateBreakout) {
if (curstatename === "Multistate") continue;
if (curstatename === "Unknown") continue;
if (StateBreakout[curstatename].curcount > 1) { // state has more than one location
// append tempwiki for this state to its wikitext
wikitext = StateBreakout[curstatename].tempwiki;
wikitext=wikitext.substr(0,wikitext.length-2); // remove trailing "; "
// remove statenames from links (match ", <statename>]]", replace with "]]")
var nameforregex = curstatename;
if (nameforregex === "D.C.") { // point fix
nameforregex = "Washington, D.C.";
}
nameregex = new RegExp(",\\s*"+nameforregex+"]]","g");
wikitext=wikitext.replace(nameregex,"]]");
wikitext = `# <span data-type="dupentry" data-refnum="${refnum}" data-dupname="${dup.dupname}">[[${dup.dupname}]] (#${refnum}) <small>(${wikitext})</small></span>\n`;
StateBreakout[curstatename].wikitext += wikitext;
StateBreakout[curstatename].totalcount++;
}
}
if (unknownflag && StateBreakout["Unknown"].curcount > 0) {
// append tempwiki to wikitext
wikitext = StateBreakout["Unknown"].tempwiki;
wikitext=wikitext.substr(0,wikitext.length-2); // remove trailing "; "
wikitext = `# <span data-type="dupentry" data-refnum="${refnum}" data-dupname="${dup.dupname}">[[${dup.dupname}]] (#${refnum}) <small>(${wikitext})</small></span>\n`;
StateBreakout["Unknown"].wikitext += wikitext; // append to preceding dup entries
StateBreakout["Unknown"].totalcount++;
}
if (multiflag) {
wikitext = '';
for (l in dup.loclist) {
// add this location to tempwiki ("[[<entryurl>|<pretty>]]", where <pretty> is just the short name for the entry)
// TODO there should be a data tag for state (data-state="<state>")
abbrevloc = dup.loclist[l].replace(/National Register of Historic Places listings (i|o)n /g,"");
wikitext+=`<span data-type="duploc" data-refnum="${refnum}" data-location="${dup.loclist[l]}">[[${dup.loclist[l]}|${abbrevloc}]]</span>; `;
}
wikitext=wikitext.substr(0,wikitext.length-2); // remove trailing "; "
wikitext = `# <span data-type="dupentry" data-refnum="${refnum}" data-dupname="${dup.dupname}">[[${dup.dupname}]] (#${refnum}) <small>(${wikitext})</small></span>\n`;
StateBreakout["Multistate"].wikitext += wikitext; // append to preceding dup entries
StateBreakout["Multistate"].totalcount++;
}
}
console.log('StateBreakout',StateBreakout);
// TODO construct final output:
// 1. all states and state-equivalents which have dups
// 2. Multistate
// 3. Unknowns
// 4. List of states and state-equivalents without dups
var nodupswiki = '';
var statesdiv = '';
for (stateno = 0; stateno < StateStructure.length; stateno++) {
curstatename = StateStructure[stateno][0];
if (StateBreakout[curstatename].totalcount > 0) {
statesdiv += StateBreakout[curstatename].header;
statesdiv += StateBreakout[curstatename].wikitext;
statesdiv += StateBreakout[curstatename].footer;
} else {
nodupswiki += "# [[National Register of Historic Places listings in "+curstatename+"|"+curstatename+"]]\n";
}
}
var nodupsdiv = '';
nodupsdiv += "==No duplicates==\n<div id=\"noduplicates\">\n{{refbegin|2}}\n";
nodupsdiv += nodupswiki;
nodupsdiv += "{{refend}}\n</div>\n";
var multistatediv = "";
if (StateBreakout["Multistate"].totalcount > 0) {
multistatediv += StateBreakout["Multistate"].header;
multistatediv += StateBreakout["Multistate"].wikitext;
multistatediv += StateBreakout["Multistate"].footer;
}
var unknowndiv = "";
if (StateBreakout["Unknown"].totalcount > 0) {
multistatediv += StateBreakout["Unknown"].header;
multistatediv += StateBreakout["Unknown"].wikitext;
multistatediv += StateBreakout["Unknown"].footer;
}
var finaldupwiki = "{{TOC right}}\n<div id=\"duplicates-div\">\n"+statesdiv+multistatediv+unknowndiv+nodupsdiv+"</div>\n";
console.log('Final Duplicates wikitext (new)',finaldupwiki);
NRISOnlyDump({
title: OutputBase+'/Duplications',
text: finaldupwiki,
summary:'Generate list of all NRHP articles duplicated across county lines (new version) as of '+DateStr+CurrentOperatorEditSummary
},EditSpan,subpages,"yes");
var multistate="==Multi-state==\n<div id=\"multistate\">\n{{refbegin|2}}\n";
var unknown="==Unknown==\n<div id=\"multistate\">\n{{refbegin|2}}\n";
for (k=0; k<Duplications.length; k++) {
partialwikitext="# [["+Duplications[k][0]+"]] (#"+Duplications[k][1]+") <small>(";
multiflag=false;
unknownflag=false;
for (l=0; l<Duplications[k][2].length; l++) {
state=getState(Duplications[k][2][l]);
if (l>0) {
if (oldstate===""||state==="") {
console.log("Found entry with unknown state: oldstate='"+oldstate+"' state='"+state+"'", Duplications[k]);
console.log("getState failed on: "+Duplications[k][2][l]);
unknownflag=true;
} else if (state!=oldstate) {
multiflag=true;
}
}
oldstate=state;
partialwikitext+="[["+Duplications[k][2][l]+"|";
partialwikitext+=Duplications[k][2][l].replace(/National Register of Historic Places listings (i|o)n /g,"")+"]]; ";
}
partialwikitext=partialwikitext.substr(0,partialwikitext.length-2);
partialwikitext+=")</small>\n";
if (unknownflag) {
unknown+=partialwikitext;
} else if (multiflag) {
multistate+=partialwikitext;
} else {
if (state!=prevstate) {
if (firststateflag) {
basewikitext+="=="+state+"==\n{{refbegin|2}}\n";
firststateflag=false;
} else {
basewikitext+="{{refend}}\n\n=="+state+"==\n{{refbegin|2}}\n";
}
}
var smalltext=partialwikitext.match(/<small>(.)*?<\/small>/g)[0];
var newsmalltext=smalltext.replace(/,[a-zA-Z .,]*?]]/g,"]]"); // remove state names from dupes inside single state
partialwikitext=partialwikitext.replace(smalltext,newsmalltext);
basewikitext+=partialwikitext;
prevstate=state;
}
}
basewikitext+="{{refend}}\n</div>";
if (multistate!="==Multi-state==\n<div id=\"multistate\">\n{{refbegin|2}}\n") {
basewikitext+="\n\n"+multistate+"\n{{refend}}\n</div>";
}
if (unknown!="==Unknown==\n<div id=\"unknown\">\n{{refbegin|2}}\n") {
basewikitext+="\n\n"+unknown+"\n{{refend}}\n</div>";
}
}
/*
NRISOnlyDump({
title: OutputBase+'/Duplications',
text: basewikitext,
summary:'Generate list of all NRHP articles duplicated across county lines (old version) as of '+DateStr+CurrentOperatorEditSummary
},EditSpan,subpages,"yes");
*/
// now dump NRIS-only data
var OutputPage=NRISOnlyFile;
EditSpan.innerHTML+="<br>Now outputting data to <a href='http://en.wikipedia.org/wiki/"+OutputPage+"'>"+OutputPage+"</a>... ";
basewikitext="{{TOC right}}\n";
if (ToBeTagged.length>0) {
basewikitext+="==Articles that need to be tagged==\n<div id=\"tobetagged-div\">\n{{refbegin|3}}\n";
for (k=0; k<ToBeTagged.length; k++) {
basewikitext+='# [['+ToBeTagged[k]+']]\n';
}
basewikitext+="{{refend}}</div>\n\n";
}
if (ToBeUntagged.length>0) {
basewikitext+="==Articles that need to be untagged==\n<div id=\"tobeuntagged-div\">\n{{refbegin|3}}\n";
for (k=0; k<ToBeUntagged.length; k++) {
basewikitext+='# [['+ToBeUntagged[k]+']]\n';
}
basewikitext+="{{refend}}</div>\n\n";
}
if (Dabs.length>0) {
basewikitext+="==Links to disambiguation pages==\n{{refbegin|3}}\n";
for (k=0; k<Dabs.length; k++) {
basewikitext+='# [['+Dabs[k][0]+']] ([['+Dabs[k][1]+'|';
basewikitext+=Dabs[k][1].replace(/National Register of Historic Places listings (i|o)n /g,"")+']])\n';
}
basewikitext+="{{refend}}\n\n";
}
if (MissingRefnum.length>0) {
basewikitext+="==Missing refnum in county list==\n{{refbegin|3}}\n";
for (k=0; k<MissingRefnum.length; k++) {
basewikitext+='# [['+MissingRefnum[k][0]+']] ([['+MissingRefnum[k][1]+'|';
basewikitext+=MissingRefnum[k][1].replace(/National Register of Historic Places listings (i|o)n /g,"")+']])\n';
}
basewikitext+="{{refend}}\n\n";
}
if (Unreferenced.length>0) {
basewikitext+="==Articles with no references==\n<div id=\"unreferenced-div\">\n{{refbegin|3}}\n";
for (k=0; k<Unreferenced.length; k++) {
basewikitext+='# [['+Unreferenced[k]+']]\n';
}
basewikitext+="{{refend}}</div>\n\n";
}
if (OneRefNotNRIS.length>0) {
basewikitext+="==Articles with only one source that is not NRIS==\n{{refbegin|3}}\n";
for (k=0; k<OneRefNotNRIS.length; k++) {
basewikitext+='# [['+OneRefNotNRIS[k]+']]\n';
}
basewikitext+="{{refend}}\n\n";
}
NRISOnlyDump({
title: OutputPage,
text: basewikitext,
summary:'Generate list of all NRHP articles to be tagged/untagged with [[Template:NRIS-only]] as of '+DateStr+CurrentOperatorEditSummary
},EditSpan,subpages,"");
// now substubs
OutputPage=SubstubsFile;
basewikitext="";
if (WorstNRIS.length>0) {
basewikitext+="==Smallest 100 NRIS-only articles by prose size==\n{{refbegin|3}}\n";
for (k=0; k<WorstNRIS.length; k++) {
basewikitext+='# [['+WorstNRIS[k][0]+']] ('+WorstNRIS[k][1]+' bytes)\n';
}
basewikitext+="{{refend}}\n\n";
}
if (WorstNotNRIS.length>0) {
basewikitext+="==Smallest 100 articles by prose size NOT tagged NRIS-only==\n{{refbegin|3}}\n";
for (k=0; k<WorstNotNRIS.length; k++) {
basewikitext+='# [['+WorstNotNRIS[k][0]+']] ('+WorstNotNRIS[k][1]+' bytes)\n';
}
basewikitext+="{{refend}}\n\n";
}
if (ShortNRIS.length>0||ShortNotNRIS.length>0) {
basewikitext+="==Articles with less than 325 bytes of prose==\n";
if (ShortNotNRIS.length>0) {
basewikitext+="===Not tagged NRIS-only===\n{{refbegin|3}}\n";
for (k=0; k<ShortNotNRIS.length; k++) {
basewikitext+='# [['+ShortNotNRIS[k][0]+']] ('+ShortNotNRIS[k][1]+' bytes)\n';
}
basewikitext+="{{refend}}\n\n";
if (ShortNRIS.length>0) basewikitext+="===Tagged NRIS-only===\n";
}
if (ShortNRIS.length>0) {
basewikitext+="{{refbegin|3}}\n";
for (k=0; k<ShortNRIS.length; k++) {
basewikitext+='# [['+ShortNRIS[k][0]+']] ('+ShortNRIS[k][1]+' bytes)\n';
}
basewikitext+="{{refend}}\n\n";
}
}
if (LongNRIS.length>0) {
basewikitext+="==Articles tagged NRIS-only with more than 325 bytes of prose==\n{{refbegin|3}}\n";
for (k=0; k<LongNRIS.length; k++) {
basewikitext+='# [['+LongNRIS[k][0]+']] ('+LongNRIS[k][1]+' bytes)\n';
}
basewikitext+="{{refend}}\n\n";
}
NRISOnlyDump({
title: OutputPage,
text: basewikitext,
summary:'Generate list of all minimal (<325 bytes) NRHP stubs as of '+DateStr+CurrentOperatorEditSummary
},EditSpan,subpages,"");
// now all NRIS-only
OutputPage=NRISOnlyFile+"/All";
basewikitext = "{{TOC right}}\n<div id=\"NRISOnly-div\">\n";
if (AllNRISOnly.length>0) {
firstletter=AllNRISOnly[0].substr(0,1);
basewikitext += "=="+firstletter+"==\n{{refbegin|3}}\n";
for (k=0; k<AllNRISOnly.length; k++) {
oldfirstletter = firstletter;
firstletter = AllNRISOnly[k].substr(0,1);
if (firstletter!=oldfirstletter) {
basewikitext+='{{refend}}\n=='+firstletter+'==\n{{refbegin|3}}\n# <li value="';
basewikitext+=parseFloat(k+1)+'">[['+AllNRISOnly[k]+']]\n';
} else {
basewikitext+='# [['+AllNRISOnly[k]+']]\n';
}
}
basewikitext+="{{refend}}</div>";
}
NRISOnlyDump({
title: OutputPage,
text: basewikitext,
summary:'Generate list of all NRHP articles sourced only to NRIS as of '+DateStr+CurrentOperatorEditSummary
},EditSpan,subpages,"NRIS");
}
// gather information about duplicates from output (old version)
function CheckDuplicates() {
var ProgressDiv=document.getElementById("ProgressDiv");
ProgressDiv.innerHTML="Checking multi-state duplicates...";
var MultiSpan = document.createElement("span");
ProgressDiv.appendChild(MultiSpan);
var temp;
var multistate=[];
var links, refnum,titles;
var li=document.getElementById("multistate").getElementsByClassName("refbegin")[0].getElementsByTagName("li");
var i, j, k, l;
for (k=0; k<li.length; k++) {
links=li[k].getElementsByTagName("a");
refnum=li[k].innerHTML.match(/[0-9]{8,9}/)[0];
titles=[refnum]; // make refnum element 0; title=1; county lists=2,3,4,...
for (l=0; l<links.length; l++) {
temp=links[l].href.replace(/https:\/\/en\.wikipedia\.org\/w(iki)?\/(index\.php\?title=)?/,"").split("&")[0];
titles.push(decodeURIComponent(temp).replace(/_/g," "));
}
multistate[multistate.length]=titles;
}
var wikitext="==National==\nThe following listings are included in two or more lists which are located in different states.";
wikitext+="\n{| class=\"wikitable sortable\" width=100%\n! {{NRHP color}} width=30% | Site\n! {{NRHP color}} width=30% | ";
wikitext+="Lists\n! {{NRHP color}} width=10% | Number of Duplicates\n! {{NRHP color}} width=30% colspan=7 | Stats\n";
var switchtabletext="<includeonly>{{#tag:ref|[[WP:NRHPPROGRESS/Duplicates#{{{1}}}{{#if:{{{statewide|}}}| Statewide}}";
switchtabletext+="{{!}}Click here]] for {{{1}}} duplicate information.}}{{#ifeq:{{{1}}}|National|</th>|{{#ifeq:{{{1}}}|";
switchtabletext+="{{First word|{{{1}}}|sep=,}}|</th>|</td><td>[[{{Remove first word|{{{1}}}|sep=,}}]]</td>}}}}<td>{{#switch:";
switchtabletext+="{{{1}}}\n";
var Total=0;
var TotalIllustrated=0;
var TotalArticled=0;
var TotalStub=0;
var TotalStartPlus=0;
var TotalUntagged=0;
var TotalUnassessed=0;
var TotalNRISOnly=0;
for (i=0; i<multistate.length; i++) {
MultiSpan.innerHTML = " "+i+" of "+multistate.length+" complete...";
wikitext+="|-\n| [["+multistate[i][1]+"]] (#"+multistate[i][0]+")\n|\n";
temp=[multistate[i][1],0]; // 0=not duplicated inside single state by default
multistate[i][1]=temp;
var oldstate="";
var state="";
var totalstates=0;
for (j=2; j<multistate[i].length; j++) {
wikitext+="*[["+multistate[i][j]+"|"+multistate[i][j].replace(/National Register of Historic Places listings (i|o)n /g,"");
wikitext+="]]\n";
state=getState(multistate[i][j]);
if (state!=oldstate) {
totalstates++;
} else {
if (multistate[i][1][1]===0) { // tag as being duplicated inside one state (and record which state) for later use
multistate[i][1][1]=state;
} else {
multistate[i][1][multistate[i][1].length]=state;
}
}
oldstate=state;
}
var duplications=totalstates-1;
Total+=duplications;
wikitext+="| align=center | "+duplications+"\n| colspan=7 | ";
var StatsStr=getDuplicateStats(multistate[i]);
if (StatsStr.indexOf("unarticled")==-1) TotalArticled+=duplications;
if (StatsStr.indexOf(", illustrated")!=-1) TotalIllustrated+=duplications;
if (StatsStr.indexOf("Stub-class")!=-1) TotalStub+=duplications;
if (StatsStr.indexOf("NRIS-only")!=-1) TotalNRISOnly+=duplications;
if (StatsStr.indexOf("Start+")!=-1) TotalStartPlus+=duplications;
if (StatsStr.indexOf("unassessed")!=-1) TotalUnassessed+=duplications;
if (StatsStr.indexOf("untagged")!=-1) TotalUntagged+=duplications;
wikitext+=StatsStr+"\n";
}
wikitext+="|-\n! colspan=2 | Total\n! "+Total+"\n! "+TotalIllustrated+"\n! "+TotalArticled+"\n! "+TotalStub+"\n! ";
wikitext+=TotalNRISOnly+"\n! "+TotalStartPlus+"\n! "+TotalUnassessed+"\n! "+TotalUntagged+"\n|}\n\n";
switchtabletext+="|National="+Total+"</td><td>"+TotalIllustrated+"</td><td>-</td><td>"+TotalArticled+"</td><td>-</td><td>";
switchtabletext+=TotalStub+"</td><td>"+TotalNRISOnly+"</td><td>"+TotalStartPlus+"</td><td>-</td><td>"+TotalUnassessed;
switchtabletext+="</td><td>"+TotalUntagged+"</td><td>-</td>\n";
MultiSpan.innerHTML = " Complete!";
for (i=0; i<multistate.length; i++) {
if (multistate[i][1][1]===0) {multistate.splice(i,1); i--;} // get rid of those not duplicated inside single states
}
var StateSpan = document.createElement("span");
ProgressDiv.appendChild(StateSpan);
StateSpan.inerHTML="test";
var CountySpan = document.createElement("span");
ProgressDiv.appendChild(CountySpan);
CountySpan.inerHTML="test";
var ThisStateSpan = document.createElement("span");
ProgressDiv.appendChild(ThisStateSpan);
ThisStateSpan.inerHTML="test";
var StateStructure = getStateStructure();
var states=document.getElementById("duplicates-div").getElementsByClassName("refbegin");
var currentState=0;
var result, inThisState;
for (i=0; i<StateStructure.length; i++) {
StateSpan.innerHTML = "<br />Now working on individual states... "+i+" of "+StateStructure.length+" complete...";
var thisstate=[];
li=states[currentState].getElementsByTagName("li");
for (j=0; j<li.length; j++) {
links=li[j].getElementsByTagName("a");
refnum=li[j].innerHTML.match(/[0-9]{8,9}/)[0];
titles=[refnum]; // make refnum element 0; title=1; county lists=2,3,4,...
for (l=0; l<links.length; l++) {
temp=links[l].href.replace(/https:\/\/en\.wikipedia\.org\/w(iki)?\/(index\.php\?title=)?/,"").split("&")[0];
titles.push(decodeURIComponent(temp).replace(/_/g," "));
}
thisstate[thisstate.length]=titles;
}
var stateName=getState(thisstate[0][2]);
wikitext+="=="+StateStructure[i][0]+"==\n";
console.log("CheckDuplicates: looking at state",stateName);
console.log("CheckDuplicates: thisstate",thisstate[thisstate.length-1]);
while (i<StateStructure.length&&stateName!=StateStructure[i][0]) {
// if the state we're looking at isn't the next one in the structure array
var tempstate=[];
for (j=0; j<multistate.length; j++) {
inThisState=0;
for (k=1; k<multistate[j][1].length; k++) { // check if any multistate duplicates are
if (multistate[j][1][k]==StateStructure[i][0]) inThisState=1; // duplicated in this state
}
if (inThisState==1) {
temp=[];
for (k=0; k<multistate[j].length; k++) { // make temp static
temp[k]=multistate[j][k];
}
for (k=2; k<temp.length; k++) {
if (getState(temp[k])!=StateStructure[i][0]) {temp.splice(k,1); k--} // only pick out those in this state
}
tempstate[tempstate.length] = [temp[0],temp[1][0],temp[2],temp[3]]; // refnum, title, list1, list2
for (k=4; k<temp.length; k++) {
tempstate[tempstate.length-1].push(temp[k]); // list3, list4,...
}
}
}
result=GenerateDuplicateTable(StateStructure,i,tempstate,stateName,CountySpan,ThisStateSpan);
wikitext+=result[0];
switchtabletext+=result[1];
i++; // look at next state in StateStructure and repeat until we find the one we're looking at
StateSpan.innerHTML = "<br />Now working on individual states... "+i+" of "+StateStructure.length+" complete...";
if (i!=StateStructure.length) wikitext+="=="+StateStructure[i][0]+"==\n";
}
if (i!=StateStructure.length) {
// now we know the state we're looking at has the duplications in thisstate array
for (j=0; j<multistate.length; j++) {
inThisState=0;
for (k=1; k<multistate[j][1].length; k++) { // check if any multistate duplicates are also
if (multistate[j][1][k]==stateName) inThisState=1; // duplicated in the current state
}
if (inThisState==1) {
temp=[];
for (k=0; k<multistate[j].length; k++) { // make temp static
temp[k]=multistate[j][k];
}
for (k=2; k<temp.length; k++) {
if (getState(temp[k])!=stateName) {temp.splice(k,1); k--} // only pick out those in this state
}
thisstate[thisstate.length] = [temp[0],temp[1][0],temp[2],temp[3]]; // refnum, title, list1, list2
for (k=4; k<temp.length; k++) {
thisstate[thisstate.length-1].push(temp[k]); // list3, list4,...
}
}
}
result=GenerateDuplicateTable(StateStructure,i,thisstate,stateName,CountySpan,ThisStateSpan);
wikitext+=result[0];
switchtabletext+=result[1];
CountySpan.innerHTML="<br /> ";
ThisStateSpan.innerHTML="<br /> ";
if (currentState<states.length-1) currentState++; // next state on the page
}
}
CountySpan.innerHTML="";
ThisStateSpan.innerHTML="";
StateSpan.innerHTML = "<br />Now working on individual states... Complete!";
ProgressDiv.innerHTML+="<br />Now outputting data to ";
ProgressDiv.innerHTML+="<a href='http://en.wikipedia.org/wiki/WP:NRHPPROGRESS/Duplicates'>WP:NRHPPROGRESS/Duplicates</a>...";
switchtabletext+="|error</td><td>error</td><td>-</td><td>error</td><td>-</td><td>error</td><td>error</td><td>error</td><td>";
switchtabletext+="-</td><td>error</td><td>error</td><td>-</td>\n}}</includeonly><noinclude>\n";
var TOCtext="__NOTOC__\n{{anchor|top}}\n{| style=\"margin:.5em; border:1px solid #000; padding:0;";
TOCtext+=" text-align:center\"\n| '''[[#National|National]]'''\n<hr />\n";
for (i=0; i<StateStructure.length; i++) {
TOCtext+="[[#"+StateStructure[i][0];
if (StateStructure[i].length>1) TOCtext+=" Statewide";
TOCtext+="|"+StateStructure[i][0]+"]] ";
for (j=1; j<StateStructure[i].length; j++) {
if (j==1) TOCtext+="(";
TOCtext+="[[#"+StateStructure[i][j][0]+", "+StateStructure[i][0]+"|";
TOCtext+=StateStructure[i][j][0].replace(", "+StateStructure[i][0],"").replace(" County","").replace("City of ","");
TOCtext+="]], ";
}
if (StateStructure[i].length>1) {
TOCtext=TOCtext.substr(0,TOCtext.length-2); // remove final comma
TOCtext+=")";
}
if (StateStructure[i][0]=="Wyoming") TOCtext+="\n<hr />\n";
else if (i!=StateStructure.length-1) TOCtext+=" – ";
}
TOCtext+="\n|}\n\n";
wikitext=switchtabletext+TOCtext+wikitext+"</noinclude>";
var d=new Date();
var months = ['January','February','March','April','May','June','July','August','September','October','November','December'];
var year=d.getYear();
if (year < 1000) year += 1900;
var DateStr = months[d.getMonth()]+" "+d.getDate()+", "+year;
// edit page with total wikitext
NRISOnlyDump({
title: 'Wikipedia:WikiProject National Register of Historic Places/Progress/Duplicates',
text: wikitext,
summary:'Generate statistics about duplicates in NRHP lists as of '+DateStr+CurrentOperatorEditSummary
},ProgressDiv,0,"duplicates");
}
// gather information about duplicates from output (new version)
/*
Document structure is as follows:
<div id="duplicates-div">
<div data-type="dup-state" data-name="dup-<state>">duplist</div> for each state
<div id="multistate" data-name="multistate" data-type="dup-multi">duplist</div>
<div id="unknown" data-name="unknown" data-type="dup-unknown">duplist</div> (optional, normally missing)
<div id="noduplicates">...</div> section with bulleted list of states with no in-state duplicates
</div>
Each duplist section has multiple entries like this:
<span data-type="dupentry" data-refnum="<refnum>" data-dupname="<name>">...loclist...</span>
Each dupentry has wikitext formatting the entry, including multiple entries of duploc:
<span data-type="duploc" data-refnum="refnum" data-location="<wikipage>">...</span>
The lists for each state include entries that may be also be included in other states, and in the multistate list.
The state-specific lists include only locations in that state, while the multistate entry
will have all locations listed. The data-location attribute contains the wiki name of the page where
the reference was found.
Unlike original method, there is no guarantee that multistate listings are organized in some form by state, or
that listings within states are meaningfully grouped. Hash tables should be used to manage entries keyed by refnum.
Basic algorithm:
1. Process multistate list. Similar to old code, but not needing to do anything state-specific.
*/
function CheckDuplicatesNew() {
var ProgressDiv=document.getElementById("ProgressDiv");
ProgressDiv.innerHTML="Checking multi-state duplicates...";
var MultiSpan = document.createElement("span");
ProgressDiv.appendChild(MultiSpan);
var errorsOccurred = 0;
var Total=0;
var TotalIllustrated=0;
var TotalArticled=0;
var TotalStub=0;
var TotalStartPlus=0;
var TotalUntagged=0;
var TotalUnassessed=0;
var TotalNRISOnly=0;
var dupnum, dupentry, duplist, wikitext, refnum, dupname, duplocations, statobj, statobjStr;
// Multistate processing
var ms_header = ''; // contains header for multistate table
ms_header="==National==\nThe following listings are included in two or more lists which are located in different states.";
ms_header+="\n{| class=\"wikitable sortable\" width=100%\n! {{NRHP color}} width=30% | Site\n! {{NRHP color}} width=30% | ";
ms_header+="Lists\n! {{NRHP color}} width=10% | Number of Duplicates\n! {{NRHP color}} width=30% colspan=7 | Stats\n";
var ms_table = ''; // contains entries in multistate table
var ms_footer = ''; // contains footer for multistate table (calculated at end)
var switch_header = ''; // contains header for javascript switch for fetching data
switch_header+="<includeonly>{{#tag:ref|[[WP:NRHPPROGRESS/Duplicates#{{{1}}}{{#if:{{{statewide|}}}| Statewide}}";
switch_header+="{{!}}Click here]] for {{{1}}} duplicate information.}}{{#ifeq:{{{1}}}|National|</th>|{{#ifeq:{{{1}}}|";
switch_header+="{{First word|{{{1}}}|sep=,}}|</th>|</td><td>[[{{Remove first word|{{{1}}}|sep=,}}]]</td>}}}}<td>{{#switch:";
switch_header+="{{{1}}}\n";
var switch_table = ''; // contains entries in the switch for each entity
var switch_footer = ''; // contains footer for javascript switch for fetching data
switch_footer+="|error</td><td>error</td><td>-</td><td>error</td><td>-</td><td>error</td><td>error</td><td>error</td><td>";
switch_footer+="-</td><td>error</td><td>error</td><td>-</td>\n}}</includeonly><noinclude>\n";
var loc_table = ''; // contains accumulated wikitext for each location in that entry
duplist = document.querySelectorAll('#multistate span[data-type="dupentry"]');
duplist = Array.from(duplist);
var dupcount, dupmax, l, assessment;
dupcount = 0;
dupmax = duplist.length;
console.log('CheckDuplicatesNew multistate count',dupmax);
var stateSet, state, loc, loc_short, stateCount, stateDupCount;
for (dupnum in duplist) {
MultiSpan.innerHTML = ` ${dupcount} of ${dupmax} complete...`;
dupcount++;
dupentry = duplist[dupnum];
dupname = dupentry.dataset.dupname;
refnum = dupentry.dataset.refnum;
duplocations = dupentry.querySelectorAll('[data-type="duploc"]'); // returns NodeList
duplocations = Array.from(duplocations); // cvt to array
duplocations = duplocations.map((x) => x.dataset.location);
try {
statobj = getDuplicateStatsObject3(refnum, dupname, duplocations);
} catch (e) {
console.log('Error in getDuplicateStatesObject',refnum,e);
statobj = null;
errorsOccurred++;
continue;
}
console.log(`CheckDuplicatesNew multistate ${dupcount}/${dupmax}`,refnum,dupentry,duplocations,statobj);
loc_table = '';
stateSet = new Set();
for (l in duplocations) {
loc = duplocations[l];
state = getState(loc);
stateSet.add(state);
/* *[[full_list_name|reduced_list_name]]
in this case, full_list_name is loc
wikitext+="*[["+multistate[i][j]+"|"+multistate[i][j].replace(/National Register of Historic Places listings (i|o)n /g,"");
wikitext+="]]\n";
*/
loc_short = loc.replace(/National Register of Historic Places listings (i|o)n /g,"");
loc_table += `*[[${loc}|${loc_short}]]\n`;
}
stateCount = stateSet.size;
stateDupCount = stateCount-1;
console.log('States for this dup',stateCount,stateSet);
Total += stateDupCount;
// update aggregators for totals row
if (statobj.articled.indexOf("unarticled")==-1) TotalArticled+=stateDupCount;
if (statobj.illustrated=="illustrated") TotalIllustrated+=stateDupCount;
if (statobj.nrisonly != null && statobj.nrisonly.indexOf("NRIS-only")!=-1) TotalNRISOnly+=stateDupCount;
if (statobj.assessment != null) {
if (statobj.assessment.indexOf("Start+")!=-1) TotalStartPlus+=stateDupCount;
if (statobj.assessment.indexOf("unassessed")!=-1) TotalUnassessed+=stateDupCount;
if (statobj.assessment.indexOf("untagged")!=-1) TotalUntagged+=stateDupCount;
}
// render table entry for this dup
statobjStr = DupStatsToString(statobj);
ms_table += `<tr><td>[[${dupname} (${refnum})</td><td>\n${loc_table}</td><td>${stateDupCount}</td><td>${statobjStr}</td></tr>`;
}
console.log('Total number of duplications '+Total+' from '+dupmax+' entries');
// render final table section for multistate
ms_footer = `\n|-\n! colspan=2 | Total ! ${Total} ! ${TotalArticled} ! ${TotalStub} ! ${TotalNRISOnly} ! ${TotalStartPlus} + ! ${TotalUnassessed} ! ${TotalUntagged}\n|}\n\n`;
ms_table = ms_header + ms_table + ms_footer;
console.log('ms_table',ms_table);
// render switch table for multistate
switch_table += `|National=<td>${Total}</td><td>${TotalIllustrated}</td><td>-</td><td>${TotalArticled}</td><td>-</td>`;
switch_table += `<td>${TotalStub}</td><td>${TotalNRISOnly}</td><td>${TotalStartPlus}</td><td>-</td>`;
switch_table += `<td>${TotalUnassessed}</td><td>${TotalUntagged}</td><td>-</td>\n`;
console.log('switch_table',switch_table);
MultiSpan.innerHTML = " Complete!";
// iterate over states
var StateSpan = document.createElement("span");
ProgressDiv.appendChild(StateSpan);
var CountySpan = document.createElement("span");
ProgressDiv.appendChild(CountySpan);
var ThisStateSpan = document.createElement("span");
ProgressDiv.appendChild(ThisStateSpan);
var ss = getStateStructure();
var scount = 0, smax = ss.length;
var stateName, countyName;
for (var s in ss) {
/* s is an array of the form [ <statename> [, <optionalcountyarrays> ]]
optionalcountyarrays are zero or more of the form [ <countyname> [, <optionalsubcountylist> ]]
optionalsubcountylist is an optional list of subcounty names
TODO for each state and subentity, we need a structure keyed by refnums similar to multistate
*/
stateName = s[0];
StateSpan.innerHTML = `<br />Now working on ${stateName}... ${scount} of ${smax} complete...`;
scount++;
// fetch dups for this state
duplist = document.querySelectorAll(`div[data-name="dup-${stateName}] span[data-type="dupentry"]`);
duplist = Array.from(duplist);
dupcount = 0;
dupmax = duplist.length;
console.log(`State ${stateName}: number of duplicates ${dupmax}`, duplist);
}
}
var TaggingState=null;
// tag/untag pages with the bot
function TagNRISOnly() {
var tobetagged=document.getElementById('tobetagged-div');
var tobeuntagged=document.getElementById('tobeuntagged-div');
var ProgressDiv = document.getElementById('ProgressDiv');
var HiddenSpan=document.createElement("span");
HiddenSpan.setAttribute("id", "HiddenSpan");
HiddenSpan.setAttribute("style", "display:none");
ProgressDiv.appendChild(HiddenSpan);
var d=new Date();
var months = ['January','February','March','April','May','June','July','August','September','October','November','December'];
var year=d.getYear();
if (year < 1000) year += 1900;
var DateStr = months[d.getMonth()]+" "+year;
TaggingState = {
TagProgress: null,
TagLinks: null,
TagSections: null,
TagTotal: 0,
TagSkipped: 0,
TagTagged: 0,
TagDone: 0,
UntagProgress: null,
UntagLinks: null,
UntagSections: null,
UntagTotal: 0,
UntagSkipped: 0,
UntagUntagged: 0,
UntagDone: 0
};
var i, j, total;
var wikitext, open, start, multipleIssues;
var full, index, oldsyntax, result;
if (tobetagged!==null) {
ProgressDiv.innerHTML = "Tagging articles that need to be tagged...";
TaggingState.TagProgress = document.createElement("span");
TaggingState.TagProgress.setAttribute("id", "ProgressSpan");
ProgressDiv.appendChild(TaggingState.TagProgress);
TaggingState.TagProgress.innerHTML="<br />";
TaggingState.TagSections=tobetagged.getElementsByClassName('refbegin');
TaggingState.TagTotal=0;
for (i=0; i<TaggingState.TagSections.length; i++) {
TaggingState.TagLinks=TaggingState.TagSections[i].getElementsByTagName('a');
TaggingState.TagTotal+=TaggingState.TagLinks.length;
}
TaggingState.TagProgress.innerHTML="<br />0 of "+TaggingState.TagTotal+" articles examined: 0 tagged, 0 skipped...";
for (i=0; i<TaggingState.TagSections.length; i++) {
links=TaggingState.TagSections[i].getElementsByTagName('a');
for (j=0; j<links.length; j++) {
wikitext=getNRISOnlyWikitext(links[j].title);
if (wikitext!="error") {
if (wikitext.match(/{{( |[\r\n])*?NRIS[\ -]only(.|[\r\n])*?}}/g)!==null) { // don't tag if already tagged
TaggingState.TagDone++;
TaggingState.TagSkipped++;
links[j].style.backgroundColor = "yellow";
TaggingState.TagProgress.innerHTML="<br />"+TaggingState.TagDone+" of "+TaggingState.TagTotal+" articles examined: "+TaggingState.TagTagged+" tagged, ";
TaggingState.TagProgress.innerHTML+=TaggingState.TagSkipped+" skipped...";
continue;
}
result=NRISOnlyQuery([links[j].title],HiddenSpan);
if (result[0][0]!=links[j].title) { // check again to make sure list isn't outdated
TaggingState.TagDone++;
TaggingState.TagSkipped++;
links[j].style.backgroundColor = "yellow";
TaggingState.TagProgress.innerHTML="<br />"+TaggingState.TagDone+" of "+TaggingState.TagTotal+" articles examined: "+TaggingState.TagTagged+" tagged, ";
TaggingState.TagProgress.innerHTML+=TaggingState.TagSkipped+" skipped...";
continue;
}
multipleIssues=wikitext.match(/{{( )*?(multiple|many|mi|article|issues)( )?(issues)?( |[\r\n])*?\|/gi);
var NRISstr="{{NRIS-only|date="+DateStr+"}}";
if (multipleIssues!==null) { // if multiple issues tag, add NRIS-only to bottom of list
open=1;
start=wikitext.indexOf(multipleIssues[0]);
index=start+multipleIssues[0].length;
while (open!==0 && index<wikitext.length) {
if (wikitext.substr(index,2)=="}}") {
open--;
index++;
} else if (wikitext.substr(index,2)=="{{") {
open++;
index++;
}
index++;
}
index-=2;
full=wikitext.substr(start,index-start);
if (wikitext.substr(index-1,1)!="\n") full+="\n";
oldsyntax=full.substr(2,full.length-2).match(/{{/g);
if (oldsyntax===null) { // if using old syntax
full = full.replace(/\|( )*?one( )?source( )*?=(.|[\r\n])*?(\||$)/g, "|"); // replace "|one source"
if (full.substr(full.length-1,1)=="|") {
full+="\n";
} else {
NRISstr="|"+NRISstr;
}
} else { // replace "{{one source}}"
full = full.replace(/{{( )*?(one|single|1)( |-)?(source|ref)( |[\r\n])*?\|(.|[\r\n])*?}}[\r\n]?/gi,"");
}
full+=NRISstr+"\n";
wikitext = wikitext.substr(0,start)+full+wikitext.substr(index,wikitext.length-index);
} else {
wikitext=wikitext.replace(/{{( )*?(one|single|1)( |-)?(source|ref)( |[\r\n])*?\|(.|[\r\n])*?}}[\r\n]?/gi,"");
wikitext = NRISstr+"\n"+wikitext;
}
NRISOnlyDump({
title: links[j].title,
text: wikitext,
summary:'Tag as only sourced to the [[National Register Information System]]'+CurrentOperatorEditSummary
},TaggingState.TagProgress,TaggingState.TagDone,"tag");
TaggingState.TagDone++;
TaggingState.TagTagged++;
TaggingState.TagProgress.innerHTML="<br />"+TaggingState.TagDone+" of "+TaggingState.TagTotal+" articles examined: "+TaggingState.TagTagged+" tagged, ";
TaggingState.TagProgress.innerHTML+=TaggingState.TagSkipped+" skipped...";
} else {
TaggingState.TagDone++;
TaggingState.TagSkipped++;
links[j].style.backgroundColor = "orange";
TaggingState.TagProgress.innerHTML="<br />"+TaggingState.TagDone+" of "+TaggingState.TagTotal+" articles examined: "+TaggingState.TagTagged+" tagged, ";
TaggingState.TagProgress.innerHTML+=TaggingState.TagSkipped+" skipped...";
}
}
}
TaggingState.TagProgress.innerHTML="<br />Tagging complete! "+TaggingState.TagTagged+" articles tagged successfully! "+TaggingState.TagSkipped;
TaggingState.TagProgress.innerHTML+=" skipped as unnecessary.";
}
if (tobeuntagged!==null) {
var str="";
if (tobetagged!==null) {
str="<br />Now u";
} else {
str="U";
}
ProgressDiv.innerHTML+=str+"ntagging articles that need to be untagged...";
TaggingState.UntagProgress = document.createElement("span");
TaggingState.UntagProgress.setAttribute("id", "ProgressSpan2");
ProgressDiv.appendChild(TaggingState.UntagProgress);
TaggingState.UntagProgress.innerHTML="<br />";
TotalDone=0;
TaggingState.UntagSections=tobeuntagged.getElementsByClassName('refbegin');
total=0;
for (i=0; i<TaggingState.UntagSections.length; i++) {
links=TaggingState.UntagSections[i].getElementsByTagName('a');
TaggingState.UntagTotal+=links.length;
}
TaggingState.UntagUntagged=0;
TaggingState.UntagSkipped=0;
TaggingState.UntagProgress.innerHTML="<br />0 of "+TaggingState.UntagTotal+" articles examined: 0 untagged, 0 skipped...";
for (i=0; i<TaggingState.UntagSections.length; i++) {
links=TaggingState.UntagSections[i].getElementsByTagName('a');
for (j=0; j<links.length; j++) {
wikitext=getNRISOnlyWikitext(links[j].title);
if (wikitext!="error") {
if (wikitext.match(/{{( |[\r\n])*?NRIS[\ -]only(.|[\r\n])*?}}[\r\n]?/g)===null) { //don't untag if already untagged
TaggingState.UntagDone++;
TaggingState.UntagSkipped++;
links[j].style.backgroundColor = "yellow";
TaggingState.UntagProgress.innerHTML="<br />"+TaggingState.UntagDone+" of "+TaggingState.UntagTotal+" articles examined: "+TaggingState.UntagUntagged+" untagged, ";
TaggingState.UntagProgress.innerHTML+=TaggingState.UntagSkipped+" skipped...";
continue;
}
result=NRISOnlyQuery([links[j].title],HiddenSpan);
if (result[1][0]!=links[j].title) { // check again to make sure list isn't outdated
TaggingState.UntagDone++;
TaggingState.UntagSkipped++;
links[j].style.backgroundColor = "yellow";
TaggingState.UntagProgress.innerHTML="<br />"+TaggingState.UntagDone+" of "+TaggingState.UntagTotal+" articles examined: "+TaggingState.UntagUntagged+" untagged, ";
TaggingState.UntagProgress.innerHTML+=TaggingState.UntagSkipped+" skipped...";
continue;
}
multipleIssues=wikitext.match(/{{( )*?(multiple|many|mi|article|issues)( )?(issues)?( |[\r\n])*?\|/gi);
if (multipleIssues!==null) { // if multiple issues tag, check to see we're not leaving only one tag behind
open=1;
start=wikitext.indexOf(multipleIssues[0]);
index=start+multipleIssues[0].length;
while (open!==0 && index<wikitext.length) {
if (wikitext.substr(index,2)=="}}") {
open--;
index++;
} else if (wikitext.substr(index,2)=="{{") {
open++;
index++;
}
index++;
}
index-=2;
full=wikitext.substr(start,index-start);
oldsyntax=full.substr(2,full.length-2).match(/{{( )*?[^(NRIS)](.|[\r\n])*?}}/g); //match non-NRIS temp
var numberTemplates;
if (oldsyntax===null) {
numberTemplates=full.substr(2,full.length-2).match(/\|/g).length - 2;
} else {
numberTemplates=oldsyntax.length;
}
if (numberTemplates===0) { // if NRIS-only is only template in multiple issues, remove entire block
wikitext=wikitext.replace(full+"}}\n","");
} else if (numberTemplates===1 && oldsyntax!==null) { // if one template left, remove multiple issues
wikitext = wikitext.replace(full+"}}", oldsyntax[0]); // and replace with other template
} else { // just remove if more than one other or if using old syntax
wikitext = wikitext.replace(/\|?{{( |[\r\n])*?NRIS[\ -]only(.|[\r\n])*?}}[\r\n]?/g, '');
}
} else {
wikitext = wikitext.replace(/{{( |[\r\n])*?NRIS[\ -]only(.|[\r\n])*?}}[\r\n]?/g, '');
}
NRISOnlyDump({
title: links[j].title,
text: wikitext,
summary:'Remove [[Template:NRIS-only]]; article has more than one reference'+CurrentOperatorEditSummary
},TaggingState.UntagProgress,TotalDone,"tag");
TaggingState.UntagDone++;
TaggingState.UntagUntagged++;
TaggingState.UntagProgress.innerHTML="<br />"+TaggingState.UntagDone+" of "+TaggingState.UntagTotal+" articles examined: "+TaggingState.UntagUntagged+" untagged, ";
TaggingState.UntagProgress.innerHTML+=TaggingState.UntagSkipped+" skipped...";
} else {
TaggingState.UntagDone++; // skip if wikitext error
TaggingState.UntagSkipped++;
links[j].style.backgroundColor = "orange";
TaggingState.UntagProgress.innerHTML="<br />"+TaggingState.UntagDone+" of "+TaggingState.UntagTotal+" articles examined: "+TaggingState.UntagUntagged+" untagged, ";
TaggingState.UntagProgress.innerHTML+=TaggingState.UntagSkipped+" skipped...";
}
}
}
TaggingState.UntagProgress.innerHTML="<br />Untagging complete! "+TaggingState.UntagUntagged+" articles untagged successfully! "+TaggingState.UntagSkipped;
TaggingState.UntagProgress.innerHTML+=" skipped as unnecessary.";
}
}
function getDuplicateStats2(ctx) {
var IllustratedStr = "illustrated";
var countytext="";
var Illustrated=0;
var j, page;
var item = ctx.entry;
ctx.errors = [];
ctx.StatsStr = "articled, unillustrated, Stub-class, NRIS-only";
var countyPromises = [];
for (j=0; j<item.counties.length; j++) {
var county = item.counties[j];
// asyncNRISOnlyWikitext(title,ctx,successfn,errorfn,completefn)
}
//console.log('getDuplicateStats2', ctx);
return Promise.resolve(ctx);
}
function DupStatsToString(statobj) {
// format: <articled>,<illustrated>[,<assessment>[,<nrisonly>]]
var result = `${statobj.articled}, ${statobj.illustrated}`;
if (statobj.assessment != null && statobj.assessment != "") result += `, ${statobj.assessment}`;
if (statobj.nrisonly != null && statobj.nrisonly != "") result += `, ${statobj.nrisonly}`;
return result;
}
function getDuplicateStatsObject(item) {
var statobj;
statobj = getDuplicateStatsObject3(item[0], item[1][0],item.slice(2));
return statobj;
}
function getDuplicateStatsObject3(refnum, dupname, duplocations) {
var IllustratedStr = "illustrated";
var countytext="";
var Illustrated=0;
var j, page;
for (j=0; j<duplocations.length; j++) {
countytext=getNRISOnlyWikitext(duplocations[j]);
if (countytext!="error") {
var StartIndex = 0;
var str = "{{NRHP row";
var skip = str.length;
var index, RowLocations = [];
while ((index = countytext.indexOf(str, StartIndex)) > -1) {
RowLocations.push(index);
StartIndex = index + skip;
}
RowLocations.push(countytext.length); // if duplicated entry happens to be last in table
var k=0;
while (RowLocations[k]<countytext.indexOf(refnum)) {
k++;
}
var CountyRow=countytext.substr(RowLocations[k-1],RowLocations[k]-RowLocations[k-1]);
CountyRow=CountyRow.replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g, ""); // get rid of commented out pictures
if (CountyRow.match(/\|[ ]*?image[ ]*?=[ ]*?[a-zA-Z0-9]/g)!==null) Illustrated++;
} else {
alert("getDuplicateStats: Error fetching "+duplocations[j]);
return;
}
}
if (Illustrated!==duplocations.length&&Illustrated!==0) IllustratedStr="partially-"+IllustratedStr;
if (Illustrated===0) IllustratedStr="un"+IllustratedStr;
var ArticledStr="articled";
var AssessmentStr="";
var NRISStr="";
var title="";
if(Object.prototype.toString.call(dupname) === '[object Array]') { // if in multistate, pick out first item in array
title=dupname[0];
} else {
title = dupname;
}
var nrisonlyquery=JSON.parse( // check if NRIS-only
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'categories',
clcategories: 'Category:All articles sourced only to NRIS',
cllimit: 'max',
titles: title,
redirects: 'true'
},
async:false
})
.responseText
);
if (nrisonlyquery.query.redirects) { // resolve any redirects
for (var r in nrisonlyquery.query.redirects) {
title=nrisonlyquery.query.redirects[r].to;
}
}
for (page in nrisonlyquery.query.pages) {
if (typeof nrisonlyquery.query.pages[page].missing!="undefined") {
ArticledStr="un"+ArticledStr;
continue;
}
if (nrisonlyquery.query.pages[page].categories) {
NRISStr="NRIS-only";
}
}
var catlist='Category:FA-Class National Register of Historic Places articles';
catlist+='|Category:A-Class National Register of Historic Places articles';
catlist+='|Category:GA-Class National Register of Historic Places articles';
catlist+='|Category:B-Class National Register of Historic Places articles';
catlist+='|Category:C-Class National Register of Historic Places articles';
catlist+='|Category:Start-Class National Register of Historic Places articles';
catlist+='|Category:Stub-Class National Register of Historic Places articles';
catlist+='|Category:Unassessed National Register of Historic Places articles';
catlist+='|Category:FL-Class National Register of Historic Places articles';
catlist+='|Category:List-Class National Register of Historic Places articles';
catlist+='|Category:Redirect-Class National Register of Historic Places articles';
if (ArticledStr!="unarticled") {
var statsquery=JSON.parse( // look at quality stats
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'categories',
clcategories: catlist,
cllimit: 'max',
titles: 'Talk:'+title,
redirects: 'true'
},
async:false
})
.responseText
);
for (page in statsquery.query.pages) {
var tagged = "no";
if (statsquery.query.pages[page].categories) {
tagged = "yes";
for (var category in statsquery.query.pages[page].categories) {
var CatTitle=statsquery.query.pages[page].categories[category].title;
if (CatTitle.indexOf("Stub")!=-1) {
AssessmentStr="Stub-class";
continue;
}
if (CatTitle.indexOf("Unassessed")!=-1 || CatTitle.indexOf("Redirect")!=-1) { // also count rdr unassessed
AssessmentStr="unassessed";
continue;
}
if (CatTitle.indexOf("List") != -1 || CatTitle.indexOf("FL-Class") != -1) {
if (statsquery.query.pages[page].title.indexOf("National Register of Historic Places")!=-1){
// references into NRHP list considered unarticled
ArticledStr="un"+ArticledStr;
continue;
} else {
// references into other lists considered stub class
AssessmentStr="Stub-class";
continue;
}
}
AssessmentStr="Start+";
}
}
if (tagged=="no") {
AssessmentStr="untagged";
}
}
}
var statobj = {};
statobj.articled = ArticledStr;
statobj.illustrated = IllustratedStr;
if (NRISStr!=="") {
statobj.nrisonly = NRISStr;
} else {
statobj.nrisonly = null;
}
if (AssessmentStr!=="") {
statobj.assessment = AssessmentStr;
} else {
statobj.assessment = null;
}
return statobj;
}
function getDuplicateStats(item)
{
var statobj = getDuplicateStatsObject(item);
var StatsStr = statobj.articled+', '+statobj.illustrated;
if (statobj.assessment != null) StatsStr+=', '+statobj.assessment;
if (statobj.nrisonly != null) StatsStr+=", "+statobj.nrisonly;
return StatsStr;
}
function GenerateDuplicateTable(StateStructure,i,list,stateName,CountySpan,ThisStateSpan) {
var wikitext="";
var switchtabletext="";
// check county-level duplicates
CountySpan.innerHTML="<br />First checking county-level duplications in "+stateName+"...";
var NumberFound=0;
var j, k, l, m;
var ListName, StatsStr;
console.log("GenerateDuplicateTable", stateName);
if (stateName=="Delaware") {
debugger;
}
for (j=1; j<StateStructure[i].length; j++) {
wikitext+="==="+StateStructure[i][j][0]+", "+StateStructure[i][0]+"===\n";
var TotalCountyDuplicates=0;
var TotalCountyArticled=0;
var TotalCountyIllustrated=0;
var TotalCountyStub=0;
var TotalCountyNRISOnly=0;
var TotalCountyStartPlus=0;
var TotalCountyUnassessed=0;
var TotalCountyUntagged=0;
for (k=0; k<list.length; k++) {
var numberlists=list[k].length-2;
var NumberInThisCounty=0;
var temp=[];
for (l=0; l<list[k].length; l++) { // make temp static
temp[l]=list[k][l];
}
for (l=2; l<temp.length; l++) {
ListName=temp[l].replace(/National Register of Historic Places listings (i|o)n /g,"");
ListName=ListName.replace(", "+stateName,"");
var oldnumber=NumberInThisCounty;
for (m=0; m<StateStructure[i][j][1].length; m++) {
if (ListName==StateStructure[i][j][1][m]) NumberInThisCounty++;
}
if (NumberInThisCounty==oldnumber) {temp.splice(l,1); l--;} // get rid of sublists not in this county
}
if (NumberInThisCounty>1) {
NumberFound++;
CountySpan.innerHTML="<br />First checking county-level duplications in "+stateName+"... ";
CountySpan.innerHTML+=NumberFound+" found so far...";
TotalCountyDuplicates++;
if (TotalCountyDuplicates==1) {
wikitext+="{| class=\"wikitable sortable\" width=100%\n! {{NRHP color}} width=30% | Site\n! {{NRHP ";
wikitext+="color}} width=30% | Lists\n! {{NRHP color}} width=10% | Number of Duplicates\n! {{NRHP ";
wikitext+="color}} width=30% colspan=7 | Stats\n";
}
wikitext+="|-\n| [["+temp[1]+"]] (#"+temp[0]+")\n|\n";
for (m=2; m<temp.length; m++) {
wikitext+="*[["+temp[m]+"|"+temp[m].replace(/National Register of Historic Places listings (i|o)n /g,"");
wikitext+="]]\n";
}
var duplications=NumberInThisCounty-1;
TotalCountyDuplicates+=duplications-1;
wikitext+="| align=center | "+duplications+"\n| colspan=7 | ";
StatsStr=getDuplicateStats(temp);
if (StatsStr.indexOf("unarticled")==-1) TotalCountyArticled+=duplications;
if (StatsStr.indexOf(", illustrated")!=-1) TotalCountyIllustrated+=duplications;
if (StatsStr.indexOf("Stub-class")!=-1) TotalCountyStub+=duplications;
if (StatsStr.indexOf("NRIS-only")!=-1) TotalCountyNRISOnly+=duplications;
if (StatsStr.indexOf("Start+")!=-1) TotalCountyStartPlus+=duplications;
if (StatsStr.indexOf("unassessed")!=-1) TotalCountyUnassessed+=duplications;
if (StatsStr.indexOf("untagged")!=-1) TotalCountyUntagged+=duplications;
wikitext+=StatsStr+"\n";
if (NumberInThisCounty==numberlists) { // if only duplicated inside this county, get rid of it
list.splice(k,1);
k--;
}
}
}
if (TotalCountyDuplicates===0) {
wikitext+="There are no duplications across sublists of "+StateStructure[i][j][0];
wikitext+=", "+StateStructure[i][0]+".\n\n";
switchtabletext+=" |"+StateStructure[i][j][0]+", "+StateStructure[i][0]+"=0</td><td>0</td><td>-</td><td>";
switchtabletext+="0</td><td>-</td><td>0</td><td>0</td><td>0</td><td>-</td><td>0</td><td>0</td><td>-</td>\n";
CountySpan.innerHTML="<br />First checking county-level duplications in "+stateName+"... None found.";
} else {
wikitext+="|-\n! colspan=2 | Total\n! "+TotalCountyDuplicates+"\n! "+TotalCountyIllustrated+"\n! ";
wikitext+=TotalCountyArticled+"\n! "+TotalCountyStub+"\n! "+TotalCountyNRISOnly+"\n! "+TotalCountyStartPlus;
wikitext+="\n! "+TotalCountyUnassessed+"\n! "+TotalCountyUntagged+"\n|}\n\n";
switchtabletext+=" |"+StateStructure[i][j][0]+", "+StateStructure[i][0]+"="+TotalCountyDuplicates+"</td><td>";
switchtabletext+=TotalCountyIllustrated+"</td><td>-</td><td>"+TotalCountyArticled+"</td><td>-</td><td>";
switchtabletext+=TotalCountyStub+"</td><td>"+TotalCountyNRISOnly+"</td><td>"+TotalCountyStartPlus;
switchtabletext+="</td><td>-</td><td>"+TotalCountyUnassessed+"</td><td>"+TotalCountyUntagged+"</td><td>-</td>\n";
CountySpan.innerHTML="<br />First checking county-level duplications in "+stateName+"... Complete! "+TotalCountyDuplicates+" found.";
}
}
if (StateStructure[i].length==1) {
CountySpan.innerHTML="<br />First checking county-level duplications in "+stateName+"... None found.";
}
ThisStateSpan.innerHTML="<br />Now checking statewide duplicates...";
if (StateStructure[i].length>1) wikitext+="==="+StateStructure[i][0]+" Statewide===\n";
if (list.length===0) {
wikitext+="There are no duplications across sublists of "+StateStructure[i][0]+".\n\n";
switchtabletext+="|"+StateStructure[i][0]+"=0</td><td>0</td><td>-</td><td>0</td><td>-</td><td>0</td><td>0</td><td>0";
switchtabletext+="</td><td>-</td><td>0</td><td>0</td><td>-</td>\n";
} else {
var TotalStateDuplicates=0;
var TotalStateArticled=0;
var TotalStateIllustrated=0;
var TotalStateStub=0;
var TotalStateNRISOnly=0;
var TotalStateStartPlus=0;
var TotalStateUnassessed=0;
var TotalStateUntagged=0;
wikitext+="{| class=\"wikitable sortable\" width=100%\n! {{NRHP color}} width=30% | Site\n! {{NRHP ";
wikitext+="color}} width=30% | Lists\n! {{NRHP color}} width=10% | Number of Duplicates\n! {{NRHP ";
wikitext+="color}} width=30% colspan=7 | Stats\n";
for (j=0; j<list.length; j++) {
ThisStateSpan.innerHTML="<br />Now checking statewide duplicates... ";
ThisStateSpan.innerHTML+=j+" of "+list.length+" examined so far...";
wikitext+="|-\n| [["+list[j][1]+"]] (#"+list[j][0]+")\n|\n";
for (m=2; m<list[j].length; m++) {
wikitext+="*[["+list[j][m]+"|";
wikitext+=list[j][m].replace(/National Register of Historic Places listings (i|o)n /g,"")+"]]\n";
}
var thisDuplicateStates=[];
for (l=2; l<list[j].length; l++) {
ListName=list[j][l].replace(/National Register of Historic Places listings (i|o)n /g,"");
ListName=ListName.replace(", "+stateName,"");
thisDuplicateStates.push(ListName);
for (k=1; k<StateStructure[i].length; k++) {
for (m=0; m<StateStructure[i][k][1].length; m++) {
if (ListName==StateStructure[i][k][1][m]) {
thisDuplicateStates[thisDuplicateStates.length-1]=StateStructure[i][k][0];
}
}
}
}
for (l=0;l<thisDuplicateStates.length; l++) { // shrink list if duplicated inside single county
for (m=l+1;m<thisDuplicateStates.length; m++) {
if (thisDuplicateStates[l]==thisDuplicateStates[m]) {thisDuplicateStates.splice(m,1);m--;}
}
}
var dupcount=thisDuplicateStates.length-1;
TotalStateDuplicates+=dupcount;
wikitext+="| align=center | "+dupcount+"\n| colspan=7 | ";
StatsStr=getDuplicateStats(list[j]);
if (StatsStr.indexOf("unarticled")==-1) TotalStateArticled+=dupcount;
if (StatsStr.indexOf(", illustrated")!=-1) TotalStateIllustrated+=dupcount;
if (StatsStr.indexOf("Stub-class")!=-1) TotalStateStub+=dupcount;
if (StatsStr.indexOf("NRIS-only")!=-1) TotalStateNRISOnly+=dupcount;
if (StatsStr.indexOf("Start+")!=-1) TotalStateStartPlus+=dupcount;
if (StatsStr.indexOf("unassessed")!=-1) TotalStateUnassessed+=dupcount;
if (StatsStr.indexOf("untagged")!=-1) TotalStateUntagged+=dupcount;
wikitext+=StatsStr+"\n";
}
wikitext+="|-\n! colspan=2 | Total\n! "+TotalStateDuplicates+"\n! "+TotalStateIllustrated+"\n! ";
wikitext+=TotalStateArticled+"\n! "+TotalStateStub+"\n! "+TotalStateNRISOnly+"\n! "+TotalStateStartPlus;
wikitext+="\n! "+TotalStateUnassessed+"\n! "+TotalStateUntagged+"\n|}\n\n";
switchtabletext+="|"+StateStructure[i][0]+"="+TotalStateDuplicates+"</td><td>"+TotalStateIllustrated+"</td><td>-</td><td>";
switchtabletext+=TotalStateArticled+"</td><td>-</td><td>"+TotalStateStub+"</td><td>"+TotalStateNRISOnly+"</td><td>";
switchtabletext+=TotalStateStartPlus+"</td><td>-</td><td>"+TotalStateUnassessed+"</td><td>"+TotalStateUntagged;
switchtabletext+="</td><td>-</td>\n";
}
return [wikitext,switchtabletext];
}
// subroutine to determine state from list name
function getState(title) {
var temp=title.split(", ");
var state=temp[temp.length-1];
if (state=="Philadelphia") state="Pennsylvania";
if (temp[0]==state||state.indexOf(':')!=-1) {
if (title.indexOf("Chicago")!=-1) state="Illinois";
else if (title.indexOf("Baltimore")!=-1) state="Maryland";
else if (title.indexOf("Boston")!=-1) state="Massachusetts";
else if (title.indexOf("Cincinnati")!=-1) state="Ohio";
else if (title.indexOf("Cleveland")!=-1) state="Ohio";
else if (title.indexOf("Denver")!=-1) state="Colorado";
else if (title.indexOf("Detroit")!=-1) state="Michigan";
else if (title.indexOf("Kansas City")!=-1) state="Missouri";
else if (title.indexOf("Los Angeles")!=-1) state="California";
else if (title.indexOf("Manhattan")!=-1) state="New York";
else if (title.indexOf("Miami")!=-1) state="Florida";
else if (title.indexOf("Milwaukee")!=-1) state="Wisconsin";
else if (title.indexOf("Philadelphia")!=-1) state="Pennsylvania";
else if (title.indexOf("Phoenix")!=-1) state="Arizona";
else if (title.indexOf("St. Louis")!=-1) state="Missouri";
else if (title.indexOf("San Francisco")!=-1) state="California";
else if (title.indexOf("Salt Lake City")!=-1) state="Utah";
else if (title.indexOf("Zion")!=-1) state="Utah";
else if (title.indexOf("Guam")!=-1) state="Guam";
else state=""; // TODO perhaps a better default is to strip "National Register of Historic Places listings in " from title and return that
}
return state;
}
function AsyncNRISDump(info,Span,subpages,doneEditing) {
var thisTitle = title;
return $.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
titles: thisTitle,
indexpageids: true,
redirects: 'true'
},
async:true,
success: function (data, textStatus,jqXHR) {
if (data && data.edit && data.edit.result && data.edit.result=="Success") {
if (doneEditing=="no") {
Span.innerHTML="<br>Dumping data to subpages of ";
Span.innerHTML+="<a href='http://en.wikipedia.org/wiki/"+OutputBase+"'>"+OutputBase+"</a>... "+subpages+" edits made.";
} else if (doneEditing=="yes") {
Span.innerHTML="<br>Dumping data to subpages of ";
Span.innerHTML+="<a href='http://en.wikipedia.org/wiki/"+OutputBase+"'>"+OutputBase+"</a>... Done! "+subpages;
Span.innerHTML+=" subpages edited successfully!";
} else if (doneEditing=="NRIS") {
Span.innerHTML+="Done! Click link to see output!";
// output technical information to console
var WarningText="NRHP Progress Warnings: ";
for (var i=0; i<WarningCount.length; i++) {
WarningText+=WarningCount[i][0]+" ("+WarningCount[i][1]+"), ";
}
if (WarningCount[0][0]!=="") {
WarningText=WarningText.substr(0,WarningText.length-2);
} else {
WarningText="NRHP Progress Warnings: none";
}
throw(WarningText);
} else if (doneEditing=="duplicates") {
Span.innerHTML+="<br>Output saved! Click link to see it.";
}
} else {
Span.innerHTML += " Edit of "+thisTitle+" returned an error! Aborting script.";
alert('Edit query error:\n'+data.error.code+': '+data.error.info+'\nClick OK to view raw wikitext for latest subpage.');
var popup = open("");
var div = popup.document.createElement("div");
div.innerHTML=thisTitle+"<br><hr><br>"+info.text;
popup.document.body.appendChild(div);
}
},
error: function (jqXHR,textStatus, errorThrown) {
Span.innerHTML += " The edit query returned an error! Aborting script.";
alert('Ajax failure. Click OK to view raw wikitext for latest subpage.');
var popup = open("");
var div = popup.document.createElement("div");
div.innerHTML=info.title+"<br><hr><br>"+info.text;
popup.document.body.appendChild(div);
},
complete: function (jqXHR,textStatus) {
if (completefn !== null) {
completefn(title,ctx,jqXHR,textStatus);
}
}
});
}
// dump lists to subpages
function NRISOnlyDump(info,Span,subpages,doneEditing) {
var api = new mw.Api();
api.postWithToken( "csrf", {
action: "edit",
title: info.title,
summary: info.summary,
text: info.text,
bot: 'true'
},
{async:false})
.done( function(data) {
if (data && data.edit && data.edit.result && data.edit.result=="Success") {
if (doneEditing=="no") {
Span.innerHTML="<br>Dumping data to subpages of ";
Span.innerHTML+="<a href='http://en.wikipedia.org/wiki/"+OutputBase+"'>"+OutputBase+"</a>... "+subpages+" edits made.";
} else if (doneEditing=="yes") {
Span.innerHTML="<br>Dumping data to subpages of ";
Span.innerHTML+="<a href='http://en.wikipedia.org/wiki/"+OutputBase+"'>"+OutputBase+"</a>... Done! "+subpages;
Span.innerHTML+=" subpages edited successfully!";
} else if (doneEditing=="NRIS") {
Span.innerHTML+="Done! Click link to see output!";
// output technical information to console
var WarningText="NRHP Progress Warnings: ";
for (var i=0; i<WarningCount.length; i++) {
WarningText+=WarningCount[i][0]+" ("+WarningCount[i][1]+"), ";
}
if (WarningCount[0][0]!=="") {
WarningText=WarningText.substr(0,WarningText.length-2);
} else {
WarningText="NRHP Progress Warnings: none";
}
throw(WarningText);
} else if (doneEditing=="duplicates") {
Span.innerHTML+="<br>Output saved! Click link to see it.";
}
} else {
Span.innerHTML += " Edit of "+info.title+" returned an error! Aborting script.";
alert('Edit query error:\n'+data.error.code+': '+data.error.info+'\nClick OK to view raw wikitext for latest subpage.');
var popup = open("");
var div = popup.document.createElement("div");
div.innerHTML=info.title+"<br><hr><br>"+info.text;
popup.document.body.appendChild(div);
}
})
.fail( function() {
Span.innerHTML += " Edit of "+info.title+" returned an error! Aborting script.";
alert('Ajax failure. Click OK to view raw wikitext for latest subpage.');
var popup = open("");
var div = popup.document.createElement("div");
div.innerHTML=info.title+"<br><hr><br>"+info.text;
popup.document.body.appendChild(div);
});
}
function getNRISOnlyProgressPageWikitext(title) { // asynchronous fetch of Progress page wikitext
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
titles: title,
indexpageids: true,
redirects: 'true'
},
error: function(ajaxResponse,status,errorThrown) {
wikitext="error";
console.log("getNRISOnlyProgressPageWikitext failed title="+title,status, errorThrown);
},
success: function(output) {
for (var page in output.query.pages) {
wikitext=output.query.pages[page].revisions[0]['*'];
}
},
complete: function() {
if (wikitext=="error") {
var ProgressDiv=document.getElementById("ProgressDiv");
ProgressDiv.innerHTML+=" Unable to fetch wikitext! Script aborted.";
} else {
SetupNRISOnlyTables();
}
}
});
}
function getNRISOnlyListWikitext(currentTable,currentRow) { // asynchronous fetch of each list's wikitext
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
titles: NRISOnlyStructure[currentTable][currentRow].Link,
indexpageids: true,
redirects: 'true'
},
error: function(ajaxResponse,status,errorThrown) {
ajaxResponse.errorThrown=errorThrown;
console.log("getNRISOnlyListWikitext failed title="+Titles[StartIndex],status, errorThrown);
},
complete: function(ajaxResponse,status) {NRISOnlyWikitextFetched(ajaxResponse,status,currentTable,currentRow);}
});
}
// async version of getNRISOnlyWikitext
//
// Returns: a Promise object
// Parameters:
// title: String title of page to fetch
// ctx: Context object passed unaltered to callbacks
// successfn: Function to call on success (required)
// Parameters: title, ctx, data, textStatus, jqXHR
// errorfn: Function to call on error (optional)
// Parameters: title,ctx,jqXHR,textStatus,errorThrown
// completefn: Function to call on request completion (after successfn or errorfn is called) (optional)
// Parameters: title,ctx,jqXHR,textStatus
//
// The first two parameters to the callback functions are always the title and ctx passed to this function. The
// remaining parameters are those that were passed to the callback in the Ajax call with one exception.
// The data passed to successfn, instead of being the data blob passed to the Ajax success callback, is first processed
// by JSON.parse.
function asyncNRISOnlyWikitext(title,ctx,successfn,errorfn,completefn) {
return $.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
titles: title,
indexpageids: true,
redirects: 'true'
},
async:true,
success: function (data, textStatus,jqXHR) {
if (successfn !== null) {
successfn(title,ctx,JSON.parse(data),textStatus,jqXHR);
}
},
error: function (jqXHR,textStatus, errorThrown) {
if (errorfn !== null) {
errorfn(title,ctx,jqXHR,textStatus,errorThrown);
}
// else we should probably log something
},
complete: function (jqXHR,textStatus) {
if (completefn !== null) {
completefn(title,ctx,jqXHR,textStatus);
}
}
});
}
function getNRISOnlyWikitext(title) { // legacy synchronous fetch Wikitext of each page for tagging
try {
var output=JSON.parse(
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
titles: title,
indexpageids: true,
redirects: 'true'
},
async:false
})
.responseText
);
for (var page in output.query.pages) {
wikitext = output.query.pages[page].revisions[0]['*'];
}
return wikitext;
}
catch(err) {
return "error";
}
}
// legacy check to see if article is NRIS-only used during tagging
function NRISOnlyQuery(toQuery,SubpageSpan) { // look through wikitext of each article to find NRIS-only ones
var ToBeTagged=[];
var ToBeUntagged=[];
var Unreferenced=[];
var OneRefNotNRIS=[];
var AllNRISOnly=[];
var Substubs=[];
var Errors=[];
var isTagged="no";
var k, l, m;
for (k=1;k<toQuery.length+1; k++) {
SubpageSpan.innerHTML = "<br />Finding NRIS-only articles... Querying page "+k+" of "+toQuery.length+" in this county...";
var wikitext=getNRISOnlyWikitext(toQuery[k-1]);
isTagged="no";
if (wikitext!="error") {
// calculate prose size
var prose=wikitext.replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,""); // strip comments
prose=prose.replace(/<ref[^e](.|[\r\n])*?([ ]*?\/|<\/ref[ ]*?)\>/gi,""); // strip refs
prose=prose.replace(/==[ ]*?External links[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,""); // strip external links section
prose=prose.replace(/==[ ]*?See also[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,""); // strip see also section
prose=prose.replace(/==[ ]*?(References|Notes)[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,""); // strip references section
// strip further reading section
prose=prose.replace(/==[ ]*?(Further|Additional) reading[ ]*?==(.|\n)*?(?=(==.*?==|$))/gi,"");
prose=prose.replace(/={2,5}.*?={2,5}/g,""); // strip section titles
// replace wikilinks with displayed text
prose=prose.replace(/\[\[(?![ ]*?Category:|[ ]*?Image:|[ ]*?File:)([^\]]*?\|)?(.*?)\]\]/gi,"$2");
prose=prose.replace(/\[[ ]*?http.*? (.*?)\]/g,"$1"); // replace inline external links with displayed text
prose=prose.replace(/'{2,5}(.*?)'{2,5}/g,"$1"); // replace bold/italic with displayed text
prose=prose.replace(/\[\[[ ]*?Category:.*?\]\]/g,""); // strip categories
prose=prose.replace(/\[\[[ ]*?(Image|File):.*?\]\]/g,""); // strip images
prose=prose.replace(/<[ ]*?gallery(.|\n)*?<[ ]*?\/[ ]*?gallery[ ]*?\>/gi,""); // strip galleries
while(true) { // strip templates
var str="{{";
var start=prose.indexOf(str);
if (start==-1) break;
var open=1;
var index=start+str.length;
while (open!==0 && index<prose.length) {
if (prose.substr(index,2)=="}}") {
open--;
index++;
} else if (prose.substr(index,2)=="{{") {
open++;
index++;
}
index++;
}
prose=prose.replace(prose.substr(start,index-start),"");
}
prose=prose.replace(/{\|(.|\n)*?\|}/g,""); // strip tables
prose=prose.replace(/ /g," "); // replace nbsp with regular space
prose=prose.replace(/<[ ]*?br.*?>/g, "\n"); // replace HTML line break with string line break
prose=prose.replace(/[ \n]+/g," "); // replace multiple spaces/linebreaks with single space
prose=prose.trim();
if (prose.length<325) Substubs[Substubs.length]=[toQuery[k-1],prose.length];
if (wikitext.match(/{{( |[\r\n])*?NRIS[\ -]only(.|[\r\n])*?}}/g)!==null) isTagged="yes";
if (wikitext.indexOf("{{GR")!=-1||wikitext.indexOf("{{sfn")!=-1||wikitext.indexOf("{{Sfn")!=-1) {
if (isTagged=="yes") ToBeUntagged.push(toQuery[k-1]);
continue;
}
var Refs=wikitext.match(/<ref[^e](.|[\r\n])*?([ ]*?\/|<\/ref[ ]*?)\>/gi);
var comments=wikitext.match(/<\!\-\-(.|[\r\n])*?\-\-\>/g);
if (Refs===null) { // if no refs, count as unreferenced
Unreferenced.push(toQuery[k-1]);
continue;
}
if (comments!==null) {
for (l=0; l<comments.length; l++) {
var CommentedRefs=comments[l].match(/<ref[^e](.|[\r\n])*?([ ]*?\/|<\/ref[ ]*?)\>/gi);
if (CommentedRefs===null) continue;
for (m=0; m<CommentedRefs.length; m++) {
for (var n=0; n<Refs.length; n++) {
if (Refs[n]==CommentedRefs[m]) {Refs.splice(n,1); n--;}
}
}
}
}
if (Refs.length===0) { // if all refs commented out, count as unreferenced
Unreferenced.push(toQuery[k-1]);
continue;
}
var citesNRIS="no";
for (l=0; l<Refs.length; l++) {
if (Refs[l].indexOf("{{NRISref")!=-1) citesNRIS="yes";
}
var namedRefs=[];
var Duplications=[];
for (l=0; l<Refs.length; l++) {
var nameOfRef=Refs[l].match(/name[ ]*?=.*?(\/|\>)/gi);
if (nameOfRef===null) {
continue;
} else {
nameOfRef = nameOfRef[0].replace(/("| )/g,'');
nameOfRef = nameOfRef.substr(5,nameOfRef.length-6);
}
namedRefs.push(nameOfRef);
Duplications.push(1);
for (m=0; m<namedRefs.length-1; m++) { // if title is duplicated, count how many times
if (nameOfRef==namedRefs[m]) {
namedRefs.splice(m,1);
Duplications[Duplications.length-1] = Duplications[m] + 1;
Duplications.splice(m,1);
m--;
}
}
}
if (namedRefs.length==2&&namedRefs[0]==namedRefs[1]) Duplications[0]++; // fix for if all refs are same
var toSubtract=0;
for (l=0; l<Duplications.length; l++) {
toSubtract = toSubtract + Duplications[l] - 1;
}
var DistinctRefs = Refs.length-toSubtract;
if (DistinctRefs>1) {
if (isTagged=="yes") ToBeUntagged.push(toQuery[k-1]); // untag if ref has been added since tag placed
continue;
}
if (citesNRIS=="no") {OneRefNotNRIS.push(toQuery[k-1]);continue;} // if only one ref and not NRIS, count that
if (isTagged=="no") ToBeTagged.push(toQuery[k-1]); // only push if the only ref is NRISref and it's not already tagged
AllNRISOnly.push(toQuery[k-1]);
} else {
Errors.push(toQuery[k-1]);
continue;
}
}
return [ToBeTagged,ToBeUntagged,Unreferenced,OneRefNotNRIS,AllNRISOnly,Substubs,Errors];
}
//$(window).on('load',CheckPermission);
$.when($.ready).then(CheckPermission);