User:AnomieBOT/source/tasks/TagDater.pm
 | Approved 2010-12-13 Wikipedia:Bots/Requests for approval/AnomieBOT 49 |
| Supplemental BFRA approval requested 2011-09-12 Wikipedia:Bots/Requests for approval/AnomieBOT 55 |
| Supplemental BFRA approved 2011-10-13 Wikipedia:Bots/Requests for approval/AnomieBOT 57 |
| Supplemental BFRA approved 2015-03-28 Wikipedia:Bots/Requests for approval/AnomieBOT 72 |
| Supplemental BFRA approved 2024-03-10 Wikipedia:Bots/Requests for approval/AnomieBOT 81 |
package tasks::TagDater;
=pod
=begin metadata
Bot: AnomieBOT
Task: TagDater
BRFA: Wikipedia:Bots/Requests for approval/AnomieBOT 49
Status: Approved 2010-12-13
+BRFA: Wikipedia:Bots/Requests for approval/AnomieBOT 55
+Status: Approved 2012-01-20
+BRFA: Wikipedia:Bots/Requests for approval/AnomieBOT 57
+Status: Approved 2011-10-13
+BRFA: Wikipedia:Bots/Requests for approval/AnomieBOT 72
+Status: Approved 2015-03-28
+BRFA: Wikipedia:Bots/Requests for approval/AnomieBOT 81
+Status: Approved 2024-03-10
Created: 2010-10-20
Date maintenance tags in articles listed in first-level subcategories of
[[:Category:Wikipedia maintenance categories sorted by month]] and [[:Category:Wikipedia categories sorted by month]].
* If a maintenance category is directly used on the page, and a corresponding template is already on the page, remove the direct category use.
* For templates in [[Wikipedia:AutoWikiBrowser/Dated templates]] or [[User:AnomieBOT/Dating rules]], and their redirects:
** If a date exists in {{para|1}} or certain other parameters, move it to {{para|date}}.
*** Exception is made for if {{para|1}} is declared in templatedata as existing and not being an alias for {{para|date}}.
** If the date is an incorrect format (e.g. MDY, DMY, YMD), correct it to the Month Year format needed by the templates.
** If all else fails, add the current Month Year as {{para|date}}
* For {{tl|multiple issues}} and redirects:
** For each parameter besides "article", "section", "expert", "text", and numbered parameters, correct the date format or fill in the current date as above.
* For {{tl|as of}}, {{tl|update after}}, and their redirects:
** If a {{para|date}} exists, remove it or move it to {{para|1}} if necessary.
** If {{para|1}} contains a recognizable date instead of having the date specified with year in 1, month in 2, and day in 3, correct it. Supply the current year if necessary.
** If no date is found or {{para|1}} is "today", "now", or certain other keywords, add the current date.
* For {{tl|disambiguation}} and its redirects:
** If it includes "cleanup" or aliases, change it to {{tl|disambiguation cleanup}} and date it.
** If it includes "one non-primary topic" or aliases, remove that and add {{tl|one other topic}}.
* For {{tl|disambiguation cleanup}} and its redirects:
** If it includes "cleanup" or aliases, remove them as redundant.
** If it includes "one non-primary topic" or aliases, remove that and add {{tl|one other topic}}.
* If the above resulted in no changes, templates used in the article are in turn checked in the same manner.
** Also, if the template has the output of {{tls|rfd}}, it will be bypassed if the bypassed redirect would be dated.
=end metadata
=cut
use utf8;
use strict;
use POSIX;
use Data::Dumper;
use AnomieBOT::API;
use AnomieBOT::Task qw/:time/;
use vars qw/@ISA/;
@ISA=qw/AnomieBOT::Task/;
# Increment this if the bot should re-scan all skipped pages
my $version=8;
# Delays to allow for human editing
my $min_delay=1200;
my $inuse_delay=7200;
my $arbitrary_untrusted_threshold=1000;
my $arbitrary_trusted_threshold=2000;
my $untrusted_delay=7200;
# List of months
my %months=(
# Common misspellings and such. Keys must be all lowercase.
# Real month names and legitimate variations
'january' => 'January',
'jan' => 'January',
'february' => 'February',
'feb' => 'February',
'march' => 'March',
'mar' => 'March',
'april' => 'April',
'apr' => 'April',
'may' => 'May',
'june' => 'June',
'jun' => 'June',
'july' => 'July',
'jul' => 'July',
'august' => 'August',
'aug' => 'August',
'september' => 'September',
'sep' => 'September',
'sept' => 'September',
'october' => 'October',
'oct' => 'October',
'november' => 'November',
'nov' => 'November',
'december' => 'December',
'dec' => 'December',
);
my %skiptags=(
'possible libel or vandalism' => 1,
);
# Non-config globals
my %inuse=(
'Category:Pages actively undergoing a major edit'=>1,
);
my @months=qw/January February March April May June July August September October November December/;
my %monthnum=();
for(my $i=0; $i<@months; $i++){
$monthnum{$months[$i]}=$i+1;
}
# Auto-spell-checking: add anything with an edit distance of 1 from a real
# month name.
foreach my $m (@months){
foreach my $mm (edits1(lc($m))) {
$months{$mm}=$m unless exists($months{$mm});
}
}
delete $months{qw/juny jule/}; # Could be "June" or "July"
my $monthre=join('|', keys %months);
$monthre=qr/$monthre/i;
sub new {
my $class=shift;
my $self=$class->SUPER::new();
$self->{'templates'}=undef;
$self->{'templates rev'}=0;
$self->{'rules'}=undef;
$self->{'rules rev'}=undef;
$self->{'multiple issues'}=undef;
$self->{'multiple issues map'}=undef;
$self->{'multiple issues rev'}=undef;
$self->{'as of'}=undef;
$self->{'disambig'}=undef;
$self->{'disambigcleanup'}=undef;
$self->{'bypass'}=undef;
$self->{'iter'}=undef;
$self->{'tpl num param cache'}={};
bless $self, $class;
return $self;
}
=pod
=for info
Approved 2010-12-13<br />[[Wikipedia:Bots/Requests for approval/AnomieBOT 49]]
=for info
Supplemental BFRA approval requested 2011-09-12<br />[[Wikipedia:Bots/Requests for approval/AnomieBOT 55]]
=for info
Supplemental BFRA approved 2011-10-13<br />[[Wikipedia:Bots/Requests for approval/AnomieBOT 57]]
=for info
Supplemental BFRA approved 2015-03-28<br />[[Wikipedia:Bots/Requests for approval/AnomieBOT 72]]
=for info
Supplemental BFRA approved 2024-03-10<br />[[Wikipedia:Bots/Requests for approval/AnomieBOT 81]]
=cut
sub approved {
return 7;
}
sub run {
my ($self, $api)=@_;
my $res;
$api->task('TagDater',0,0,qw/d::Talk d::Redirects d::Templates d::Trial/);
$res=$self->refresh_cache($api);
return $res if defined($res);
my $iter=$self->{'iter'};
if(!defined($iter)){
my %cats = ();
for my $cat ( 'Category:Wikipedia categories sorted by month', 'Category:Wikipedia maintenance categories sorted by month' ){
$res=$api->query(
generator => 'categorymembers',
gcmtitle => $cat,
gcmnamespace => 14,
gcmtype => 'subcat',
gcmlimit => 'max',
prop => 'categoryinfo',
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to get subcats of $cat: ".$res->{'error'}."\n");
return 60;
}
for my $c (values %{$res->{'query'}{'pages'}}) {
$cats{$c->{'title'}} = $c;
}
}
$iter=$api->iterator(
generator => 'categorymembers',
gcmtitle => [map $_->{'title'}, sort {
my $ret=0;
if($a->{'title'} ne $b->{'title'}){
$ret=1 if $a->{'title'} eq 'Category:Articles with invalid date parameter in template';
$ret=-1 if $b->{'title'} eq 'Category:Articles with invalid date parameter in template';
}
$ret=($b->{'categoryinfo'}{'pages'}<=>$a->{'categoryinfo'}{'pages'}) unless $ret;
$ret;
} values %cats],
gcmnamespace => 0,
gcmtype => 'page',
gcmlimit => 'max',
prop => 'info|categories',
cllimit => 'max',
clcategories => join('|', keys %inuse)
);
$self->{'iter'}=$iter;
}
# Spend a max of 5 minutes on this task before restarting
my %skip=%{$api->store->{'skip'}};
my %newskip=();
my $waituntil=time()+600;
$self->{'reloaded_template_redirects'}=0;
while($_=$iter->next){
return 0 if $api->halting;
if(!$_->{'_ok_'}){
$api->warn("Failed to retrieve members in ".$iter->iterval.": ".$_->{'error'}."\n");
return 60;
}
# Skip? Continue skipping.
if(exists($skip{$_->{'lastrevid'}})){
$newskip{$_->{'lastrevid'}}=1;
next;
}
# Deleted since we read the category?
next if exists($_->{'missing'});
my $title=$_->{'title'};
# Don't try fixing any page touched too recently, to give the real
# editor a chance to fix it.
my $until = $self->check_delay( $api, $_ );
if ( $until > time() ) {
$waituntil=$until if $until<$waituntil;
next;
}
#$api->log("Checking for undated templates because of ".$iter->iterval." in $title");
my ($ret,$didanything)=$self->check_page($api, $iter->iterval, $title, $_->{'lastrevid'}, 1, \%skip, \%newskip, \$waituntil, 0);
return $ret if defined($ret);
next unless $didanything;
} continue {
$self->{'reloaded_template_redirects'}=0;
}
# No more pages to check for now
$api->store->{'skip'}=\%newskip;
$self->{'iter'}=undef;
return $waituntil-time();
}
sub refresh_cache {
my $self=shift;
my $api=shift;
my $res;
# Flush store if the code has been updated
if(($api->store->{'version'}//0) < $version){
$api->store->{'skip'}={};
$api->store->{'templates ts'}=0;
$api->store->{'multiple issues rev'}=0;
$api->store->{'rules ts'}=0;
$api->store->{'version'}=$version;
}
if(($api->store->{'templates ts'}//0) > time()-86400){
$self->{'templates'}=$api->store->{'templates'};
$self->{'templates rev'}=$api->store->{'templates rev'};
} else {
$self->{'templates'}=undef;
$self->{'templates rev'}=0;
}
if($self->{'templates rev'}){
$res=$api->query(
titles => 'Wikipedia:AutoWikiBrowser/Dated templates',
prop => 'info',
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load info for Wikipedia:AutoWikiBrowser/Dated templates: ".$res->{'error'}."\n");
return 60;
}
if((values %{$res->{'query'}{'pages'}})[0]{'lastrevid'} != $self->{'templates rev'}){
$self->{'templates'}=undef;
$api->flush_redirect_cache();
}
}
my %templates;
if(defined($self->{'templates'})){
%templates=%{$self->{'templates'}};
} else {
# Get the list of tempates
$api->log("Reloading list of templates from WP:AWB/DT");
my $ts=time();
$res=$api->query(
titles => 'Wikipedia:AutoWikiBrowser/Dated templates',
prop => 'revisions',
rvprop => 'ids|content',
rvslots => 'main',
rvlimit => 1,
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load Wikipedia:AutoWikiBrowser/Dated templates: ".$res->{'error'}."\n");
return 60;
}
$res=(values %{$res->{'query'}{'pages'}})[0]{'revisions'}[0];
my $txt=$res->{'slots'}{'main'}{'*'};
$txt=$api->strip_nowiki($txt);
$txt=~s/_/ /g;
$txt=~s/\{\{\s*Template\s*:/\{\{/gi;
my @templates=($txt=~/\{\{\s*[tT]lx?\s*\|\s*([^|]+?)\s*(?:\||\}\})/g);
%templates=$api->redirects_to_resolved(map "Template:$_", @templates);
if(exists($templates{''})){
$api->warn("Failed to get redirects to target templates: ".$templates{''}{'error'}."\n");
return 60;
}
$self->{'templates'}=\%templates;
$self->{'templates rev'}=$res->{'revid'};
$api->store->{'templates'}=$self->{'templates'};
$api->store->{'templates rev'}=$self->{'templates rev'};
$api->store->{'templates ts'}=$ts;
$api->log("Done reloading list of templates from WP:AWB/DT");
# Updated template list, so don't skip anything
$api->store->{'skip'}={};
}
if(($api->store->{'rules ts'}//0) > time()-86400){
$self->{'rules'}=$api->store->{'rules'};
$self->{'rules rev'}=$api->store->{'rules rev'};
} else {
$self->{'rules'}=undef;
$self->{'rules rev'}=0;
}
if($self->{'rules rev'}){
$res=$api->query(
titles => 'User:AnomieBOT/Dating rules',
prop => 'info',
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load info for User:AnomieBOT/Dating rules: ".$res->{'error'}."\n");
return 60;
}
if((values %{$res->{'query'}{'pages'}})[0]{'lastrevid'} != $self->{'rules rev'}){
$self->{'rules'}=undef;
$api->flush_redirect_cache();
}
}
my %rules;
if(defined($self->{'rules'})){
%rules=%{$self->{'rules'}};
} else {
# Get the list of tempates
$api->log("Reloading rules from User:AnomieBOT/Dating rules");
my $ts=time();
$res=$api->query(
titles => 'User:AnomieBOT/Dating rules',
prop => 'revisions',
rvprop => 'ids|content',
rvslots => 'main',
rvlimit => 1,
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load User:AnomieBOT/Dating rules: ".$res->{'error'}."\n");
return 60;
}
$res=(values %{$res->{'query'}{'pages'}})[0]{'revisions'}[0];
my $txt=$res->{'slots'}{'main'}{'*'};
$txt=$api->strip_nowiki($txt);
$txt=~s/[_\t]/ /g;
$txt=~s/.*\n==\s*Rules\s*==\s*\n//gs;
my @rules = ();
my %templates = ();
my @lines = ($txt=~/^\* *\{\{ *[tT]lx? *\| *([^|}]+?) *(?:\}\})(.*)$/gm);
for(my $i=0; $i<@lines; $i+=2){
my ($template,$flags) = ($lines[$i], $lines[$i+1]);
my $rule = {
template => "Template:$template",
with => {},
without => {},
ignore => {},
keep => {},
dateparameter => 'date',
};
while( $flags=~s/^ +(with|without|ignore|keep) *((?: *\{\{para\|[^}]+\}\})+)// ) {
my ($k, $params) = ($1, $2);
$k=~s/\s+//g;
$api->process_templates($params, sub {
my ($param,$value)=($_[1][0], $_[1][1]//'*');
$param=~s/^\s*|\s*$//g;
$value=~s/^\s*|\s*$//g;
$value=quotemeta($value);
$value=~s/\\\//|/g;
$value=~s/\\\*/.*/g;
$rule->{$k}{$param}=$value;
});
}
if ( $flags=~s/^ +date +parameter +\{\{para\|\s*([^}]+?)\s*\}\}// ) {
$rule->{'dateparameter'} = $1;
}
$flags=~s/ +\(.*\)//;
next unless $flags=~/^\s*$/;
push @rules, $rule;
$templates{"Template:$template"} = 1;
}
%templates = $api->redirects_to_resolved( sort keys %templates );
if(exists($templates{''})){
$api->warn("Failed to get redirects to rules templates: ".$templates{''}{'error'}."\n");
return 60;
}
my %rmap=();
while(my ($k,$v)=each %templates){
push @{$rmap{$v}}, $k;
}
for my $rule (@rules) {
my $t = $templates{$rule->{'template'}};
$rule->{'template'} = $templates{$rule->{'template'}};
for my $t (@{$rmap{$rule->{'template'}}}){
push @{$rules{$t}}, $rule;
}
}
$self->{'rules'}=\%rules;
$self->{'rules rev'}=$res->{'revid'};
$api->store->{'rules'}=$self->{'rules'};
$api->store->{'rules rev'}=$self->{'rules rev'};
$api->store->{'rules ts'}=$ts;
$api->log("Done reloading rules from User:AnomieBOT/Dating rules");
# Updated rule list, so don't skip anything
$api->store->{'skip'}={};
}
# Get the list of tempates
my %mi=$api->redirects_to_resolved('Template:Multiple issues');
if(exists($mi{''})){
$api->warn("Failed to get redirects to {{multiple issues}}: ".$mi{''}{'error'}."\n");
return 60;
}
$self->{'multiple issues'}=\%mi;
if($self->{'multiple issues rev'}){
$res=$api->query(
titles => 'Template:Multiple issues',
prop => 'info',
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load info for Template:Multiple issues: ".$res->{'error'}."\n");
return 60;
}
if((values %{$res->{'query'}{'pages'}})[0]{'lastrevid'} != $self->{'multiple issues rev'}){
$self->{'multiple issues map'}=undef;
}
}
my %mimap;
if(defined($self->{'multiple issues map'})){
%mimap=%{$self->{'multiple issues map'}};
} else {
$api->log("Reloading Template:Multiple issues");
$res=$api->rawpage($mi{"Template:Multiple issues"});
if($res->{'code'} ne 'success'){
$api->warn("Failed to get content of {{multiple issues}}: ".$res->{'error'}."\n");
return 60;
}
%mimap=();
$api->process_templates($res->{'content'}, sub {
my $name=shift;
my $params=shift;
return undef unless $name eq 'Multiple issues/message';
my ($nn, $t)=(undef, undef);
foreach my $p ($api->process_paramlist(@$params)){
$nn=$p->{'value'} if $p->{'name'} eq 'name';
$t=$p->{'value'} if $p->{'name'} eq 'template';
}
return undef unless(defined($nn) && defined($t));
$nn=~s/[{}]//g;
foreach my $n (split /\|/, $nn){
$n=~s/^\s*|\s*$//g;
$mimap{$n}=$t;
}
});
$self->{'multiple issues map'}=\%mimap;
$api->log("Done reloading Template:Multiple issues");
}
# Get the list of tempates
my %asof=$api->redirects_to_resolved('Template:As of', 'Template:Update after');
if(exists($asof{''})){
$api->warn("Failed to get redirects to {{as of}}: ".$asof{''}{'error'}."\n");
return 60;
}
$self->{'as of'}=\%asof;
# Get the list of tempates
my %disambig=$api->redirects_to_resolved('Template:Disambiguation');
if(exists($disambig{''})){
$api->warn("Failed to get redirects to {{disambiguation}}: ".$disambig{''}{'error'}."\n");
return 60;
}
$self->{'disambig'}=\%disambig;
my %disambigcleanup=$api->redirects_to_resolved('Template:Disambiguation cleanup');
if(exists($disambigcleanup{''})){
$api->warn("Failed to get redirects to {{disambiguation cleanup}}: ".$disambigcleanup{''}{'error'}."\n");
return 60;
}
$self->{'disambigcleanup'}=\%disambigcleanup;
# Clear the list of templates to bypass
$self->{'bypass'}={};
return undef;
}
sub fixcurrent {
my $api=shift;
my $dt=shift;
$dt=$api->process_templates($dt, sub {
my $n=lc(shift);
return strftime('%B', gmtime) if($n eq 'subst:currentmonthname' || $n eq 'currentmonthname');
return strftime('%Y', gmtime) if($n eq 'subst:currentyear' || $n eq 'currentyear');
return undef;
});
return $dt;
}
sub fixdate {
my $origdt=shift;
my $dt=$origdt;
my $chk=(shift//\&chkdate);
my $d=qr{[\s_/.,=+-]};
my $c='';
$c.="|$1" if $dt=~s/(reason\s*=.*)$//s;
$c.=$1 if $dt=~s/((?>\s*<!--.*?-->))$//s;
$dt=~s/[\s_]+/ /g;
$dt=~s/^(?:$d*date$d*=?)+//i;
$dt=~s/^$d+|$d+$//;
$dt=~s/^"(.*)"$/$1/;
return strftime('%B %Y', gmtime).$c if $dt=~/^(?:|Undated|now|today|Monthname YYYY)$/i;
return $chk->($months{lc($1)},$2).$c if $dt=~/^(?:\d{1,2}(?:$d*(?i:st|nd|rd|th))?$d*)?($monthre)$d*(\d{4})$/;
return $chk->($months{lc($1)},$2).$c if $dt=~/^($monthre)$d*(?:\d{1,2}(?:$d*(?i:st|nd|rd|th))?$d+)?(\d{4})$/;
return $chk->($months{lc($2)},$1).$c if $dt=~/^(\d{4})$d*($monthre)(?:$d*\d{1,2}(?:$d*(?i:st|nd|rd|th))?)?$/;
return $chk->($months[$2-1],$1).$c if $dt=~/^(\d{4})$d+(0[1-9]|1[0-2])$d+\d{2}$/;
return $chk->($months[$2-1],$1).$c if $dt=~/^(\d{4})-(0[1-9]|1[0-2])$/;
return $chk->($months{lc($1)},$2).$c if $dt=~/^\d{1,2}:\d{2}, \d{1,2} ($monthre) (\d{4}) \(UTC\)$/;
return $origdt if length($origdt)>30;
return undef;
}
sub chkdate {
my ($m,$y)=@_;
return strftime('%B %Y', gmtime) if $y<2000; # Almost certainly an error
my $indt=$y*12+$monthnum{$m}-1;
my $curdt=22800+(gmtime)[5]*12+(gmtime)[4];
return ($indt<=$curdt)?"$m $y":strftime('%B %Y', gmtime);
}
sub nochkdate {
my ($m,$y)=@_;
return "$m $y";
}
sub chkasofdate {
my $v=shift;
my $dt=fixdate($v, \&nochkdate)//'';
return 0 if $dt ne $v;
return 0 unless $dt=~/ (\d+)$/;
return ($1>2004);
}
sub edits1 {
my %r=();
my $x;
foreach my $w (@_){
for(my $i=1; $i<length($w); $i++){
$x=$w; substr($x,$i,1)=''; $r{$x}=1; # deletion
if($i>1){
$x=$w; substr($x,$i-1,2)=reverse(substr($x,$i-1,2)); $r{$x}=1; # transposition
}
foreach my $c (' ', 'a'..'z') {
$x=$w; substr($x,$i,1)=$c; $r{$x}=1; # replacement
$x=$w; substr($x,$i,0)=$c; $r{$x}=1; # insertion
}
}
$r{$w.$_}=1 foreach ('a'..'z'); # insertion at end
}
return keys %r;
}
sub check_page {
my ($self,$api,$cat,$title,$lastrevid,$recurse,$skip,$newskip,$waituntil,$istransclude)=@_;
#$api->log("Checking $title because of $cat");
# Ok, check the page
my $tok=$api->edittoken($title, EditRedir=>1);
if($tok->{'code'} eq 'shutoff'){
$api->warn("Task disabled: ".$tok->{'content'}."\n");
return (300,0);
}
if($tok->{'code'} eq 'pageprotected' || $tok->{'code'} eq 'botexcluded'){
# Skip protected and excluded pages until the next edit
$api->warn("Cannot edit $title: ".$tok->{'error'}."\n") if $tok->{'ns'}==0;
$skip->{$tok->{'lastrevid'}}=1;
$newskip->{$tok->{'lastrevid'}}=1;
$api->store->{'skip'}=$skip;
return (undef,0);
}
if($tok->{'code'} ne 'success'){
$api->warn("Failed to get edit token for $title: ".$tok->{'error'}."\n");
return (undef,0);
}
return (undef,0) if exists($tok->{'missing'});
if($tok->{'lastrevid'} ne $lastrevid){
# Someone edited in between loading the cat and getting the
# token. We'll catch the new revision next time around.
$api->log("$title was edited since cat list was loaded, abort");
return (undef,0);
}
if ( ! $istransclude ) {
my ($ret,$didanything) = $self->check_direct_cat_use($api,$tok,$cat,$title);
return ($ret, $didanything) if defined( $ret ) || $didanything;
}
return $self->check_page2($api,$tok,$cat,$title,$recurse,$skip,$newskip,$waituntil,$istransclude);
}
sub check_page2 {
my ($self,$api,$tok,$cat,$title,$recurse,$skip,$newskip,$waituntil,$istransclude)=@_;
my $res;
my $fail = undef;
my %bypass=%{$self->{'bypass'}};
my %asof=%{$self->{'as of'}};
my %disambig=%{$self->{'disambig'}};
my %disambigcleanup=%{$self->{'disambigcleanup'}};
my %mi=%{$self->{'multiple issues'}};
my %mimap=%{$self->{'multiple issues map'}};
my %rules=%{$self->{'rules'}};
my %templates=%{$self->{'templates'}};
my $df = undef;
my %fixed=();
local our $checksub;
$checksub = sub {
my $name=shift;
my $params=shift;
my $wikitext=shift;
shift; # $data
my $oname=shift;
return undef if defined($fail);
# Some people are just strange...
my $n=lc($name); $n=~s/[\s_]*//g;
if(!$istransclude){
if($n eq 'subst:currentmonthname'){
$name=~s/[\s_]*//g; $name=~s/^subst:/subst:/i;
$fixed{"{{$name}}"}=1;
return strftime('%B', gmtime);
}
if($n eq 'subst:currentyear'){
$name=~s/[\s_]*//g; $name=~s/^subst:/subst:/i;
$fixed{"{{$name}}"}=1;
return strftime('%Y', gmtime);
}
}
if($n=~/^subst:(\d{4})$/){
$fixed{"{{$n}}"}=1;
return $1;
}
for my $m (@months) {
if($n eq lc("subst:$m")){
$name=~s/[\s_]*//g; $name=~s/^subst:/subst:/i;
$fixed{"{{$name}}"}=1;
return $m;
}
}
# RfD to bypass?
if(exists($bypass{"Template:$name"})){
my $txt=$wikitext;
my $target = $bypass{"Template:$name"};
$txt=~s/^(\{\{\s*)\Q$oname\E(\s*)/$1$target$2/;
my %oldfixed = %fixed;
my $txt2=$api->process_templates($txt, $checksub);
return undef if defined($fail);
%fixed = %oldfixed;
if($txt ne $txt2){
$fixed{"{{$oname}} (RfD) → $txt2"} = 1;
return $txt2;
}
}
# {{multiple issues}}?
if(exists($mi{"Template:$name"})){
my @fixed=();
my $txt="{{$oname";
my ($needdate,$havedate)=(0,0);
my $needfix=0;
for(my $i=0; $i<@$params; $i++){
if($params->[$i]=~/^(\s*)(\S[^={}]+?)(\s*=\s*$monthre \d{4})(\s*)(\S[^={}]+?)(\s*=\s*$monthre \d{4})/){
splice @$params, $i, 1, "$1$2$3", "$4$5$6";
push @fixed, "$2=|$5=";
}
if($params->[$i]=~/^(\s*)(\S[^={}]+?)(\s*=\s*$monthre \d{4})\s*\{\{/){
$needfix=1;
}
}
if($needfix){
# The template seems to have a broken attempt at mixing old-
# and new-style parameters; this is often caused by user
# scripts or other bots assuming if any new-style parameters
# are present that it can just append its new tag. So just
# parse everything out and convert it all to new-style.
my @tmpl=();
for(my $i=0; $i<@$params; $i++){
$params->[$i]=$api->process_templates($params->[$i], sub {
push @tmpl, $_[2];
return '';
});
}
my %p=();
foreach ($api->process_paramlist(@$params)){
$p{$_->{'name'}}=$_->{'value'};
}
my $section=(($p{'section'}//'')!~/^\s*$/)?'section':'article';
foreach ($api->process_paramlist(@$params)){
next if $_->{'value'}=~/^\s*$/;
next if $_->{'name'}=~/^\s*\d+\s*$/;
my $v=fixdate($_->{'value'}) // strftime('%B %Y', gmtime);
if($_->{'name'} eq 'cleanup'){
push @tmpl, "{{Cleanup|$section|reason=".($p{'reason'}//'')."|date=$v}}";
} elsif($_->{'name'} eq 'expert'){
push @tmpl, "{{Expert-subject|".$_->{'value'}."|$section|date=$v}}";
} elsif(exists($mimap{$_->{'name'}})){
push @tmpl, "{{".$mimap{$_->{'name'}}."|$section|date=$v}}";
} elsif($_->{'name'} eq 'reason' || $_->{'name'} eq 'date'){
# Converted above
} else {
$txt.='|'.$_->{'text'};
}
}
$txt.="|\n".join("\n", @tmpl)."\n}}";
$fixed{"{{multiple issues}} (to new syntax)"}=1;
return $txt;
}
foreach ($api->process_paramlist(@$params)){
my $isvar=($istransclude && $_->{'value'}=~/\{\{\{.*\}\}\}/s);
return undef if $isvar;
my $k=defined($_->{'oname'})?$_->{'name'}.'=':'';
my $ov=$_->{'value'};
my $v=$ov;
if($_->{'name'}!~/^(?:\d+|section|article|text|expert|reason)$/){
$ov=~s/[ _]+/ /g;
$ov=~s/^\s+|\s+$//g;
$v=$ov;
$v=fixcurrent($api,$v) unless $istransclude;
$v=($v=~/^[\s=]*(?:|Undated|now|today|y|yes|1)$/i)?strftime('%B %Y', gmtime):fixdate($v);
if(!defined($v)){
$v=strftime('%B %Y', gmtime);
$api->log("Unrecognized \"date\" value $ov");
}
}
$needdate=1 if($_->{'name'} eq 'expert' && $v ne '');
$havedate=1 if($_->{'name'} eq 'date');
if($v ne $ov){
push @fixed, $_->{'name'}.'=';
$txt.="|$k$v";
} else {
$txt.='|'.$_->{'text'};
}
}
if($needdate && !$havedate){
$txt.="|date=".strftime('%B %Y', gmtime);
push @fixed, 'expert=';
}
return undef unless @fixed;
$txt.="}}";
$fixed{"{{$name|".join('|',@fixed)."}}"}=1;
return $txt;
}
# {{as of}} or {{update after}}?
if(exists($asof{"Template:$name"})){
my $isAsof = $asof{"Template:$name"} eq $asof{"Template:As of"};
my @pp=$api->process_paramlist(@$params);
my %pp=();
foreach (@pp){
$pp{$_->{'name'}}=$_;
}
my $ok=0;
my $ch=0;
my $c='';
my $m=(gmtime)[4]+1;
my $y=(gmtime)[5]+1900;
if(exists($pp{'date'}) && !exists($pp{'1'})){
$pp{'date'}{'name'}='1';
$pp{'date'}{'oname'}='';
$pp{'date'}{'text'}=$pp{'date'}{'value'};
$pp{'1'}=$pp{'date'};
delete $pp{'date'};
$ch=1;
}
# Ok, figure out what we have
my @v=(
exists($pp{'1'})?$pp{'1'}{'value'}//'':'',
exists($pp{'2'})?$pp{'2'}{'value'}//'':'',
exists($pp{'3'})?$pp{'3'}{'value'}//'':'',
);
my @ov=@v;
my @c=('','','');
my @t=();
for(my $i=0; $i<3; $i++){
$v[$i]=~s/^\s+|\s+$//g;
$c[$i].=$1 if $v[$i]=~s/((?>\s*<!--.*?-->)+)$//s;
if($v[$i]=~/^$monthre$/){
$t[$i]='m';
} elsif($v[$i]=~/^\d+$/){
if($v[$i]>=1000){
$t[$i]='y';
} elsif($v[$i]<=12){
$t[$i]='md';
} elsif($v[$i]<=31){
$t[$i]='d';
} else {
$t[$i]='?';
}
} elsif($v[$i] eq ''){
$t[$i]='e';
} else {
$t[$i]='?';
}
}
# If we have a month and something that could be a month or day,
# the something is a day. If we have a day and something that could
# be a month or a day, the something is a month. If we have an
# empty, a year, and a month/day, assume it's a month.
if(grep $_ eq 'md', @t){
if(grep $_ eq 'm', @t){
@t=map { $_ eq "md"?"d":$_ } @t;
} elsif(grep $_ eq 'd', @t){
@t=map { $_ eq "md"?"m":$_ } @t;
} elsif(join('-', sort @t) eq 'e-md-y'){
@t=map { $_ eq "md"?"m":$_ } @t;
}
}
# Assume the user knew what they were doing for y-md-md
my $t=join('-', @t);
@t=('y','m','d') if $t eq 'y-md-md';
# If we have two month/days but they're the same, it doesn't matter.
$t=join('-', sort @t);
if($t eq 'md-md-y' || $t eq 'e-md-md'){
if($t[1] eq 'e' || $t[1] eq 'y'){
@v=@v[1,0,2];
@c=@c[1,0,2];
@t=@t[1,0,2];
$ch=1;
} elsif($t[2] eq 'e' || $t[2] eq 'y'){
@v=@v[2,0,1];
@c=@c[2,0,1];
@t=@t[2,0,1];
$ch=1;
}
@t[1,2]=qw/m d/ if $v[1]==$v[2];
}
# Move any empties to the end
unless(join('-', @t) eq 'e-e-e'){
while($t[0] eq 'e'){
@t=@t[1,2,0];
@v=@v[1,2,0];
@c=@c[1,2,0];
$ch=1;
}
if($t[1] eq 'e' && $t[2] ne 'e'){
@t=@t[0,2,1];
@v=@v[0,2,1];
@c=@c[0,2,1];
$ch=1;
}
}
# Now, handle any cases we can recognize
$t=join('-', @t);
my $wasmd = $t =~ /m-(.-)*d/;
if($t eq 'y-m-d' || $t eq 'y-m-e' || $t eq 'y-e-e'){
# Will need a change if the month name is spelled wrong
$ch=1 unless $v[1] eq ($months{lc($v[1])} // $v[1]);
# Nothing to change here?
return undef unless $ch;
$ok=1;
} elsif($t eq 'm-y-d' || $t eq 'm-y-e'){
$ok=1;
@v=@v[1,0,2];
@c=@c[1,0,2];
} elsif($t eq 'y-d-m'){
$ok=1;
@v=@v[0,2,1];
@c=@c[0,2,1];
} elsif($t eq 'm-d-y'){
$ok=1;
@v=@v[2,0,1];
@c=@c[2,0,1];
} elsif($t eq 'd-y-m'){
$ok=1;
@v=@v[1,2,0];
@c=@c[1,2,0];
} elsif($t eq 'd-m-y'){
$ok=1;
@v=@v[2,1,0];
@c=@c[2,1,0];
} elsif($t eq 'd-m-e' || $t eq 'm-d-e'){
$ok=1;
@v=@v[1,0,2] if $t eq 'd-m-e';
@c=@c[1,0,2] if $t eq 'd-m-e';
my $mn=($v[0]=~/^\d+$/?$v[0]:$monthnum{$months{lc($v[0])}});
@v=($mn>$m?$y-1:$y, @v[0,1]);
@c=@c[2,0,1];
} elsif($t eq 'e-e-e'){
$ok=1;
@v=gmtime;
@v=($v[5]+1900, $v[4]+1, $v[3]);
} elsif($t eq '?-e-e'){
# The odd one, try to parse the first arg
my $v=$v[0];
my $isvar=($istransclude && $v=~/\{\{\{.*\}\}\}/s);
return undef if $isvar;
return undef if $v=~/^\s*\d+\s*$/;
$ok=1;
my $d=qr{[\s_/.,=-]};
$v=fixcurrent($api,$v) unless $istransclude;
$v=~s/^\s+|\s+$//g;
$c.=$1 if $v=~s/\s*(,)$//;
my $v2=$v; $v2=~s/^$d+|$d+$//g;
if($isAsof && chkasofdate($v2)){
# {{as of|Month Year}} works for years after 2004
return undef if chkasofdate($v.$c);
$v[0]=fixdate($v, \&nochkdate);
} elsif($v=~/^(?:(\d{1,2})(?:$d*(?i:st|nd|rd|th))?$d*)?($monthre)$d*(\d{4})?$/){
my $mn=$monthnum{$months{lc($2)}};
@v=($3//($mn>$m?$y-1:$y), $mn, $1//'');
} elsif($v=~/^($monthre)$d*(?:(\d{1,2})(?:$d*(?i:st|nd|rd|th))?$d+)?(\d{4})?$/){
$wasmd = 1;
my $mn=$monthnum{$months{lc($1)}};
@v=($3//($mn>$m?$y-1:$y), $mn, $2//'');
} elsif($v=~/^(\d{4})?$d*($monthre)(?:$d*(\d{1,2})(?:$d*(?i:st|nd|rd|th))?)?$/){
my $mn=$monthnum{$months{lc($2)}};
@v=($1//($mn>$m?$y-1:$y), $mn, $3//'');
} elsif($v=~/^(\d{4})$d+(\d{2})$d+(\d{2})$/){
@v=($1,$2,$3);
} elsif($v=~/^(\d{4})-(\d{2})$/){
@v=($1,$2,'');
} elsif($v=~/^(now|today)$/i){
@v=($y,$m,strftime('%d', gmtime));
} else {
$api->log("Unrecognized \"as of\" value $v");
return undef;
}
}
if(!$ok){
$api->log("Unrecognized \"as of\" value ".join('|', @ov));
return undef;
}
# Make sure month is valid
$v[1]=$months{lc($v[1])} unless $v[1] eq ($months{lc($v[1])} // $v[1]);
$v[0].=$c[0];
$v[1].=$c[1];
$v[2].=$c[2];
my $txt="{{$oname";
my $havedf = 0;
unshift @pp, { name=>'1' } unless exists($pp{'1'});
foreach (@pp){
if($_->{'name'} eq '1'){
$txt.="|".$v[0];
$txt.="|".$v[1] if($v[1] ne '' || $v[2] ne '' || exists($pp{'4'}));
$txt.="|".$v[2] if($v[2] ne '' || exists($pp{'4'}));
} elsif($_->{'name'} eq '2' || $_->{'name'} eq '3'){
# Skip
} else {
$havedf = 1 if $_->{'name'} eq 'df';
$txt.="|".$_->{'text'};
}
}
if ( !$havedf && $isAsof ) {
if ( !defined( $df ) ) {
$res = $api->query(
titles => $title,
prop => 'categories',
cllimit => 'max',
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load categories for $title: ".$res->{'error'}."\n");
return 60;
}
if ( grep { $_->{'title'} =~ /^Category:Use mdy dates/; } @{(values %{$res->{'query'}{'pages'}})[0]{'categories'}} ) {
$df = 'mdy';
} elsif ( grep { $_->{'title'} =~ /^Category:Use dmy dates/; } @{(values %{$res->{'query'}{'pages'}})[0]{'categories'}} ) {
$df = 'dmy';
} else {
$df = '';
}
}
$txt .= '|df=US' if ( $df eq 'mdy' || $df eq '' && $wasmd );
}
$txt.="}}".$c;
$fixed{"{{$name}}"}=1;
return $txt;
}
# {{disambiguation|cleanup}}? {{disambiguation|non-primary}}?
# {{disambiguation cleanup|cleanup}}? {{disambiguation cleanup|non-primary}}?
if(exists($disambig{"Template:$name"}) || exists($disambigcleanup{"Template:$name"})){
my $iscleanup = exists($disambigcleanup{"Template:$name"});
my @found_something=();
my $keep=$iscleanup;
my $found_date=undef;
my $need_date=$iscleanup;
my $tpl=$oname;
my $tparams='';
my %extra=();
foreach ($api->process_paramlist(@$params)){
my $isvar=($istransclude && $_->{'value'}=~/\{\{\{.*\}\}\}/s);
if($_->{'name'}=~/^[1-9][0-9]*$/){
if($_->{'value'}=~/^\s*(?:cleanup|clean up|clean-up)\s*$/){
my $v = $_->{'value'};
$v =~ s/^\s+|\s+$//g;
push @found_something, $v;
if ( ! $iscleanup ) {
$tpl="disambiguation cleanup";
$keep=1;
$need_date=1;
}
next;
}
if($_->{'value'}=~/^\s*(?:non-primary|non-primary topic|one non-primary topic)\s*$/){
my $v = $_->{'value'};
$v =~ s/^\s+|\s+$//g;
push @found_something, $v;
$extra{'one other topic'} = 1;
next;
}
$_->{'name'}-=scalar @found_something;
$_->{'oname'}=~s/[1-9][0-9]*/$_->{name}/ if defined($_->{'oname'});
}
if($_->{'name'} eq 'date'){
my $ov=$_->{'value'};
$ov=~s/[ _]+/ /g;
$ov=~s/^\s+|\s+$//g;
$ov=fixcurrent($api,$ov) unless $istransclude;
$_->{'value'}=($ov=~/^[\s=]*(?:|Undated|now|today|y|yes|1)$/i)?strftime('%B %Y', gmtime):fixdate($ov);
if(!defined($_->{'value'})){
$_->{'value'}=strftime('%B %Y', gmtime);
$api->log("Unrecognized \"date\" value $ov");
}
$found_date=$_->{'value'};
}
$tparams.='|';
$tparams.=$_->{'oname'}.'=' if defined($_->{'oname'});
$tparams.=$_->{'value'};
$keep=1 if $_->{'name'} ne 'date';;
}
return undef unless ( @found_something || $need_date && ! defined( $found_date ) );
my $txt = '';
if ( $keep ) {
$txt.="{{$tpl$tparams";
$txt.='|date='.strftime('%B %Y', gmtime) if ( $need_date && ! defined( $found_date ) );
$txt.="}}";
}
foreach my $t (keys %extra) {
$txt .= "\n" if $txt ne '';
$txt .= "{{$t|date=" . ( $found_date // strftime('%B %Y', gmtime) ) . "}}";
}
$fixed{"{{$name|" . join( "|", @found_something ) . "}}"}=1;
return $txt;
}
# Rules
if(exists($rules{"Template:$name"})){
for my $rule (@{$rules{"Template:$name"}}){
my %with_left = %{$rule->{'with'}};
my $dateparam = $rule->{'dateparameter'};
my $found_with='';
my $found_without=0;
my $found_date=0;
my $fixed_date=0;
# Check if we have the date parameter first
foreach ($api->process_paramlist(@$params)){
if($_->{'name'} eq $dateparam){
$found_date=1;
}
}
$oname=~s/^\s*(?i:Template\s*:\s*)?//;
my $txt="{{$oname";
foreach ($api->process_paramlist(@$params)){
my $isvar=($istransclude && $_->{'value'}=~/\{\{\{.*\}\}\}/s);
my $name=$_->{'name'};
if(exists($rule->{'ignore'}{$name})){
my $re=$rule->{'ignore'}{$name};
if($_->{'value'}=~/^(?:$re)$/s){
$txt.='|'.$_->{'text'};
next;
}
}
if(exists($rule->{'with'}{$name})){
my $re=$rule->{'with'}{$name};
if($_->{'value'}=~/^(?:$re)$/s){
$found_with.='|'.$_->{'text'};
delete $with_left{$name};
}
}
if(exists($rule->{'without'}{$name})){
my $re=$rule->{'without'}{$name};
$found_without=1 if $_->{'value'} !~ /^\s*(?><!--.*?-->\s*)*$/s and $_->{'value'}=~/^(?:$re)$/s;
}
my $ov=$_->{'value'};
$ov=~s/[ _]+/ /g;
$ov=~s/^\s+|\s+$//g;
my $v=$ov;
if(!defined($_->{'oname'}) && $name=~/^[1-5]$/ && !$found_date){
my $v2=$v;
$v2=fixcurrent($api,$v2) unless $istransclude;
if($v ne '' && defined(fixdate($v2)) && length($v2)<=30){
if(exists($rule->{'keep'}{$name})){
my $re=$rule->{'keep'}{$name};
$txt.='|'.$_->{'text'} if $v ne '' && $v=~/^(?:$re)$/s;
}
$name = $_->{'oname'} = $dateparam;
$fixed_date=1;
}
}
if($name eq $dateparam){
$found_date=1;
$v=fixcurrent($api,$v) unless $istransclude;
$v=($v=~/^[\s=]*(?:|Undated|now|today|y|yes|1)$/i)?strftime('%B %Y', gmtime):fixdate($v);
if(!defined($v)){
$v=strftime('%B %Y', gmtime);
$api->log("Unrecognized \"$dateparam\" value $ov");
}
$fixed_date=1 if $v ne $ov;
}
if($fixed_date || $v ne $ov){
$txt.='|'.$_->{'oname'}.'=' if defined($_->{'oname'});
$txt.=$v;
} else {
$txt.='|'.$_->{'text'};
}
}
next if($found_without || %with_left || ($found_date && !$fixed_date));
$txt.="|$dateparam=".strftime('%B %Y', gmtime) unless $found_date;
$txt.="}}";
$fixed{"{{$name$found_with}}"}=1;
return $txt;
}
# Rule templates override WP:AFD/DT
return undef;
}
# Any generic dated template?
return undef unless exists($templates{"Template:$name"});
my $found_date=0;
my $any=0;
$oname=~s/^\s*(?i:Template\s*:\s*)?//;
my $txt="{{$oname";
foreach ($api->process_paramlist(@$params)){
my $isvar=($istransclude && $_->{'value'}=~/\{\{\{.*\}\}\}/s);
my $ok=defined($_->{'oname'})?$_->{'name'}.'=':'';
my $k=$ok;
my $ov=$_->{'value'};
my $v=$ov;
$k='date=' if $k eq 'Date=';
$k='date=' if $k eq 'dates=';
if($k eq '' && $_->{'name'}=~/^[1-5]$/){
if($v=~/^[\s=]*(?:date|now)\s*$/i){ $any=1; next; }
my $v2=$v;
$v2=fixcurrent($api,$v2) unless $istransclude;
$k='date=' if($v ne '' && defined(fixdate($v2)) && length($v2)<=30 && ! $self->tpl_has_num_param( $api, $templates{"Template:$name"}, $_->{'name'}, \$fail ));
return undef if defined($fail);
}
if($k eq 'date='){
return undef if $isvar;
if($found_date){ $any=1; next; }
$found_date=1;
$ov=~s/[ _]+/ /g;
$ov=~s/^\s+|\s+$//g;
$v=$ov;
$v=fixcurrent($api,$v) unless $istransclude;
$v=fixdate($v);
if(!defined($v)){
$v=strftime('%B %Y', gmtime);
$api->log("Unrecognized \"date\" value $ov");
}
}
if($k ne $ok || $v ne $ov){
$any=1;
$txt.="|$k$v";
} else {
$txt.='|'.$_->{'text'};
}
}
$txt.="|date=".strftime('%B %Y', gmtime) unless $found_date;
$txt.="}}";
$fixed{"{{$name}}"}=1 if($any || !$found_date);
return ($any || !$found_date)?$txt:undef;
};
# Check tags
my @tags=@{$tok->{'revisions'}[0]{'tags'} // []};
for my $tag (@tags) {
if(exists($skiptags{$tag})){
$api->log("Skipping revision ".$tok->{'revisions'}[0]{'revid'}." of $title because of tag '$tag'");
return (undef,0);
}
}
# Get page text
my $intxt=$tok->{'revisions'}[0]{'slots'}{'main'}{'*'};
# Check for broken wrapper templates, and RfDs
my $iswrapper=0;
if($istransclude){
my $test=$intxt;
if($test=~m{<onlyinclude>} && $test=~m{</onlyinclude>}){
$test=join("", $test=~m{<onlyinclude>(.*?)</onlyinclude>}gs);
}
$test=~s{<!--.*?-->}{}gs;
$test=$api->strip_nowiki($test);
$test=~s{<noinclude>.*?</noinclude>}{}gs;
$test=~s{<noinclude\s*/>}{}g;
$test=~s{</?includeonly>}{}g;
$test=~s{<includeonly\s*/>}{}g;
$test=~s{<!--.*?-->}{}gs;
$test=$api->strip_nowiki($test);
$test=$api->process_templates($test, sub {
my $name=shift;
return '' if exists($mi{"Template:$name"});
return '' if exists($asof{"Template:$name"});
return '' if exists($disambig{"Template:$name"});
return '' if exists($rules{"Template:$name"});
return '' if exists($templates{"Template:$name"});
return undef;
});
$test=~s/\[\[\s*Category\s*:.*?\]\]//gi;
$test=~s/\s//g;
$iswrapper=($test eq '');
if(!exists($self->{'bypass'}{$title})){
my $redirre = $api->redirect_regex();
$redirre=~s/\^(\\s\*)/$1/;
if($intxt =~ m!^\{\{<includeonly>safesubst:</includeonly>#invoke:RfD\|\|.*\|content=\s*\n$redirre\[\[\s*(?i:Template)\s*:\s*([^]|]+?)\s*\]\]!s) {
my $target = $1;
$api->log("Found RfD $title => Template:$target, adding to bypass list");
$self->{'bypass'}{$title} = $1;
return (undef,1);
}
}
}
# Scan the page for templates needing dating
%fixed=();
my $outtxt=$api->process_templates($intxt, $checksub);
return ($fail,0) if defined($fail);
if(%fixed && $iswrapper){
$api->whine("[BRFA55] Possible broken wrapper template [[$title]]", "The page [[$title]] is transcluded in other pages and appears to consist of nothing but an invocation of a template that should be dated but isn't. Please fix it (most likely by adding {{para|date|<nowiki>{{{date|}}}</nowiki>}} to the dated template invocation), or fix me.");
return (undef,0);
}
my $didanything=0;
if($recurse && !%fixed){
# Nothing found in the page, check templates
my $iter=$api->iterator(
titles => $title,
generator => 'templates',
gtllimit => 'max',
prop => 'info|categories',
cllimit => 'max',
clcategories => join('|', keys %inuse)
);
while(my $t=$iter->next){
return (0,0) if $api->halting;
if(!$t->{'_ok_'}){
$api->warn("Failed to retrieve templates in $title: ".$t->{'error'}."\n");
return (60,0);
}
# Skip transcluded redirects, the real template will be listed
# later.
next if exists($t->{'redirect'});
# Deleted since we read the category?
next if exists($t->{'missing'});
# Skip things that aren't wikitext.
next if $t->{'contentmodel'} ne 'wikitext';
# Skip? Continue skipping.
if(exists($skip->{$t->{'lastrevid'}})){
$newskip->{$t->{'lastrevid'}}=1;
next;
}
my $ttitle=$t->{'title'};
# Skip the page itself. People these days really like writing Scribunto modules that try to load and parse the page's own wikitext.
next if $ttitle eq $title;
# Skip any of our target templates, they should themselves be dated.
# (and most are probably protected anyway)
next if exists($templates{$ttitle});
next if exists($asof{$ttitle});
next if exists($mi{$ttitle});
# Don't try fixing any page touched too recently, to give the real
# editor a chance to fix it.
my $until = $self->check_delay( $api, $t );
if ( $until > time() ) {
$waituntil=$until if $until<$waituntil;
next;
}
#$api->log("Checking for undated templates because of ".$iter->iterval." in $title");
my ($ret,$da)=$self->check_page($api, "$cat transcluded in $title", $ttitle, $t->{'lastrevid'}, 0, $skip, $newskip, $waituntil,1);
return $ret if defined($ret);
$didanything=1 if $da;
}
}
# Need to edit?
if($outtxt ne $intxt){
my $summary="Dating maintenance tags: ".join(' ', keys %fixed);
$summary="[BRFA55] $summary" if $istransclude;
$api->log("$summary in $title (because of $cat)");
$summary="Dating maintenance tags: [too many to list]" if length($summary)>500;
my $r=$api->edit($tok, $outtxt, $summary, 1, 1);
if($r->{'code'} ne 'success'){
$api->warn("Write failed on $title: ".$r->{'error'}."\n");
return (undef,$didanything);
} else {
return (0,$didanything) if $istransclude;
}
} else {
if($didanything){
# We just edited a template included in this page, retry it.
my @args = @_;
$args[5]=0; # No recursion this time
return check_page2(@args);
}
if($tok->{'ns'} != 0){
# Don't bother about fancy stuff in templates
$skip->{$tok->{'lastrevid'}}=1;
$newskip->{$tok->{'lastrevid'}}=1;
$api->store->{'skip'}=$skip;
return (undef,$didanything);
}
if(!$self->{'reloaded_template_redirects'}){
# Maybe someone just created a new redirect. Check for that.
my $res=$api->query([],
titles => $title,
generator => 'templates',
gtllimit => 'max',
redirects => 1,
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load templates in $title: ".$res->{'error'}."\n");
return (60,$didanything);
}
my $any=0;
foreach my $t (@{$res->{'query'}{'redirects'} // []}) {
my $from=$t->{'from'};
my $to=$t->{'to'};
if(!exists($templates{$from}) && exists($templates{$to})){
$any=1;
$templates{$from}=$templates{$to};
}
}
$self->{'reloaded_template_redirects'}=1;
if($any){
$self->{'templates'}=\%templates;
$api->store->{'templates'}=$self->{'templates'};
return check_page2(@_);
}
}
# Don't bother logging "invalid date" when it's probably because a
# valid maintenance category just doesn't exist yet.
my @missingcats=();
my $fixedcat=0;
my $notincat=0;
my $expensivecat=0;
my $expensivecat2=0;
$res=$api->query(
action => 'parse',
page => $title,
prop => 'categories',
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to parse $title: ".$res->{'error'}."\n");
return (60,$didanything);
}
my @cats=@{$res->{'parse'}{'categories'}};
@cats=map { $_->{'*'}=~s/_/ /g; 'Category:'.$_->{'*'}; } @cats;
if($cat eq 'Category:Articles with invalid date parameter in template'){
my $re=join('|', @months); $re=qr/$re/;
my @cats2=grep m/ from $re \d{4}$/, @cats;
if(@cats2){
$res=$api->query([],
titles => join('|',@cats2),
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to load categories for $title: ".$res->{'error'}."\n");
return (60,$didanything);
}
foreach (values %{$res->{'query'}{'pages'}}) {
push @missingcats, $_->{'title'} if(exists($_->{'missing'}));
}
}
$fixedcat=!grep $_ eq 'Category:Articles with invalid date parameter in template', @cats;
$expensivecat=grep $_ eq 'Category:Pages with too many expensive parser function calls', @cats;
$expensivecat2=grep $_ eq 'Category:Pages containing omitted template arguments', @cats;
} else {
$notincat=!grep $_ eq $cat, @cats;
}
if($notincat){
$api->log("Probable template screw-up in $title (regarding $cat)");
$api->query( action => 'purge', titles => $title, forcelinkupdate => 1 );
} elsif(@missingcats || $fixedcat){
my $mc=@missingcats?': '.join('; ', @missingcats):'';
$api->log("Probable missing date category in $title$mc");
$api->query( action => 'purge', titles => $title, forcelinkupdate => 1 ) if $fixedcat;
} elsif($expensivecat){
$api->log("Probable \"invalid\" date because of too many expensive parser function calls in $title");
$skip->{$tok->{'lastrevid'}}=1;
$newskip->{$tok->{'lastrevid'}}=1;
$api->store->{'skip'}=$skip;
} elsif($expensivecat2){
$api->log("Probable \"invalid\" date because of too huge of template arguments in $title");
$skip->{$tok->{'lastrevid'}}=1;
$newskip->{$tok->{'lastrevid'}}=1;
$api->store->{'skip'}=$skip;
} else {
$api->log("Nothing to do in $title (because of $cat)");
$skip->{$tok->{'lastrevid'}}=1;
$newskip->{$tok->{'lastrevid'}}=1;
$api->store->{'skip'}=$skip;
}
}
return (undef,$didanything);
}
sub check_delay {
my ($self, $api, $t) = @_;
my $title = $t->{'title'};
my $lastmod=ISO2timestamp($t->{'touched'});
if(time()-$lastmod<$min_delay){
#$api->log("$title touched too recently, leave it for later");
return $lastmod+$min_delay;
}
# Any page marked with {{inuse}} should be left for longer.
if(time()-$lastmod<$inuse_delay &&
grep { exists($inuse{$_->{'title'}}) } @{$t->{'categories'}}){
$api->log("$title marked {{inuse}} and last touched less than $inuse_delay seconds ago, leave it for later");
return $lastmod+$inuse_delay;
}
# To try to avoid "fixing" vandalism, we choose some arbitrary groups and
# edit count limits to trust and wait longer if the page hasn't been edited
# by someone "trusted" since someone "untrusted" edited.
my $res=$api->query([],
titles => $title,
prop => 'revisions',
rvprop => 'user|timestamp',
rvlimit => 'max',
rvend => timestamp2ISO(time()-$untrusted_delay)
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to retrieve revisions for $title: ".$res->{'error'}."\n");
return time()+60;
}
my (@users, %uts);
foreach (@{(values %{$res->{'query'}{'pages'}})[0]{'revisions'}}) {
next unless defined $_->{'user'};
next if exists($uts{$_->{'user'}});
push @users, $_->{'user'};
$uts{$_->{'user'}} = ISO2timestamp($_->{'timestamp'});
}
$res=$api->query([],
list => 'users',
usprop => 'editcount|groups',
ususers => join("|", @users)
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to retrieve edit counts for editors of $title: ".$res->{'error'}."\n");
return time()+60;
}
my %u=map { my $n=$_->{'name'}; "$n#g" => ($_->{'groups'} // []), "$n#e" => ($_->{'editcount'} // 0) } @{$res->{'query'}{'users'}};
foreach my $u (@users) {
next if grep(/^(?:bot)$/, @{$u{"$u#g"}}); # Skip bots
last if grep(/^(?:sysop|reviewer)$/, @{$u{"$u#g"}}); # Trust these
last if $u{"$u#e"}>$arbitrary_trusted_threshold; # Trust these too
next if $u{"$u#e"}>$arbitrary_untrusted_threshold; # Neutral on these
$api->log("$title touched too recently by untrusted user $u");
return $uts{$u}+$untrusted_delay;
}
return 0;
}
sub tpl_has_num_param {
my ( $self, $api, $name, $num, $fail ) = @_;
return $self->{'tpl num param cache'}{$name}[$num] // 0 if defined( $self->{'tpl num param cache'}{ $name } );
my $res = $api->query(
action => 'templatedata',
titles => $name,
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to get templatedata for $name: ".$res->{'error'}."\n");
$$fail = 60;
return undef;
}
$self->{'tpl num param cache'}{$name} = [];
my $page = (values %{$res->{'pages'}})[0] // {};
my %params = %{$page->{'params'} // {}};
while(my ($k,$v)=each %params){
my @nums = ();
my $isdate = $k eq 'date';
push @nums, $k if $k =~ /^\d+$/;
foreach my $a (@{$v->{'aliases'} // []}) {
push @nums, $a if $a =~ /^\d+$/;
$isdate = 1 if $a eq 'date';
}
foreach my $n (@nums) {
$self->{'tpl num param cache'}{$name}[$n] = ! $isdate;
}
}
return $self->{'tpl num param cache'}{$name}[$num] // 0;
}
sub check_direct_cat_use {
my ( $self, $api, $tok, $cat, $title ) = @_;
# Get page text
my $revid=$tok->{'revisions'}[0]{'revid'};
my $intxt=$tok->{'revisions'}[0]{'slots'}{'main'}{'*'};
my $nowiki = {};
my $outtxt = $api->strip_nowiki( $intxt, $nowiki );
# Construct a regex matching the category wikitext.
my $catre = '(?i:' . quotemeta( substr( $cat, 9, 1 ) ) . ')' . quotemeta( substr( $cat, 10 ) );
$catre =~ s/(?:\\?\s)+/[\\s_]+/g;
$catre = qr/\[\[\s*(?i:Category)\s*:\s*$catre\s*(?:\|[^]]*)?\]\]/;
# This check only applies if the category is directly used in the page.
while ( $outtxt =~ s/\s*\n\s*$catre\s*?(\n\s*|$)/$1/g ){}
$outtxt = $api->replace_nowiki( $outtxt, $nowiki );
return (undef, 0) if $outtxt eq $intxt;
# Only remove it by bot if there's already an appropriate template on the page.
# If there's not already an appropriate template, better for a human to decide which template to add (if any).
my %tpl = ();
$api->process_templates($outtxt, sub {
my $name=shift;
shift; # $params
shift; # $wikitext
shift; # $data
my $oname=shift;
$tpl{$name} = 1 if exists($self->{'templates'}{"Template:$name"});
return undef;
} );
return (undef, 0) unless %tpl;
my $res = $api->query(
action => 'expandtemplates',
title => $title,
revid => $revid,
text => join( "\n\n", map { "{{$_}}\n🤖🤖🤖 AnomieBOT TagDater $_ 🤖🤖🤖" } keys %tpl ),
prop => 'wikitext',
formatversion => 2,
);
if($res->{'code'} ne 'success'){
$api->warn("Failed to expand templates extracted from $title: ".$res->{'error'});
return (60,0);
}
my $txt = $api->strip_nowiki( $res->{'expandtemplates'}{'wikitext'} );
unless ( $txt =~ /$catre.*?\n🤖🤖🤖 AnomieBOT TagDater (.+?) 🤖🤖🤖(?:\n|$)/ ) {
$api->log( "Found direct use of [[$cat]] in $title, but no existing template. Leaving for a human." );
return (undef, 0);
}
my $tpl = $1;
my $summary = "[[$cat]] should not be used directly. The template {{$tpl}} already on the page already handles the categorization correctly.";
# Try to find who added the cat to the page.
my $ts = ISO2timestamp( $tok->{'revisions'}[0]{'timestamp'} );
my $iter = $api->iterator(
titles => $title,
prop => 'revisions',
rvprop => 'ids|user|timestamp|content',
rvslots => 'main',
rvlimit => 1,
rvstartid => $revid,
rvdir => 'older',
formatversion => 2,
);
my ($user, $rev);
while($_=$iter->next){
return (0,0) if $api->halting;
if(!$_->{'_ok_'}){
$api->warn("Failed to retrieve revision from $title: ".$_->{'error'}."\n");
return (60,0);
}
if ( exists($_->{'revisions'}[0]{'slots'}{'main'}{'texthidden'}) || exists($_->{'revisions'}[0]{'slots'}{'main'}{'userhidden'}) ) {
$user = undef;
} else {
my $txt = $api->strip_nowiki( $_->{'revisions'}[0]{'slots'}{'main'}{'content'} );
last unless $txt =~ /$catre/;
$rev = $_->{'revisions'}[0]{'revid'};
$user = $_->{'revisions'}[0]{'user'};
}
# Only look back a few months at most, longer than that is not worth the effort.
# Mostly the bot should either catch it right away or never, but it's possible someone added the cat and then later someone added a corresponding template.
if ( ISO2timestamp( $_->{'revisions'}[0]{'timestamp'} ) < $ts - 86400 * 90 ) {
$user = undef;
last;
}
}
$summary .= " ([[Special:Diff/$rev|added]] by [[User:$user]])" if defined( $user );
$api->log( "$summary in $title" );
my $r=$api->edit($tok, $outtxt, $summary, 0, 1);
if($r->{'code'} ne 'success'){
$api->warn("Write failed on $title: ".$r->{'error'}."\n");
return (undef,0);
} else {
return (undef,1);
}
}
# This function can be used to run the bot over arbitrary page content.
# Something like:
# perl -we 'use tasks::TagDater; tasks::TagDater::unit_test($cat,$revid,$flag[,$filename]);'
# Flags:
# 1 = is transclusion (also forces don't-recurse)
# 2 = don't recurse
sub unit_test {
my $cat=shift;
my $revid=shift;
my $flag=shift;
my $filename=shift//undef;
$|=1;
binmode STDOUT, ':utf8';
binmode STDERR, ':utf8';
my $dir="/tmp/anomiebot-test";
die "Could not create directory $dir: $!\n" if(!-d $dir && !mkdir($dir));
if(-e $dir.'/test'){
unlink($dir.'/test');
die "Could not remove test file in $dir: $!\n" if(-e $dir.'/test');
}
open(X, ">", $dir.'/test') or die("Could not create test file in $dir: $!\n");
close(X);
unlink($dir.'/test');
my $self=tasks::TagDater->new();
my $api=AnomieBOT::API->new('conf.ini', 7);
$api->{'noedit'}=$dir;
$api->login();
$api->DEBUG(-1);
$api->task('TagDater',0,10,qw/d::Talk d::Redirects d::Templates/);
my $res=$self->refresh_cache($api);
die "init failed\n" if defined($res);
$res=$api->query(revids=>$revid, prop=>'revisions', rvprop=>'ids|timestamp|content|flags|user|size|comment|tags', rvslots=>'main');
die "Failed to fetch info for revid $revid: ".$res->{'error'}."\n" if $res->{'code'} ne 'success';
$res=(values %{$res->{'query'}{'pages'}})[0];
my $title=$res->{'title'};
my $tok=$api->edittoken($title, EditRedir=>1);
die "Failed to get edit token: ".$tok->{'error'}."\n" if $tok->{'code'} ne 'success';
die "Page missing\n" if exists($tok->{'missing'});
$tok->{'revisions'}=$res->{'revisions'};
$tok->{'lastrevid'}=$revid;
if($filename){
open X, '<:utf8', $filename or die "Could not open $filename: $!\n";
{ local $/=undef; $tok->{'revisions'}[0]{'slots'}{'main'}{'*'}=<X>; }
close X;
}
my $waituntil=0;
if ( ! ( $flag & 1 ) ) {
my ($ret,$didanything) = $self->check_direct_cat_use($api,$tok,$cat,$title);
print Data::Dumper->Dump([$ret,$didanything], [qw/ret didanything/]);
}
my ($ret,$didanything)=$self->check_page2($api, $tok, $cat, $title, !($flag&3), {}, {}, \$waituntil, $flag&1);
print Data::Dumper->Dump([$ret,$didanything], [qw/ret didanything/]);
}
1;