From d92f767fb772c9b11293134bd67bc1261aea8f1e Mon Sep 17 00:00:00 2001
From: Simon McVittie
Date: Thu, 27 Aug 2009 23:30:41 +0100
Subject: [PATCH 1/7] inline: if using pagenames, don't add a dependency on
"page1 or page2 or..."
This is unnecessary and just slows us down (by a factor of 2, in the
pessimal case where every page has an inline with pagenames); it's also
not possible to optimize it into add_depends_exact calls.
---
IkiWiki/Plugin/inline.pm | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/IkiWiki/Plugin/inline.pm b/IkiWiki/Plugin/inline.pm
index a501566b5..d5ad11d43 100644
--- a/IkiWiki/Plugin/inline.pm
+++ b/IkiWiki/Plugin/inline.pm
@@ -195,10 +195,10 @@ sub preprocess_inline (@) {
@list = map { bestlink($params{page}, $_) }
split ' ', $params{pagenames};
-
- $params{pages} = join(" or ", @list);
}
else {
+ add_depends($params{page}, $params{pages});
+
@list = pagespec_match_list(
[ grep { $_ ne $params{page} } keys %pagesources ],
$params{pages}, location => $params{page});
@@ -247,7 +247,6 @@ sub preprocess_inline (@) {
@list=@list[0..$params{show} - 1];
}
- add_depends($params{page}, $params{pages});
# Explicitly add all currently displayed pages as dependencies, so
# that if they are removed or otherwise changed, the inline will be
# sure to be updated.
From c80a3cbcfde1a00ffdb0e62f70ee3a7d6bf442df Mon Sep 17 00:00:00 2001
From: Simon McVittie
Date: Thu, 27 Aug 2009 23:25:58 +0100
Subject: [PATCH 2/7] Add depends_exact: simplified dependency tracking for
dependencies on a single page
Let E be the number of dependencies per page of the form "A depends on B and
nothing else", let D be the number of other dependencies per page,
let P be the total number of pages, and let C be the number of changed
pages in a refresh.
This patch should speed up a refresh from O(E*C*P + D*C*P) to
O(C + E*P + D*C*P), assuming that hash lookups are O(1).
In practice, plugins like inline and map produce a lot of these very simple
dependencies, and my album plugin's combination of inline with a large
number of pages causes it to suffer particularly badly.
In testing on a wiki with about 7000 objects (3500 full pages, 3500
images), a full rebuild continued to take about 5:30, and a refresh
after touching about 350 pages and 350 images reduced from 5:30 to 1:30.
As with my previous optimizations, this change will result in downgrades not
working correctly until the wiki is rebuilt.
---
IkiWiki.pm | 23 ++++++++++++++++++++---
IkiWiki/Render.pm | 17 +++++++++++++++++
2 files changed, 37 insertions(+), 3 deletions(-)
diff --git a/IkiWiki.pm b/IkiWiki.pm
index 5563a03af..f6a985556 100644
--- a/IkiWiki.pm
+++ b/IkiWiki.pm
@@ -14,14 +14,14 @@ use open qw{:utf8 :std};
use vars qw{%config %links %oldlinks %pagemtime %pagectime %pagecase
%pagestate %wikistate %renderedfiles %oldrenderedfiles
%pagesources %destsources %depends %hooks %forcerebuild
- %loaded_plugins};
+ %loaded_plugins %depends_exact};
use Exporter q{import};
our @EXPORT = qw(hook debug error template htmlpage add_depends pagespec_match
pagespec_match_list bestlink htmllink readfile writefile
pagetype srcfile pagename displaytime will_render gettext urlto
targetpage add_underlay pagetitle titlepage linkpage
- newpagefile inject add_link
+ newpagefile inject add_link add_depends_exact
%config %links %pagestate %wikistate %renderedfiles
%pagesources %destsources);
our $VERSION = 3.00; # plugin interface version, next is ikiwiki version
@@ -1475,7 +1475,8 @@ sub loadindex () {
%oldrenderedfiles=%pagectime=();
if (! $config{rebuild}) {
%pagesources=%pagemtime=%oldlinks=%links=%depends=
- %destsources=%renderedfiles=%pagecase=%pagestate=();
+ %destsources=%renderedfiles=%pagecase=%pagestate=
+ %depends_exact=();
}
my $in;
if (! open ($in, "<", "$config{wikistatedir}/indexdb")) {
@@ -1515,6 +1516,11 @@ sub loadindex () {
$links{$page}=$d->{links};
$oldlinks{$page}=[@{$d->{links}}];
}
+ if (exists $d->{depends_exact}) {
+ $depends_exact{$page}={
+ map { $_ => 1 } @{$d->{depends_exact}}
+ };
+ }
if (exists $d->{dependslist}) {
$depends{$page}={
map { $_ => 1 } @{$d->{dependslist}}
@@ -1570,6 +1576,10 @@ sub saveindex () {
$index{page}{$src}{dependslist} = [ keys %{$depends{$page}} ];
}
+ if (exists $depends_exact{$page}) {
+ $index{page}{$src}{depends_exact} = [ keys %{$depends_exact{$page}} ];
+ }
+
if (exists $pagestate{$page}) {
foreach my $id (@hookids) {
foreach my $key (keys %{$pagestate{$page}{$id}}) {
@@ -1744,6 +1754,13 @@ sub add_depends ($$) {
return 1;
}
+sub add_depends_exact ($$) {
+ my $page = shift;
+ my $dep = shift;
+
+ $depends_exact{$page}{$dep} = 1;
+}
+
sub file_pruned ($$) {
require File::Spec;
my $file=File::Spec->canonpath(shift);
diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm
index da2d7b4cc..121fc97e3 100644
--- a/IkiWiki/Render.pm
+++ b/IkiWiki/Render.pm
@@ -210,6 +210,7 @@ sub render ($) {
if (defined $type) {
my $page=pagename($file);
delete $depends{$page};
+ delete $depends_exact{$page};
will_render($page, htmlpage($page), 1);
return if $type=~/^_/;
@@ -224,6 +225,7 @@ sub render ($) {
}
else {
delete $depends{$file};
+ delete $depends_exact{$file};
will_render($file, $file, 1);
if ($config{hardlink}) {
@@ -431,6 +433,7 @@ sub refresh () {
# internal pages are not rendered
my $page=pagename($file);
delete $depends{$page};
+ delete $depends_exact{$page};
foreach my $old (@{$renderedfiles{$page}}) {
delete $destsources{$old};
}
@@ -454,10 +457,24 @@ sub refresh () {
if (%rendered || @del || @internal) {
my @changed=(keys %rendered, @del);
+ my %changedpages = map { pagename($_) => 1 } @changed;
+
# rebuild dependant pages
F: foreach my $f (@$files) {
next if $rendered{$f};
my $p=pagename($f);
+
+ if (exists $depends_exact{$p}) {
+ foreach my $d (keys %{$depends_exact{$p}}) {
+ if (exists $changedpages{$d}) {
+ debug(sprintf(gettext("building %s, which depends on %s"), $f, $p));
+ render($f);
+ $rendered{$f}=1;
+ next F;
+ }
+ }
+ }
+
if (exists $depends{$p}) {
foreach my $d (keys %{$depends{$p}}) {
my $sub=pagespec_translate($d);
From ec2367cfbfb0053c61bb394af5ac6f5f930325fd Mon Sep 17 00:00:00 2001
From: Simon McVittie
Date: Fri, 28 Aug 2009 02:11:26 +0100
Subject: [PATCH 3/7] Fix typo in dependency debug message
---
IkiWiki/Render.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm
index 121fc97e3..f1ee140db 100644
--- a/IkiWiki/Render.pm
+++ b/IkiWiki/Render.pm
@@ -467,7 +467,7 @@ sub refresh () {
if (exists $depends_exact{$p}) {
foreach my $d (keys %{$depends_exact{$p}}) {
if (exists $changedpages{$d}) {
- debug(sprintf(gettext("building %s, which depends on %s"), $f, $p));
+ debug(sprintf(gettext("building %s, which depends on %s"), $f, $d));
render($f);
$rendered{$f}=1;
next F;
From b6992ec56629f4a893c7f016aaa81be25326356b Mon Sep 17 00:00:00 2001
From: Simon McVittie
Date: Fri, 28 Aug 2009 15:41:26 +0100
Subject: [PATCH 4/7] Auto-detect "simple dependencies" instead of requiring
callers to use add_depends_exact()
---
IkiWiki.pm | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/IkiWiki.pm b/IkiWiki.pm
index f6a985556..e4fcf1d6f 100644
--- a/IkiWiki.pm
+++ b/IkiWiki.pm
@@ -21,7 +21,7 @@ our @EXPORT = qw(hook debug error template htmlpage add_depends pagespec_match
pagespec_match_list bestlink htmllink readfile writefile
pagetype srcfile pagename displaytime will_render gettext urlto
targetpage add_underlay pagetitle titlepage linkpage
- newpagefile inject add_link add_depends_exact
+ newpagefile inject add_link
%config %links %pagestate %wikistate %renderedfiles
%pagesources %destsources);
our $VERSION = 3.00; # plugin interface version, next is ikiwiki version
@@ -1748,19 +1748,19 @@ sub add_depends ($$) {
my $page=shift;
my $pagespec=shift;
+ if ($pagespec =~ /$config{wiki_file_regexp}/ &&
+ $pagespec !~ /[\s*?()!]/) {
+ # a simple dependency, which can be matched by string eq
+ $depends_exact{$page}{$pagespec} = 1;
+ return 1;
+ }
+
return unless pagespec_valid($pagespec);
$depends{$page}{$pagespec} = 1;
return 1;
}
-sub add_depends_exact ($$) {
- my $page = shift;
- my $dep = shift;
-
- $depends_exact{$page}{$dep} = 1;
-}
-
sub file_pruned ($$) {
require File::Spec;
my $file=File::Spec->canonpath(shift);
From 2a7f3b91d4fa46a67d04f963785f957563ac0cfc Mon Sep 17 00:00:00 2001
From: Simon McVittie
Date: Fri, 28 Aug 2009 15:22:07 +0100
Subject: [PATCH 5/7] Force %depends_exact to lower case, fixing incorrect
case-sensitivity
---
IkiWiki.pm | 2 +-
IkiWiki/Render.pm | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/IkiWiki.pm b/IkiWiki.pm
index e4fcf1d6f..8bded8a0e 100644
--- a/IkiWiki.pm
+++ b/IkiWiki.pm
@@ -1751,7 +1751,7 @@ sub add_depends ($$) {
if ($pagespec =~ /$config{wiki_file_regexp}/ &&
$pagespec !~ /[\s*?()!]/) {
# a simple dependency, which can be matched by string eq
- $depends_exact{$page}{$pagespec} = 1;
+ $depends_exact{$page}{lc $pagespec} = 1;
return 1;
}
diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm
index f1ee140db..5367b1309 100644
--- a/IkiWiki/Render.pm
+++ b/IkiWiki/Render.pm
@@ -457,7 +457,7 @@ sub refresh () {
if (%rendered || @del || @internal) {
my @changed=(keys %rendered, @del);
- my %changedpages = map { pagename($_) => 1 } @changed;
+ my %lcchanged = map { lc(pagename($_)) => 1 } @changed;
# rebuild dependant pages
F: foreach my $f (@$files) {
@@ -466,7 +466,7 @@ sub refresh () {
if (exists $depends_exact{$p}) {
foreach my $d (keys %{$depends_exact{$p}}) {
- if (exists $changedpages{$d}) {
+ if (exists $lcchanged{$d}) {
debug(sprintf(gettext("building %s, which depends on %s"), $f, $d));
render($f);
$rendered{$f}=1;
From 20a38fbf6d4fda7d811239ae2980a166bc32ad0d Mon Sep 17 00:00:00 2001
From: Simon McVittie
Date: Fri, 28 Aug 2009 15:28:52 +0100
Subject: [PATCH 6/7] Avoid duplicating debug message for building a page due
to a dependency
As per Joey's review
---
IkiWiki/Render.pm | 25 ++++++++++++++-----------
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm
index 5367b1309..85ae0f8e9 100644
--- a/IkiWiki/Render.pm
+++ b/IkiWiki/Render.pm
@@ -460,23 +460,22 @@ sub refresh () {
my %lcchanged = map { lc(pagename($_)) => 1 } @changed;
# rebuild dependant pages
- F: foreach my $f (@$files) {
+ foreach my $f (@$files) {
next if $rendered{$f};
my $p=pagename($f);
+ my $reason = undef;
if (exists $depends_exact{$p}) {
foreach my $d (keys %{$depends_exact{$p}}) {
if (exists $lcchanged{$d}) {
- debug(sprintf(gettext("building %s, which depends on %s"), $f, $d));
- render($f);
- $rendered{$f}=1;
- next F;
+ $reason = $d;
+ last;
}
}
}
- if (exists $depends{$p}) {
- foreach my $d (keys %{$depends{$p}}) {
+ if (exists $depends{$p} && ! defined $reason) {
+ D: foreach my $d (keys %{$depends{$p}}) {
my $sub=pagespec_translate($d);
next if $@ || ! defined $sub;
@@ -487,14 +486,18 @@ sub refresh () {
next if $file eq $f;
my $page=pagename($file);
if ($sub->($page, location => $p)) {
- debug(sprintf(gettext("building %s, which depends on %s"), $f, $page));
- render($f);
- $rendered{$f}=1;
- next F;
+ $reason = $page;
+ last D;
}
}
}
}
+
+ if (defined $reason) {
+ debug(sprintf(gettext("building %s, which depends on %s"), $f, $reason));
+ render($f);
+ $rendered{$f}=1;
+ }
}
# handle backlinks; if a page has added/removed links,
From 54b3d55aadf9c6ee2b4b3aa68b3a1794a9394fd4 Mon Sep 17 00:00:00 2001
From: Simon McVittie
Date: Fri, 28 Aug 2009 15:50:02 +0100
Subject: [PATCH 7/7] Mark as done
---
doc/todo/optimize_simple_dependencies.mdwn | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/todo/optimize_simple_dependencies.mdwn b/doc/todo/optimize_simple_dependencies.mdwn
index 44163311b..50ecafa05 100644
--- a/doc/todo/optimize_simple_dependencies.mdwn
+++ b/doc/todo/optimize_simple_dependencies.mdwn
@@ -47,6 +47,10 @@ equally valid.)
--[[smcv]]
+> Now [[merged|done]] --[[smcv]]
+
+----
+
> We discussed this on irc; I had some worries that things may have been
> switched to `add_depends_exact` that were not pure page names. My current
> feeling is it's all safe, but who knows. It's easy to miss something.