urosm
/
ikiwiki
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changed to a binary index file, written using Storable, for speed 

During refresh of a wiki with 800 files, loadindex was using more total
time than any other function, and saveindex was also in the top ten.
Rewriting them to use Storable makes them three times as fast.

0.7 seconds is saved on my laptop in profiling mode.
master
Joey Hess 2008-03-21 09:07:44 -04:00
parent bf7360347e
commit 213eb2e408
2 changed files with 47 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
IkiWiki.pm
View File

@ -7,6 +7,7 @@ use Encode;
use HTML::Entities; use HTML::Entities;
use URI::Escape q{uri_escape_utf8}; use URI::Escape q{uri_escape_utf8};
use POSIX; use POSIX;
use Storable;
use open qw{:utf8 :std}; use open qw{:utf8 :std};
use vars qw{%config %links %oldlinks %pagemtime %pagectime %pagecase use vars qw{%config %links %oldlinks %pagemtime %pagectime %pagecase
@ -896,39 +897,39 @@ sub loadindex () { #{{{
%oldrenderedfiles=%pagectime=(); %oldrenderedfiles=%pagectime=();
if (! $config{rebuild}) { if (! $config{rebuild}) {
%pagesources=%pagemtime=%oldlinks=%links=%depends= %pagesources=%pagemtime=%oldlinks=%links=%depends=
%destsources=%renderedfiles=%pagecase=%pagestate=(); %destsources=%renderedfiles=%pagecase=%pagestate=();
} }
open (my $in, "<", "$config{wikistatedir}/index") || return; open (my $in, "<", "$config{wikistatedir}/index") || return;
while (<$in>) { my $ret=Storable::fd_retrieve($in);
$_=possibly_foolish_untaint($_); if (! defined $ret) {
chomp; return 0;
my %items; }
$items{link}=[]; my %index=%$ret;
$items{dest}=[]; foreach my $page (keys %index) {
foreach my $i (split(/ /, $_)) { my %d=%{$index{$page}};
my ($item, $val)=split(/=/, $i, 2); $pagectime{$page}=$d{ctime};
push @{$items{$item}}, decode_entities($val);
}
next unless exists $items{src}; # skip bad lines for now
my $page=pagename($items{src}[0]);
if (! $config{rebuild}) { if (! $config{rebuild}) {
$pagesources{$page}=$items{src}[0]; $pagemtime{$page}=$d{mtime};
$pagemtime{$page}=$items{mtime}[0]; $pagesources{$page}=$d{src};
$oldlinks{$page}=[@{$items{link}}]; $renderedfiles{$page}=$d{dest};
$links{$page}=[@{$items{link}}]; if (exists $d{links} && ref $d{links}) {
$depends{$page}=$items{depends}[0] if exists $items{depends}; $links{$page}=$d{links};
$destsources{$_}=$page foreach @{$items{dest}}; $oldlinks{$page}=[@{$d{links}}];
$renderedfiles{$page}=[@{$items{dest}}]; }
$pagecase{lc $page}=$page; if (exists $d{depends}) {
foreach my $k (grep /_/, keys %items) { $depends{$page}=$d{depends};
my ($id, $key)=split(/_/, $k, 2); }
$pagestate{$page}{decode_entities($id)}{decode_entities($key)}=$items{$k}[0]; if (exists $d{state}) {
$pagestate{$page}=$d{state};
} }
} }
$oldrenderedfiles{$page}=[@{$items{dest}}]; $oldrenderedfiles{$page}=[@{$d{dest}}];
$pagectime{$page}=$items{ctime}[0]; }
foreach my $page (keys %pagesources) {
$pagecase{lc $page}=$page;
}
foreach my $page (keys %renderedfiles) {
$destsources{$_}=$page foreach @{$renderedfiles{$page}};
} }
return close($in); return close($in);
} #}}} } #}}}
@ -938,9 +939,9 @@ sub saveindex () { #{{{
my %hookids; my %hookids;
foreach my $type (keys %hooks) { foreach my $type (keys %hooks) {
$hookids{encode_entities($_)}=1 foreach keys %{$hooks{$type}}; $hookids{$_}=1 foreach keys %{$hooks{$type}};
} }
my @hookids=sort keys %hookids; my @hookids=keys %hookids;
if (! -d $config{wikistatedir}) { if (! -d $config{wikistatedir}) {
mkdir($config{wikistatedir}); mkdir($config{wikistatedir});
@ -948,26 +949,32 @@ sub saveindex () { #{{{
my $newfile="$config{wikistatedir}/index.new"; my $newfile="$config{wikistatedir}/index.new";
my $cleanup = sub { unlink($newfile) }; my $cleanup = sub { unlink($newfile) };
open (my $out, '>', $newfile) || error("cannot write to $newfile: $!", $cleanup); open (my $out, '>', $newfile) || error("cannot write to $newfile: $!", $cleanup);
my %index;
foreach my $page (keys %pagemtime) { foreach my $page (keys %pagemtime) {
next unless $pagemtime{$page}; next unless $pagemtime{$page};
my $line="mtime=$pagemtime{$page} ".
"ctime=$pagectime{$page} ". $index{$page}={
"src=$pagesources{$page}"; ctime => $pagectime{$page},
$line.=" dest=$_" foreach @{$renderedfiles{$page}}; mtime => $pagemtime{$page},
my %count; src => $pagesources{$page},
$line.=" link=$_" foreach grep { ++$count{$_} == 1 } @{$links{$page}}; dest => $renderedfiles{$page},
links => $links{$page},
};
if (exists $depends{$page}) { if (exists $depends{$page}) {
$line.=" depends=".encode_entities($depends{$page}, " \t\n"); $index{$page}{depends} = $depends{$page};
} }
if (exists $pagestate{$page}) { if (exists $pagestate{$page}) {
foreach my $id (@hookids) { foreach my $id (@hookids) {
foreach my $key (keys %{$pagestate{$page}{$id}}) { foreach my $key (keys %{$pagestate{$page}{$id}}) {
$line.=' '.$id.'_'.encode_entities($key, " \t\n")."=".encode_entities($pagestate{$page}{$id}{$key}, " \t\n"); $index{$page}{state}{$id}{$key}=$pagestate{$page}{$id}{$key};
} }
} }
} }
print $out $line."\n" || error("failed writing to $newfile: $!", $cleanup);
} }
my $ret=Storable::nstore_fd(\%index, $out);
return if ! defined $ret || ! $ret;
close $out || error("failed saving to $newfile: $!", $cleanup); close $out || error("failed saving to $newfile: $!", $cleanup);
rename($newfile, "$config{wikistatedir}/index") || rename($newfile, "$config{wikistatedir}/index") ||
error("failed renaming $newfile to $config{wikistatedir}/index", $cleanup); error("failed renaming $newfile to $config{wikistatedir}/index", $cleanup);

1
debian/changelog vendored
View File

@ -58,6 +58,7 @@ ikiwiki (2.41) UNRELEASED; urgency=low
* smiley: Detect smileys inside pre and code tags, and do not expand. * smiley: Detect smileys inside pre and code tags, and do not expand.
* inline: Crazy optimisation to work around slow markdown. * inline: Crazy optimisation to work around slow markdown.
* Precompile pagespecs, about 10% overall speedup. * Precompile pagespecs, about 10% overall speedup.
* Changed to a binary index file, written using Storable, for speed.
-- martin f. krafft <madduck@debian.org> Sun, 02 Mar 2008 17:46:38 +0100 -- martin f. krafft <madduck@debian.org> Sun, 02 Mar 2008 17:46:38 +0100