From 7166c2d24491d9e80ad0a455c6b7220485853bfb Mon Sep 17 00:00:00 2001 From: Greg Banks Date: Fri, 4 Mar 2011 16:31:07 +1100 Subject: [PATCH] Add some more testing scripts Useful for setting up data to test an IMAP instance. - split-by-thread.pl - given an mbox format file, read it and split it into multiple smaller mbox files, one per thread. - imap-append.pl - given an mbox format file, read it and append all its messages to an IMAP format. - sprinkle.pl - given an mbox format file, read it and sprinkle it's messages over several dozen new IMAP folders with random names below a given root folder name. --- imap-append.pl | 61 ++++++++++++++++++++++ split-by-thread.pl | 125 +++++++++++++++++++++++++++++++++++++++++++++ sprinkle.pl | 106 ++++++++++++++++++++++++++++++++++++++ tominstall.sh | 3 ++ 4 files changed, 295 insertions(+) create mode 100755 imap-append.pl create mode 100755 split-by-thread.pl create mode 100755 sprinkle.pl diff --git a/imap-append.pl b/imap-append.pl new file mode 100755 index 000000000..cc7dfaf80 --- /dev/null +++ b/imap-append.pl @@ -0,0 +1,61 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use DateTime; +use Cassandane::Generator; +use Cassandane::Util::DateTime qw(to_iso8601); +use Cassandane::MessageStoreFactory; + +sub usage +{ + die "Usage: imap-append.pl [ -f format [maildir] | -u uri]"; +} + +my %params; +while (my $a = shift) +{ + if ($a eq '-f') + { + usage() if defined $params{uri}; + $params{type} = shift; + } + elsif ($a eq '-u') + { + usage() if defined $params{type}; + $params{uri} = shift; + } + elsif ($a eq '-v') + { + $params{verbose} = 1; + } + elsif ($a =~ m/^-/) + { + usage(); + } + else + { + usage() if defined $params{filename}; + $params{filename} = $a; + } +} + +my $imap_store = Cassandane::MessageStoreFactory->create( + type => 'imap', + host => 'storet1m.internal', + port => 2143, + folder => 'inbox', + username => 'muttster@vmtom.com', + password => 'testpw', + verbose => 1, + ); +my $mbox_store = Cassandane::MessageStoreFactory->create(%params); + +$imap_store->write_begin(); +$mbox_store->read_begin(); +while (my $msg = $mbox_store->read_message()) +{ + $imap_store->write_message($msg); +} +$mbox_store->read_end(); +$imap_store->write_end(); diff --git a/split-by-thread.pl b/split-by-thread.pl new file mode 100755 index 000000000..383841304 --- /dev/null +++ b/split-by-thread.pl @@ -0,0 +1,125 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use DateTime; +use Cassandane::MessageStoreFactory; + +sub usage +{ + die "Usage: split-by-thread.pl [ -f format [maildir] | -u uri]"; +} + +my %params; +while (my $a = shift) +{ + if ($a eq '-f') + { + usage() if defined $params{uri}; + $params{type} = shift; + } + elsif ($a eq '-u') + { + usage() if defined $params{type}; + $params{uri} = shift; + } + elsif ($a eq '-v') + { + $params{verbose} = 1; + } + elsif ($a =~ m/^-/) + { + usage(); + } + else + { + usage() if defined $params{path}; + $params{path} = $a; + } +} + +sub extract_refs +{ + my ($msg) = @_; + my $str = $msg->get_header('references'); + return if !defined $str; + my @refs; + + for (;;) + { + my ($msgid, $rem) = ($str =~ m/[\s,]*(<[^\s<>]+@[^\s<>]+>)(.*)/); + last if !defined $msgid; + push(@refs, $msgid); + last if !defined $rem || !length $rem; + $str = $rem; + } + + return @refs; +} + +my $store = Cassandane::MessageStoreFactory->create(%params); + +my %threads_by_msgid; +my @threads; +my $next_thread_id = 1; +my $max_msgs = 500; + +sub thread_new +{ + my $t = + { + id => $next_thread_id++, + messages => [], + }; + push(@threads, $t); + return $t; +} + +sub thread_add_message +{ + my ($thread, $msg) = @_; + + push(@{$thread->{messages}}, $msg); + $threads_by_msgid{$msg->get_header('message-id')} = $thread; +} + +$store->read_begin(); +while (my $msg = $store->read_message()) +{ + my $msgid = $msg->get_header('message-id'); + die "duplicate msgid $msgid" + if defined $threads_by_msgid{$msgid}; + + my $thread; + foreach my $ref (extract_refs($msg)) + { + my $t = $threads_by_msgid{$ref}; + die "Thread clash!" + if (defined $t && + defined $thread && + $t->{id} != $thread->{id}); + $thread = $t; + } + + $thread = thread_new() + if !defined $thread; + thread_add_message($thread, $msg); + + last if (--$max_msgs == 0); +} +$store->read_end(); + +foreach my $t (@threads) +{ + next if scalar(@{$t->{messages}}) < 8; + + my $store = Cassandane::MessageStoreFactory->create( + type => 'mbox', + path => sprintf("x/thread%04u", $t->{id})); + $store->write_begin(); + foreach my $msg (@{$t->{messages}}) + { + $store->write_message($msg); + } + $store->write_end(); +} diff --git a/sprinkle.pl b/sprinkle.pl new file mode 100755 index 000000000..8e8338ab1 --- /dev/null +++ b/sprinkle.pl @@ -0,0 +1,106 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use DateTime; +use Cassandane::MessageStoreFactory; + +sub usage +{ + die "Usage: sprinkle.pl imapuri mbox ..."; +} + +my $base_folder; +my $num_remaining = 0; +my $num_written = 0; +my @words; +my $verbose = 1; + +# Extract some well-formatted short words from the dictionary file +sub read_words +{ + my $filename = "/usr/share/dict/words"; + my $i = 0; + my $stride = 200; + open DICT,'<',$filename + or die "Cannot open $filename for reading: $!"; + while () + { + chomp; + $_ = lc; + next unless m/^[a-z]+$/; + next if length $_ > 5 || length $_ < 2; + next if $i++ < $stride; + $i = 0; + push(@words, $_); + last if scalar @words == 200; + } + close DICT; +} + +sub choose_folder +{ + my @parts; + my $nparts = int(rand(7)); + + for (my $i = 0 ; $i < $nparts ; $i++) + { + push(@parts, $words[int(rand(scalar @words))]); + } + + my $folder = join('.', ($base_folder, @parts)); + printf STDERR "--> choosing folder %s\n", $folder if $verbose; + return $folder; +} + +sub sprinkle +{ + my ($path, $imap_store) = @_; + + my $mbox_store = Cassandane::MessageStoreFactory->create(( + type => 'mbox', + path => $path )) + or die "Cannot create MBOX message store"; + + $mbox_store->read_begin(); + while (my $msg = $mbox_store->read_message()) + { + if ($num_remaining == 0) + { + $imap_store->write_end() + if $num_written; + $imap_store->set_folder(choose_folder()); + $imap_store->write_begin(); + $num_remaining = 1 + int(rand(300)); + $num_written = 0; + printf STDERR "--> choosing %u messages\n", $num_remaining if $verbose; + } + $imap_store->write_message($msg); + $num_remaining--; + $num_written++; + } + $mbox_store->read_end(); +} + +read_words(); +my $imap_store = Cassandane::MessageStoreFactory->create(( + type => 'imap', + host => 'slott02', + port => 2144, + folder => 'inbox.sprinkle', + username => 'test@vmtom.com', + password => 'testpw', + verbose => ($verbose > 1 ? 1 : 0), + )) + or die "Cannot create IMAP message store"; +$base_folder = $imap_store->{folder}; + +while (my $a = shift) +{ + sprinkle($a, $imap_store); +} + +$imap_store->write_end() + if $num_written; +$imap_store->disconnect(); + diff --git a/tominstall.sh b/tominstall.sh index fe06b93bd..f242e7015 100755 --- a/tominstall.sh +++ b/tominstall.sh @@ -5,6 +5,9 @@ INSTALLABLES="\ genmail3.pl \ listmail.pl \ pop3showafter.pl \ + split-by-thread.pl \ + imap-append.pl \ + sprinkle.pl \ testrunner.pl \ Cassandane \ "