3 # Copyright (c) 2001 University of Cambridge.
4 # See the file NOTICE for conditions of use and distribution.
6 # Perl script to generate statistics from one or more Exim log files.
8 # Usage: eximstats [<options>] <log file> <log file> ...
10 # 1996-05-21: Ignore lines not starting with valid date/time, just in case
11 # these get into a log file.
12 # 1996-11-19: Add the -h option to control the size of the histogram,
13 # and optionally turn it off.
14 # Use some Perl 5 things; it should be everywhere by now.
15 # Add the Perl -w option and rewrite so no warnings are given.
16 # Add the -t option to control the length of the "top" listing.
17 # Add the -ne, -nt options to turn off errors and transport
19 # Add information about length of time on queue, and -q<list> to
20 # control the intervals and turn it off.
21 # Add count and percentage of delayed messages to the Received
23 # Show total number of errors.
24 # Add count and percentage of messages with errors to Received
26 # Add information about relaying and -nr to suppress it.
27 # 1997-02-03 Merged in some of the things Nigel Metheringham had done:
29 # Added received histogram as well as delivered
30 # Added local senders' league table
31 # Added local recipients' league table
32 # 1997-03-10 Fixed typo "destinationss"
33 # Allow for intermediate address between final and original
34 # when testing for relaying
35 # Give better message when no input
36 # 1997-04-24 Fixed bug in layout of error listing that was depending on
37 # text length (output line got repeated).
38 # 1997-05-06 Bug in option decoding when only one option.
39 # Overflow bug when handling very large volumes.
40 # 1997-10-28 Updated to handle revised log format that might show
41 # HELO name as well as host name before IP number
42 # 1998-01-26 Bugs in the function for calculating the number of seconds
43 # since 1970 from a log date
44 # 1998-02-02 Delivery to :blackhole: doesn't have a T= entry in the log
45 # line; cope with this, thereby avoiding undefined problems
46 # Very short log line gave substring error
47 # 1998-02-03 A routed delivery to a local transport may not have <> in the
48 # log line; terminate the address at white space, not <
49 # 1998-09-07 If first line of input was a => line, $thissize was undefined;
51 # 1998-12-21 Adding of $thissize from => line should have been adding $size.
52 # Oops. Should have looked more closely when fixing the previous
54 # 1999-11-12 Increased the field widths for printed integers; numbers are
55 # bigger than originally envisaged.
56 # 2001-03-21 Converted seconds() routine to use Time::Local, fixing a bug
57 # whereby seconds($timestamp) - id_seconds($id) gave an
59 # Added POD documentation.
60 # Moved usage instructions into help() subroutine.
61 # Added 'use strict' and declared all global variables.
62 # Added '-html' flag and resultant code.
63 # Added '-cache' flag and resultant code.
64 # Added add_volume() routine and converted all volume variables
65 # to use it, fixing the overflow problems for individual hosts
67 # Converted all volume output to GB/MB/KB as appropriate.
68 # Don't store local user stats if -nfl is specified.
69 # Modifications done by: Steve Campbell (<steve@computurn.com>)
70 # 2001-04-02 Added the -t_remote_users flag. Steve Campbell.
71 # 2001-10-15 Added the -domain flag. Steve Campbell.
72 # 2001-10-16 Accept files on STDIN or on the command line. Steve Campbell.
73 # 2001-10-21 Removed -domain flag and added -bydomain, -byhost, and -byemail.
74 # We now generate our main parsing subroutine as an eval statement
75 # which improves performance dramatically when not all the results
76 # are required. We also cache the last timestamp to time convertion.
78 # NOTE: 'Top 50 destinations by (message count|volume)' lines are
79 # now 'Top N (host|email|domain) destinations by (message count|volume)'
80 # where N is the topcount. Steve Campbell.
82 # 2001-10-30 V1.16 Joachim Wieland.
83 # Fixed minor bugs in add_volume() when taking over this version
84 # for use in Exim 4: -w gave uninitialized value warnings in
85 # two situations: for the first addition to a counter, and if
86 # there were never any gigabytes, thereby leaving the $gigs
88 # Initialized $last_timestamp to stop a -w uninitialized warning.
89 # Minor layout tweak for grand totals (nitpicking).
90 # Put the IP addresses for relaying stats in [] and separated by
91 # a space from the domain name.
92 # Removed the IPv4-specific address test when picking out addresses
93 # for relaying. Anything inside [] is OK.
95 # 2002-07-02 Philip Hazel
96 # Fixed "uninitialized variable" message that occurred for relay
97 # messages that arrived from H=[1.2.3.4] hosts (no name shown).
98 # This bug didn't affect the output.
100 # 2002-04-15 V1.17 Joachim Wieland.
101 # Added -charts, -chartdir. -chartrel options which use
102 # GD::Graph modules to create graphical charts of the statistics.
104 # 2002-04-15 V1.18 Steve Campbell.
105 # Added a check for $domain to to stop a -w uninitialized warning.
106 # Added -byemaildomain option.
107 # Only print HTML header links to included tables!
109 # 2002-08-02 V1.19 Steve Campbell.
110 # Changed the debug mode to dump the parser onto STDERR rather
111 # than STDOUT. Documented the -d flag into the help().
112 # Rejoined the divergent 2002-04-15 and 2002-07-02 releases.
114 # 2002-08-21 V1.20 Steve Campbell.
115 # Added the '-merge' option to allow merging of previous reports.
116 # Fixed a missing semicolon when doing -bydomain.
117 # Make volume charts plot the data gigs and bytes rather than just bytes.
118 # Only process log lines with $flag =~ /<=|=>|->|==|\*\*|Co/
119 # Converted Emaildomain to Edomain - the column header was too wide!
120 # This changes the text output slightly. You can revert to the old
121 # column widths by changing $COLUMN_WIDTHS to 7;
123 # 2002-09-04 V1.21 Andreas J Mueller
124 # Local deliveries domain now defaults to 'localdomain'.
125 # Don't match F=<From> when looking for the user.
127 # 2002-09-05 V1.22 Steve Campbell
128 # Fixed a perl 5.005 incompatibility problem ('our' variables).
130 # 2002-09-11 V1.23 Steve Campbell
131 # Stopped -charts option from throwing errors on null data.
132 # Don't print out 'Errors encountered' unless there are any.
134 # 2002-10-21 V1.23a Philip Hazel - patch from Tony Finch put in until
135 # Steve's eximstats catches up.
136 # Handle log files that include the timezone after the timestamp.
137 # Switch to assuming that log timestamps are in local time, with
138 # an option for UTC timestamps, as in Exim itself.
140 # 2003-02-05 V1.24 Steve Campbell
141 # Added in Sergey Sholokh's code to convert '<' and '>' characters
142 # in HTML output. Also added code to convert them back with -merge.
143 # Fixed timestamp offsets to convert to seconds rather than minutes.
144 # Updated -merge to work with output files using timezones.
145 # Added cacheing to speed up the calculation of timezone offsets.
147 # 2003-02-07 V1.25 Steve Campbell
148 # Optimised the usage of mktime() in the seconds subroutine.
149 # Removed the now redundant '-cache' option.
150 # html2txt() now explicitly matches HTML tags.
151 # Implemented a new sorting algorithm - the top_n_sort() routine.
152 # Added Danny Carroll's '-nvr' flag and code.
154 # 2003-03-13 V1.26 Steve Campbell
155 # Implemented HTML compliance changes recommended by Bernard Massot.
156 # Bug fix to allow top_n_sort() to handle null keys.
157 # Convert all domains and edomains to lowercase.
158 # Remove preceding dots from domains.
160 # 2003-03-13 V1.27 Steve Campbell
161 # Replaced border attributes with 'border=1', as recommended by
164 # 2003-06-03 V1.28 John Newman
165 # Added in the ability to skip over the parsing and evaulation of
166 # specific transports as passed to eximstats via the new "-nt/.../"
167 # command line argument. This new switch allows the viewing of
168 # not more accurate statistics but more applicable statistics when
169 # special transports are in use (ie; SpamAssassin). We need to be
170 # able to ignore transports such as this otherwise the resulting
171 # local deliveries are significantly skewed (doubled)...
173 # 2003-11-06 V1.29 Steve Campbell
174 # Added the '-pattern "Description" "/pattern/"' option.
176 # 2004-02-17 V1.30 Steve Campbell
177 # Added warnings if required GD::Graph modules are not available or
178 # insufficient -chart* options are specified.
180 # 2004-02-20 V1.31 Andrea Balzi
181 # Only show the Local Sender/Destination links if the tables exist.
183 # 2004-07-05 V1.32 Steve Campbell
184 # Fix '-merge -h0' divide by zero error.
186 # 2004-07-15 V1.33 Steve Campbell
187 # Documentation update - I've converted the subroutine
188 # documentation from POD to comments.
190 # 2004-12-10 V1.34 Steve Campbell
191 # Eximstats can now parse syslog lines as well as mainlog lines.
193 # 2004-12-20 V1.35 Wouter Verhelst
194 # Pie charts by volume were actually generated by count. Fixed.
196 # 2005-02-07 V1.36 Gregor Herrmann / Steve Campbell
197 # Added average sizes to HTML Top tables.
199 # 2005-04-26 V1.37 Frank Heydlauf
200 # Added -xls and the ability to specify output files.
202 # 2005-04-29 V1.38 Steve Campbell
203 # Use FileHandles for outputing results.
204 # Allow any combination of xls, txt, and html output.
205 # Fixed display of large numbers with -nvr option
206 # Fixed merging of reports with empty tables.
208 # 2005-05-27 V1.39 Steve Campbell
209 # Added the -include_original_destination flag
210 # Removed tabs and trailing whitespace.
212 # 2005-06-03 V1.40 Steve Campbell
213 # Whilst parsing the mainlog(s), store information about
214 # the messages in a hash of arrays rather than using
215 # individual hashes. This is a bit cleaner and results in
216 # dramatic memory savings, albeit at a slight CPU cost.
218 # 2005-06-15 V1.41 Steve Campbell
219 # Added the -show_rt<list> flag.
220 # Added the -show_dt<list> flag.
222 # 2005-06-24 V1.42 Steve Campbell
223 # Added Histograms for user specified patterns.
225 # 2005-06-30 V1.43 Steve Campbell
226 # Bug fix for V1.42 with -h0 specified. Spotted by Chris Lear.
228 # 2005-07-26 V1.44 Steve Campbell
229 # Use a glob alias rather than an array ref in the generated
230 # parser. This improves both readability and performance.
232 # 2005-09-30 V1.45 Marco Gaiarin / Steve Campbell
233 # Collect SpamAssassin and rejection statistics.
234 # Don't display local sender or destination tables unless
235 # there is data to show.
236 # Added average volumes into the top table text output.
238 # 2006-02-07 V1.46 Steve Campbell
239 # Collect data on the number of addresses (recipients)
240 # as well as the number of messages.
242 # 2006-05-05 V1.47 Steve Campbell
243 # Added 'Message too big' to the list of mail rejection
244 # reasons (thanks to Marco Gaiarin).
246 # 2006-06-05 V1.48 Steve Campbell
247 # Mainlog lines which have GMT offsets and are too short to
248 # have a flag are now skipped.
250 # 2006-11-10 V1.49 Alain Williams
251 # Added the -emptyok flag.
253 # 2006-11-16 V1.50 Steve Campbell
254 # Fixes for obtaining the IP address from reject messages.
256 # 2006-11-27 V1.51 Steve Campbell
257 # Another update for obtaining the IP address from reject messages.
259 # 2006-11-27 V1.52 Steve Campbell
260 # Tally any reject message containing SpamAssassin.
262 # 2007-01-31 V1.53 Philip Hazel
263 # Allow for [pid] after date in log lines
265 # 2007-02-14 V1.54 Daniel Tiefnig
266 # Improved the '($parent) =' pattern match.
268 # 2007-03-19 V1.55 Steve Campbell
269 # Differentiate between permanent and temporary rejects.
271 # 2007-03-29 V1.56 Jez Hancock
272 # Fixed some broken HTML links and added missing column headers.
274 # 2007-03-30 V1.57 Steve Campbell
275 # Fixed Grand Total Summary Domains, Edomains, and Email columns
276 # for Rejects, Temp Rejects, Ham, and Spam rows.
278 # 2007-04-11 V1.58 Steve Campbell
279 # Fix to get <> and blackhole to show in edomain tables.
281 # 2007-09-20 V1.59 Steve Campbell
282 # Added the -bylocaldomain option
284 # 2007-09-20 V1.60 Heiko Schlittermann
285 # Fix for misinterpreted log lines
289 # For documentation on the logfile format, see
290 # http://www.exim.org/exim-html-4.50/doc/html/spec_48.html#IX2793
294 eximstats - generates statistics from Exim mainlog or syslog files.
298 eximstats [Output] [Options] mainlog1 mainlog2 ...
299 eximstats -merge [Options] report.1.txt report.2.txt ... > weekly_report.txt
307 Output the results in plain text to STDOUT.
309 =item B<-txt>=I<filename>
311 Output the results in plain text. Filename '-' for STDOUT is accepted.
315 Output the results in HTML to STDOUT.
317 =item B<-html>=I<filename>
319 Output the results in HTML. Filename '-' for STDOUT is accepted.
323 Output the results in Excel compatible Format to STDOUT.
324 Requires the Spreadsheet::WriteExcel CPAN module.
326 =item B<-xls>=I<filename>
328 Output the results in Excel compatible format. Filename '-' for STDOUT is accepted.
339 histogram divisions per hour. The default is 1, and
340 0 suppresses histograms. Valid values are:
342 0, 1, 2, 3, 5, 10, 15, 20, 30 or 60.
346 Don't display error information.
350 Don't display relaying information.
352 =item B<-nr>I</pattern/>
354 Don't display relaying information that matches.
358 Don't display transport information.
360 =item B<-nt>I</pattern/>
362 Don't display transport information that matches
366 List of times for queuing information single 0 item suppresses.
370 Display top <number> sources/destinations
371 default is 50, 0 suppresses top listing.
375 Omit local sources/destinations in top listing.
377 =item B<-t_remote_users>
379 Include remote users in the top source/destination listings.
381 =item B<-include_original_destination>
383 Include the original destination email addresses rather than just
384 using the final ones.
385 Useful for finding out which of your mailing lists are receiving mail.
387 =item B<-show_dt>I<list>
389 Show the delivery times (B<DT>)for all the messages.
391 Exim must have been configured to use the +deliver_time logging option
392 for this option to work.
394 I<list> is an optional list of times. Eg -show_dt1,2,4,8 will show
395 the number of messages with delivery times under 1 second, 2 seconds, 4 seconds,
396 8 seconds, and over 8 seconds.
398 =item B<-show_rt>I<list>
400 Show the receipt times for all the messages. The receipt time is
401 defined as the Completed hh:mm:ss - queue_time_overall - the Receipt hh:mm:ss.
402 These figures will be skewed by pipelined messages so might not be that useful.
404 Exim must have been configured to use the +queue_time_overall logging option
405 for this option to work.
407 I<list> is an optional list of times. Eg -show_rt1,2,4,8 will show
408 the number of messages with receipt times under 1 second, 2 seconds, 4 seconds,
409 8 seconds, and over 8 seconds.
413 Show results by sending host. This may be combined with
414 B<-bydomain> and/or B<-byemail> and/or B<-byedomain>. If none of these options
415 are specified, then B<-byhost> is assumed as a default.
419 Show results by sending domain.
420 May be combined with B<-byhost> and/or B<-byemail> and/or B<-byedomain>.
424 Show results by sender's email address.
425 May be combined with B<-byhost> and/or B<-bydomain> and/or B<-byedomain>.
427 =item B<-byemaildomain> or B<-byedomain>
429 Show results by sender's email domain.
430 May be combined with B<-byhost> and/or B<-bydomain> and/or B<-byemail>.
432 =item B<-pattern> I<Description> I</Pattern/>
434 Look for the specified pattern and count the number of lines in which it appears.
435 This option can be specified multiple times. Eg:
437 -pattern 'Refused connections' '/refused connection/'
442 This option allows eximstats to merge old eximstat reports together. Eg:
444 eximstats mainlog.sun > report.sun.txt
445 eximstats mainlog.mon > report.mon.txt
446 eximstats mainlog.tue > report.tue.txt
447 eximstats mainlog.wed > report.web.txt
448 eximstats mainlog.thu > report.thu.txt
449 eximstats mainlog.fri > report.fri.txt
450 eximstats mainlog.sat > report.sat.txt
451 eximstats -merge report.*.txt > weekly_report.txt
452 eximstats -merge -html report.*.txt > weekly_report.html
458 You can merge text or html reports and output the results as text or html.
462 You can use all the normal eximstat output options, but only data
463 included in the original reports can be shown!
467 When merging reports, some loss of accuracy may occur in the top I<n> lists.
468 This will be towards the ends of the lists.
472 The order of items in the top I<n> lists may vary when the data volumes
473 round to the same value.
479 Create graphical charts to be displayed in HTML output.
480 Only valid in combination with I<-html>.
482 This requires the following modules which can be obtained
483 from http://www.cpan.org/modules/01modules.index.html
495 To install these, download and unpack them, then use the normal perl installation procedure:
502 =item B<-chartdir>I <dir>
504 Create the charts in the directory <dir>
506 =item B<-chartrel>I <dir>
508 Specify the relative directory for the "img src=" tags from where to include
513 Specify that it's OK to not find any valid log lines. Without this
514 we will output an error message if we don't find any.
518 Debug flag. This outputs the eval()'d parser onto STDOUT which makes it
519 easier to trap errors in the eval section. Remember to add 1 to the line numbers to allow for the
526 Eximstats parses exim mainlog and syslog files to output a statistical
527 analysis of the messages processed. By default, a text
528 analysis is generated, but you can request other output formats
529 using flags. See the help (B<-help>) to learn
530 about how to create charts from the tables.
534 There is a web site at http://www.exim.org - this contains details of the
535 mailing list exim-users@exim.org.
539 This program does not perfectly handle messages whose received
540 and delivered log lines are in different files, which can happen
541 when you have multiple mail servers and a message cannot be
542 immediately delivered. Fixing this could be tricky...
544 Merging of xls files is not (yet) possible. Be free to implement :)
552 # use Time::Local; # PH/FANF
555 use vars qw($HAVE_GD_Graph_pie $HAVE_GD_Graph_linespoints $HAVE_Spreadsheet_WriteExcel);
556 eval { require GD::Graph::pie; };
557 $HAVE_GD_Graph_pie = $@ ? 0 : 1;
558 eval { require GD::Graph::linespoints; };
559 $HAVE_GD_Graph_linespoints = $@ ? 0 : 1;
560 eval { require Spreadsheet::WriteExcel; };
561 $HAVE_Spreadsheet_WriteExcel = $@ ? 0 : 1;
564 ##################################################
566 ##################################################
567 # 'use vars' instead of 'our' as perl5.005 is still in use out there!
568 use vars qw(@tab62 @days_per_month $gig);
569 use vars qw($VERSION);
570 use vars qw($COLUMN_WIDTHS);
571 use vars qw($WEEK $DAY $HOUR $MINUTE);
575 (0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, # 0-9
576 0,10,11,12,13,14,15,16,17,18,19,20, # A-K
577 21,22,23,24,25,26,27,28,29,30,31,32, # L-W
578 33,34,35, 0, 0, 0, 0, 0, # X-Z
579 0,36,37,38,39,40,41,42,43,44,45,46, # a-k
580 47,48,49,50,51,52,53,54,55,56,57,58, # l-w
583 @days_per_month = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334);
584 $gig = 1024 * 1024 * 1024;
587 # How much space do we allow for the Hosts/Domains/Emails/Edomains column headers?
591 $HOUR = 60 * $MINUTE;
595 # Declare global variables.
596 use vars qw($total_received_data $total_received_data_gigs $total_received_count);
597 use vars qw($total_delivered_data $total_delivered_data_gigs $total_delivered_messages $total_delivered_addresses);
598 use vars qw(%timestamp2time); #Hash of timestamp => time.
599 use vars qw($last_timestamp $last_time); #The last time convertion done.
600 use vars qw($last_date $date_seconds); #The last date convertion done.
601 use vars qw($last_offset $offset_seconds); #The last time offset convertion done.
602 use vars qw($localtime_offset);
603 use vars qw($i); #General loop counter.
604 use vars qw($debug); #Debug mode?
605 use vars qw($ntopchart); #How many entries should make it into the chart?
606 use vars qw($gddirectory); #Where to put files from GD::Graph
608 # SpamAssassin variables
609 use vars qw($spam_score $spam_score_gigs);
610 use vars qw($ham_score $ham_score_gigs);
611 use vars qw(%ham_count_by_ip %spam_count_by_ip);
612 use vars qw(%rejected_count_by_ip %rejected_count_by_reason);
613 use vars qw(%temporarily_rejected_count_by_ip %temporarily_rejected_count_by_reason);
615 #For use in Speadsheed::WriteExcel
616 use vars qw($workbook $ws_global $ws_relayed $ws_errors);
617 use vars qw($row $col $row_hist $col_hist);
618 use vars qw($run_hist);
619 use vars qw($f_default $f_header1 $f_header2 $f_header2_m $f_headertab $f_percent); #Format Header
622 use vars qw($txt_fh $htm_fh $xls_fh);
626 # The following are parameters whose values are
627 # set by command line switches:
628 use vars qw($show_errors $show_relay $show_transport $transport_pattern);
629 use vars qw($topcount $local_league_table $include_remote_users $do_local_domain);
630 use vars qw($hist_opt $hist_interval $hist_number $volume_rounding $emptyOK);
631 use vars qw($relay_pattern @queue_times @user_patterns @user_descriptions);
632 use vars qw(@rcpt_times @delivery_times);
633 use vars qw($include_original_destination);
634 use vars qw($txt_fh $htm_fh $xls_fh);
636 use vars qw(%do_sender); #Do sender by Host, Domain, Email, and/or Edomain tables.
637 use vars qw($charts $chartrel $chartdir $charts_option_specified);
638 use vars qw($merge_reports); #Merge old reports ?
640 # The following are modified in the parse() routine, and
641 # referred to in the print_*() routines.
642 use vars qw($delayed_count $relayed_unshown $begin $end);
643 use vars qw(%messages @message);
644 use vars qw(%received_count %received_data %received_data_gigs);
645 use vars qw(%delivered_messages %delivered_data %delivered_data_gigs %delivered_addresses);
646 use vars qw(%received_count_user %received_data_user %received_data_gigs_user);
647 use vars qw(%delivered_messages_user %delivered_addresses_user %delivered_data_user %delivered_data_gigs_user);
648 use vars qw(%delivered_messages_local_domain %delivered_addresses_local_domain %delivered_data_local_domain %delivered_data_gigs_local_domain);
649 use vars qw(%transported_count %transported_data %transported_data_gigs);
650 use vars qw(%relayed %errors_count $message_errors);
651 use vars qw(@qt_all_bin @qt_remote_bin);
652 use vars qw($qt_all_overflow $qt_remote_overflow);
653 use vars qw(@dt_all_bin @dt_remote_bin %rcpt_times_bin);
654 use vars qw($dt_all_overflow $dt_remote_overflow %rcpt_times_overflow);
655 use vars qw(@received_interval_count @delivered_interval_count);
656 use vars qw(@user_pattern_totals @user_pattern_interval_count);
658 use vars qw(%report_totals);
661 use vars qw($SIZE $FROM_HOST $FROM_ADDRESS $ARRIVAL_TIME $REMOTE_DELIVERED $PROTOCOL);
662 use vars qw($DELAYED $HAD_ERROR);
667 $REMOTE_DELIVERED = 4;
674 ##################################################
676 ##################################################
678 #######################################################################
679 # get_filehandle($file,\%output_files);
680 # Return a filehandle writing to $file.
682 # If %output_files is defined, check that $output_files{$file}
683 # doesn't exist and die if it does, or set it if it doesn't.
684 #######################################################################
686 my($file,$output_files_href) = @_;
688 $file = '-' if ($file eq '');
690 if (defined $output_files_href) {
691 die "You can only output to '$file' once! Use -h for help.\n" if exists $output_files_href->{$file};
692 $output_files_href->{$file} = 1;
700 unlink $file or die "Failed to rm $file: $!";
703 my $fh = new IO::File $file, O_WRONLY|O_CREAT|O_EXCL;
704 die "new IO::File $file failed: $!" unless (defined $fh);
709 #######################################################################
712 # $rounded_volume = volume_rounded($bytes,$gigabytes);
714 # Given a data size in bytes, round it to KB, MB, or GB
717 # Eg 12000 => 12KB, 15000000 => 14GB, etc.
719 # Note: I've experimented with Math::BigInt and it results in a 33%
720 # performance degredation as opposed to storing numbers split into
721 # bytes and gigabytes.
722 #######################################################################
734 if ($volume_rounding) {
738 $rounded = sprintf("%6d", $x);
740 elsif ($x < 10000000) {
741 $rounded = sprintf("%4dKB", ($x + 512)/1024);
744 $rounded = sprintf("%4dMB", ($x + 512*1024)/(1024*1024));
747 # Values between 1GB and 10GB are printed in MB
749 $rounded = sprintf("%4dMB", ($g * 1024) + ($x + 512*1024)/(1024*1024));
752 # Handle values over 10GB
753 $rounded = sprintf("%4dGB", $g + ($x + $gig/2)/$gig);
757 # We don't want any rounding to be done.
758 # and we don't need broken formated output which on one hand avoids numbers from
759 # being interpreted as string by Spreadsheed Calculators, on the other hand
760 # breaks if more than 4 digits! -> flexible length instead of fixed length
761 # Format the return value at the output routine! -fh
762 #$rounded = sprintf("%d", ($g * $gig) + $x);
764 $rounded = sprintf("%.0f", ($g * $gig) + $x);
771 #######################################################################
774 # un_round($rounded_volume,\$bytes,\$gigabytes);
776 # Given a volume in KB, MB or GB, as generated by volume_rounded(),
777 # do the reverse transformation and convert it back into Bytes and Gigabytes.
778 # These are added to the $bytes and $gigabytes parameters.
780 # Given a data size in bytes, round it to KB, MB, or GB
783 # EG: 500 => (500,0), 14GB => (0,14), etc.
784 #######################################################################
786 my($rounded,$bytes_sref,$gigabytes_sref) = @_;
788 if ($rounded =~ /(\d+)GB/) {
789 $$gigabytes_sref += $1;
791 elsif ($rounded =~ /(\d+)MB/) {
792 $$gigabytes_sref += $1 / 1024;
793 $$bytes_sref += (($1 % 1024 ) * 1024 * 1024);
795 elsif ($rounded =~ /(\d+)KB/) {
796 $$gigabytes_sref += $1 / (1024 * 1024);
797 $$bytes_sref += ($1 % (1024 * 1024) * 1024);
799 elsif ($rounded =~ /(\d+)/) {
800 # We need to turn off integer in case we are merging an -nvr report.
802 $$gigabytes_sref += int($1 / $gig);
803 $$bytes_sref += $1 % $gig;
806 #Now reduce the bytes down to less than 1GB.
807 add_volume($bytes_sref,$gigabytes_sref,0) if ($$bytes_sref > $gig);
811 #######################################################################
814 # add_volume(\$bytes,\$gigs,$size);
816 # Add $size to $bytes/$gigs where this is a number split into
817 # bytes ($bytes) and gigabytes ($gigs). This is significantly
818 # faster than using Math::BigInt.
819 #######################################################################
821 my($bytes_ref,$gigs_ref,$size) = @_;
822 $$bytes_ref = 0 if ! defined $$bytes_ref;
823 $$gigs_ref = 0 if ! defined $$gigs_ref;
824 $$bytes_ref += $size;
825 while ($$bytes_ref > $gig) {
832 #######################################################################
835 # $formatted_time = format_time($seconds);
837 # Given a time in seconds, break it down into
838 # weeks, days, hours, minutes, and seconds.
840 # Eg 12005 => 3h20m5s
841 #######################################################################
853 $p .= "$w"."w" if $w > 0;
854 $p .= "$d"."d" if $d > 0;
855 $p .= "$h"."h" if $h > 0;
856 $p .= "$m"."m" if $m > 0;
857 $p .= "$s"."s" if $s > 0 || $p eq "";
862 #######################################################################
865 # $seconds = unformat_time($formatted_time);
867 # Given a time in weeks, days, hours, minutes, or seconds, convert it to seconds.
869 # Eg 3h20m5s => 12005
870 #######################################################################
872 my($formated_time) = pop @_;
875 while ($formated_time =~ s/^(\d+)([wdhms]?)//) {
876 $time += $1 if ($2 eq '' || $2 eq 's');
877 $time += $1 * 60 if ($2 eq 'm');
878 $time += $1 * 60 * 60 if ($2 eq 'h');
879 $time += $1 * 60 * 60 * 24 if ($2 eq 'd');
880 $time += $1 * 60 * 60 * 24 * 7 if ($2 eq 'w');
886 #######################################################################
889 # $time = seconds($timestamp);
891 # Given a time-of-day timestamp, convert it into a time() value using
892 # POSIX::mktime. We expect the timestamp to be of the form
893 # "$year-$mon-$day $hour:$min:$sec", with month going from 1 to 12,
894 # and the year to be absolute (we do the necessary conversions). The
895 # timestamp may be followed with an offset from UTC like "+$hh$mm"; if the
896 # offset is not present, and we have not been told that the log is in UTC
897 # (with the -utc option), then we adjust the time by the current local
898 # time offset so that it can be compared with the time recorded in message
901 # To improve performance, we only use mktime on the date ($year-$mon-$day),
902 # and only calculate it if the date is different to the previous time we
903 # came here. We then add on seconds for the '$hour:$min:$sec'.
905 # We also store the results of the last conversion done, and only
906 # recalculate if the date is different.
908 # We used to have the '-cache' flag which would store the results of the
909 # mktime() call. However, the current way of just using mktime() on the
910 # date obsoletes this.
911 #######################################################################
915 # Is the timestamp the same as the last one?
916 return $last_time if ($last_timestamp eq $timestamp);
918 return 0 unless ($timestamp =~ /^((\d{4})\-(\d\d)-(\d\d))\s(\d\d):(\d\d):(\d\d)( ([+-])(\d\d)(\d\d))?/o);
920 unless ($last_date eq $1) {
922 my(@timestamp) = (0,0,0,$4,$3,$2);
923 $timestamp[5] -= 1900;
925 $date_seconds = mktime(@timestamp);
927 my $time = $date_seconds + ($5 * 3600) + ($6 * 60) + $7;
929 # SC. Use cacheing. Also note we want seconds not minutes.
930 #my($this_offset) = ($10 * 60 + $11) * ($9 . "1") if defined $8;
931 if (defined $8 && ($8 ne $last_offset)) {
933 $offset_seconds = ($10 * 60 + $11) * 60;
934 $offset_seconds = -$offset_seconds if ($9 eq '-');
939 #$time -= $this_offset;
940 $time -= $offset_seconds;
941 } elsif (defined $localtime_offset) {
942 $time -= $localtime_offset;
945 # Store the last timestamp received.
946 $last_timestamp = $timestamp;
953 #######################################################################
956 # $time = id_seconds($message_id);
958 # Given a message ID, convert it into a time() value.
959 #######################################################################
961 my($sub_id) = substr((pop @_), 0, 6);
963 my(@c) = split(//, $sub_id);
964 while($#c >= 0) { $s = $s * 62 + $tab62[ord(shift @c) - ord('0')] }
968 #######################################################################
971 # $seconds = wdhms_seconds($string);
973 # Convert a string in a week/day/hour/minute/second format (eg 4h10s)
975 #######################################################################
977 if ($_[0] =~ /^(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?/) {
978 return((($1||0) * $WEEK) + (($2||0) * $DAY) + (($3||0) * $HOUR) + (($4||0) * $MINUTE) + ($5||0));
983 #######################################################################
986 # $queued = queue_time($completed_tod, $arrival_time, $id);
988 # Given the completed time of day and either the arrival time
989 # (preferred), or the message ID, calculate how long the message has
992 #######################################################################
994 my($completed_tod, $arrival_time, $id) = @_;
996 # Note: id_seconds() benchmarks as 42% slower than seconds()
997 # and computing the time accounts for a significant portion of
999 if (defined $arrival_time) {
1000 return(seconds($completed_tod) - seconds($arrival_time));
1003 return(seconds($completed_tod) - id_seconds($id));
1008 #######################################################################
1009 # calculate_localtime_offset();
1011 # $localtime_offset = calculate_localtime_offset();
1013 # Calculate the the localtime offset from gmtime in seconds.
1015 # $localtime = time() + $localtime_offset.
1017 # These are the same semantics as ISO 8601 and RFC 2822 timezone offsets.
1018 # (West is negative, East is positive.)
1019 #######################################################################
1021 # $localtime = gmtime() + $localtime_offset. OLD COMMENT
1022 # This subroutine commented out as it's not currently in use.
1024 #sub calculate_localtime_offset {
1025 # # Pick an arbitrary date, convert it to localtime & gmtime, and return the difference.
1026 # my (@sample_date) = (0,0,0,5,5,100);
1027 # my $localtime = timelocal(@sample_date);
1028 # my $gmtime = timegm(@sample_date);
1029 # my $offset = $localtime - $gmtime;
1033 sub calculate_localtime_offset {
1034 # Assume that the offset at the moment is valid across the whole
1035 # period covered by the logs that we're analysing. This may not
1036 # be true around the time the clocks change in spring or autumn.
1038 # mktime works on local time and gmtime works in UTC
1039 my $local = mktime(gmtime($utc));
1040 return $local - $utc;
1045 #######################################################################
1046 # print_duration_table();
1048 # print_duration_table($title, $message_type, \@times, \@values, $overflow);
1050 # Print a table showing how long a particular step took for
1051 # the messages. The parameters are:
1052 # $title Eg "Time spent on the queue"
1053 # $message_type Eg "Remote"
1054 # \@times The maximum time a message took for it to increment
1055 # the corresponding @values counter.
1056 # \@values An array of message counters.
1057 # $overflow The number of messages which exceeded the maximum
1059 #######################################################################
1060 sub print_duration_table {
1062 my($title, $message_type, $times_aref, $values_aref, $overflow) = @_;
1063 my(@chartdatanames);
1066 my $printed_one = 0;
1067 my $cumulative_percent = 0;
1069 my $queue_total = $overflow;
1070 map {$queue_total += $_} @$values_aref;
1072 my $temp = "$title: $message_type";
1075 my $txt_format = "%5s %4s %6d %5.1f%% %5.1f%%\n";
1076 my $htm_format = "<tr><td align=\"right\">%s %s</td><td align=\"right\">%d</td><td align=\"right\">%5.1f%%</td><td align=\"right\">%5.1f%%</td>\n";
1079 printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
1081 print $htm_fh "<hr><a name=\"$title $message_type\"></a><h2>$temp</h2>\n";
1082 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
1083 print $htm_fh "<tr><th>Time</th><th>Messages</th><th>Percentage</th><th>Cumulative Percentage</th>\n";
1086 $ws_global->write($row++, $col, "$title: ".$message_type, $f_header2);
1087 my @content=("Time", "Messages", "Percentage", "Cumulative Percentage");
1088 &set_worksheet_line($ws_global, $row++, 1, \@content, $f_headertab);
1092 for ($i = 0; $i <= $#$times_aref; ++$i) {
1093 if ($$values_aref[$i] > 0)
1095 my $percent = ($values_aref->[$i] * 100)/$queue_total;
1096 $cumulative_percent += $percent;
1098 my @content=($printed_one? " " : "Under",
1099 format_time($times_aref->[$i]),
1100 $values_aref->[$i], $percent, $cumulative_percent);
1103 printf $htm_fh ($htm_format, @content);
1104 if (!defined($values_aref->[$i])) {
1105 print $htm_fh "Not defined";
1109 printf $txt_fh ($txt_format, @content);
1110 if (!defined($times_aref->[$i])) {
1111 print $txt_fh "Not defined";
1117 &set_worksheet_line($ws_global, $row, 0, [@content[0,1,2]], $f_default);
1118 &set_worksheet_line($ws_global, $row++, 3, [$content[3]/100,$content[4]/100], $f_percent);
1120 if (!defined($times_aref->[$i])) {
1122 $ws_global->write($row++, $col, "Not defined" );
1126 push(@chartdatanames,
1127 ($printed_one? "" : "Under") . format_time($times_aref->[$i]));
1128 push(@chartdatavals, $$values_aref[$i]);
1133 if ($overflow && $overflow > 0) {
1134 my $percent = ($overflow * 100)/$queue_total;
1135 $cumulative_percent += $percent;
1137 my @content = ("Over ", format_time($times_aref->[-1]),
1138 $overflow, $percent, $cumulative_percent);
1140 printf $txt_fh ($txt_format, @content) if $txt_fh;
1141 printf $htm_fh ($htm_format, @content) if $htm_fh;
1144 &set_worksheet_line($ws_global, $row, 0, [@content[0,1,2]], $f_default);
1145 &set_worksheet_line($ws_global, $row++, 3, [$content[3]/100,$content[4]/100], $f_percent);
1150 push(@chartdatanames, "Over " . format_time($times_aref->[-1]));
1151 push(@chartdatavals, $overflow);
1153 #printf("Unknown %6d\n", $queue_unknown) if $queue_unknown > 0;
1155 print $htm_fh "</table></td><td>";
1157 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0)) {
1162 my $graph = GD::Graph::pie->new(200, 200);
1163 my $pngname = "$title-$message_type.png";
1164 $pngname =~ s/[^\w\-\.]/_/;
1166 my $graph_title = "$title ($message_type)";
1167 $graph->set(title => $graph_title) if (length($graph_title) < 21);
1169 my $gd = $graph->plot(\@data) or warn($graph->error);
1171 open(IMG, ">$chartdir/$pngname") or die "Could not write $chartdir/$pngname: $!\n";
1175 print $htm_fh "<img src=\"$chartrel/$pngname\">";
1178 print $htm_fh "</td></tr></table>\n";
1185 print $txt_fh "\n" if $txt_fh;
1186 print $htm_fh "\n" if $htm_fh;
1191 #######################################################################
1192 # print_histogram();
1194 # print_histogram('Deliveries|Messages received|$pattern', $unit, @interval_count);
1196 # Print a histogram of the messages delivered/received per time slot
1197 # (hour by default).
1198 #######################################################################
1199 sub print_histogram {
1200 my($text, $unit, @interval_count) = @_;
1201 my(@chartdatanames);
1205 # save first row of print_histogram for xls output
1213 for ($i = 0; $i < $hist_number; $i++)
1214 { $maxd = $interval_count[$i] if $interval_count[$i] > $maxd; }
1216 my $scale = int(($maxd + 25)/50);
1217 $scale = 1 if $scale == 0;
1220 if ($unit !~ s/y$/ies/) {
1225 # make and output title
1226 my $title = sprintf("$text per %s",
1227 ($hist_interval == 60)? "hour" :
1228 ($hist_interval == 1)? "minute" : "$hist_interval minutes");
1230 my $txt_htm_title = $title . " (each dot is $scale $unit)";
1232 printf $txt_fh ("%s\n%s\n\n", $txt_htm_title, "-" x length($txt_htm_title)) if $txt_fh;
1235 print $htm_fh "<hr><a name=\"$text\"></a><h2>$txt_htm_title</h2>\n";
1236 print $htm_fh "<table border=0 width=\"100%\">\n";
1237 print $htm_fh "<tr><td><pre>\n";
1241 $title =~ s/Messages/Msg/ ;
1243 $ws_global->write($row++, $col_hist+1, $title, $f_headertab);
1249 for ($i = 0; $i < $hist_number; $i++) {
1250 my $c = $interval_count[$i];
1252 # If the interval is an hour (the maximum) print the starting and
1253 # ending hours as a label. Otherwise print the starting hour and
1254 # minutes, which take up the same space.
1257 if ($hist_opt == 1) {
1258 $temp = sprintf("%02d-%02d", $hour, $hour + 1);
1260 print $txt_fh $temp if $txt_fh;
1261 print $htm_fh $temp if $htm_fh;
1266 $ws_global->write($row, 0, [$temp], $f_default);
1270 push(@chartdatanames, $temp);
1275 { $temp = sprintf("%02d:%02d", $hour, $minutes) }
1277 { $temp = sprintf(" :%02d", $minutes) }
1279 print $txt_fh $temp if $txt_fh;
1280 print $htm_fh $temp if $htm_fh;
1281 if (($xls_fh) and ($run_hist==0)) {
1283 $temp = sprintf("%02d:%02d", $hour, $minutes);
1284 $ws_global->write($row, 0, [$temp], $f_default);
1287 push(@chartdatanames, $temp);
1288 $minutes += $hist_interval;
1289 if ($minutes >= 60) {
1294 push(@chartdatavals, $c);
1296 printf $txt_fh (" %6d %s\n", $c, "." x ($c/$scale)) if $txt_fh;
1297 printf $htm_fh (" %6d %s\n", $c, "." x ($c/$scale)) if $htm_fh;
1298 $ws_global->write($row++, $col_hist+1, [$c], $f_default) if $xls_fh;
1302 printf $txt_fh "\n" if $txt_fh;
1303 printf $htm_fh "\n" if $htm_fh;
1307 print $htm_fh "</pre>\n";
1308 print $htm_fh "</td><td>\n";
1309 if ($HAVE_GD_Graph_linespoints && $charts && ($#chartdatavals > 0)) {
1310 # calculate the graph
1315 my $graph = GD::Graph::linespoints->new(300, 300);
1318 y_label => 'Amount',
1320 x_labels_vertical => 1
1322 my $pngname = "histogram_$text.png";
1323 $pngname =~ s/[^\w\._]/_/g;
1325 my $gd = $graph->plot(\@data) or warn($graph->error);
1327 open(IMG, ">$chartdir/$pngname") or die "Could not write $chartdir/$pngname: $!\n";
1331 print $htm_fh "<img src=\"$chartrel/$pngname\">";
1334 print $htm_fh "</td></tr></table>\n";
1337 $col_hist++; # where to continue next times
1339 $row+=2; # leave some space after history block
1340 $run_hist=1; # we have done this once or more
1345 #######################################################################
1346 # print_league_table();
1348 # print_league_table($league_table_type,\%message_count,\%address_count,\%message_data,\%message_data_gigs, $spreadsheet, $row_sref);
1350 # Given hashes of message count, address count, and message data,
1351 # which are keyed by the table type (eg by the sending host), print a
1352 # league table showing the top $topcount (defaults to 50).
1353 #######################################################################
1354 sub print_league_table {
1355 my($text,$m_count,$a_count,$m_data,$m_data_gigs,$spreadsheet, $row_sref) = @_;
1356 my($name) = ($topcount == 1)? "$text" : "$topcount ${text}s";
1357 my($title) = "Top $name by message count";
1358 my(@chartdatanames) = ();
1359 my(@chartdatavals) = ();
1360 my $chartotherval = 0;
1361 $text = ucfirst($text);
1363 # Align non-local addresses to the right (so all the .com's line up).
1364 # Local addresses are aligned on the left as they are userids.
1365 my $align = ($text !~ /local/i) ? 'right' : 'left';
1368 ################################################
1369 # Generate the printf formats and table headers.
1370 ################################################
1371 my(@headers) = ('Messages');
1372 #push(@headers,'Addresses') if defined $a_count;
1373 push(@headers,'Addresses') if defined $a_count && %$a_count;
1374 push(@headers,'Bytes','Average') if defined $m_data;
1376 my $txt_format = "%10s " x @headers . " %s\n";
1377 my $txt_col_headers = sprintf $txt_format, @headers, $text;
1378 my $htm_format = "<tr>" . '<td align="right">%s</td>'x@headers . "<td align=\"$align\" nowrap>%s</td></tr>\n";
1379 my $htm_col_headers = sprintf $htm_format, @headers, $text;
1380 $htm_col_headers =~ s/(<\/?)td/$1th/g; #Convert <td>'s to <th>'s for the header.
1383 ################################################
1384 # Write the table headers
1385 ################################################
1386 printf $txt_fh ("%s\n%s\n%s", $title, "-" x length($title),$txt_col_headers) if $txt_fh;
1389 print $htm_fh <<EoText;
1390 <hr><a name="$text count"></a><h2>$title</h2>
1391 <table border=0 width="100%">
1395 print $htm_fh $htm_col_headers
1399 $spreadsheet->write(${$row_sref}++, 0, $title, $f_header2);
1400 $spreadsheet->write(${$row_sref}++, 0, [@headers, $text], $f_headertab);
1405 foreach my $key (top_n_sort($topcount,$m_count,$m_data_gigs,$m_data)) {
1407 # When displaying the average figures, we calculate the average of
1408 # the rounded data, as the user would calculate it. This reduces
1409 # the accuracy slightly, but we have to do it this way otherwise
1410 # when using -merge to convert results from text to HTML and
1411 # vice-versa discrepencies would occur.
1412 my $messages = $$m_count{$key};
1413 my @content = ($messages);
1414 push(@content, $$a_count{$key}) if defined $a_count;
1415 if (defined $m_data) {
1416 my $rounded_volume = volume_rounded($$m_data{$key},$$m_data_gigs{$key});
1417 my($data,$gigs) = (0,0);
1418 un_round($rounded_volume,\$data,\$gigs);
1419 my $rounded_average = volume_rounded($data/$messages,$gigs/$messages);
1420 push(@content, $rounded_volume, $rounded_average);
1424 printf $txt_fh ($txt_format, @content, $key) if $txt_fh;
1428 $htmlkey =~ s/>/\>\;/g;
1429 $htmlkey =~ s/</\<\;/g;
1430 printf $htm_fh ($htm_format, @content, $htmlkey);
1432 $spreadsheet->write(${$row_sref}++, 0, [@content, $key], $f_default) if $xls_fh;
1434 if (scalar @chartdatanames < $ntopchart) {
1435 push(@chartdatanames, $key);
1436 push(@chartdatavals, $$m_count{$key});
1439 $chartotherval += $$m_count{$key};
1443 push(@chartdatanames, "Other");
1444 push(@chartdatavals, $chartotherval);
1446 print $txt_fh "\n" if $txt_fh;
1448 print $htm_fh "</table>\n";
1449 print $htm_fh "</td><td>\n";
1450 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0))
1452 # calculate the graph
1457 my $graph = GD::Graph::pie->new(300, 300);
1460 y_label => 'Amount',
1461 title => 'By count',
1463 my $gd = $graph->plot(\@data) or warn($graph->error);
1467 open(IMG, ">$chartdir/${temp}_count.png") or die "Could not write $chartdir/${temp}_count.png: $!\n";
1471 print $htm_fh "<img src=\"$chartrel/${temp}_count.png\">";
1474 print $htm_fh "</td><td>\n";
1475 print $htm_fh "</td></tr></table>\n\n";
1477 ++${$row_sref} if $xls_fh;
1480 if (defined $m_data) {
1483 $title = "Top $name by volume";
1485 printf $txt_fh ("%s\n%s\n%s", $title, "-" x length($title),$txt_col_headers) if $txt_fh;
1488 print $htm_fh <<EoText;
1489 <hr><a name="$text volume"></a><h2>$title</h2>
1490 <table border=0 width="100%">
1494 print $htm_fh $htm_col_headers;
1497 $spreadsheet->write(${$row_sref}++, 0, $title, $f_header2);
1498 $spreadsheet->write(${$row_sref}++, 0, [@headers, $text], $f_headertab);
1501 @chartdatanames = ();
1502 @chartdatavals = ();
1505 foreach my $key (top_n_sort($topcount,$m_data_gigs,$m_data,$m_count)) {
1506 # The largest volume will be the first (top of the list).
1507 # If it has at least 1 gig, then just use gigabytes to avoid
1508 # risking an integer overflow when generating the pie charts.
1509 if ($$m_data_gigs{$key}) {
1513 my $messages = $$m_count{$key};
1514 my @content = ($messages);
1515 push(@content, $$a_count{$key}) if defined $a_count;
1516 my $rounded_volume = volume_rounded($$m_data{$key},$$m_data_gigs{$key});
1517 my($data ,$gigs) = (0,0);
1518 un_round($rounded_volume,\$data,\$gigs);
1519 my $rounded_average = volume_rounded($data/$messages,$gigs/$messages);
1520 push(@content, $rounded_volume, $rounded_average );
1523 printf $txt_fh ($txt_format, @content, $key) if $txt_fh;
1526 $htmlkey =~ s/>/\>\;/g;
1527 $htmlkey =~ s/</\<\;/g;
1528 printf $htm_fh ($htm_format, @content, $htmlkey);
1530 $spreadsheet->write(${$row_sref}++, 0, [@content, $key], $f_default) if $xls_fh;
1533 if (scalar @chartdatanames < $ntopchart) {
1535 if ($$m_data_gigs{$key}) {
1536 push(@chartdatanames, $key);
1537 push(@chartdatavals, $$m_data_gigs{$key});
1541 push(@chartdatanames, $key);
1542 push(@chartdatavals, $$m_data{$key});
1546 $chartotherval += ($use_gig) ? $$m_data_gigs{$key} : $$m_data{$key};
1549 push(@chartdatanames, "Other");
1550 push(@chartdatavals, $chartotherval);
1552 print $txt_fh "\n" if $txt_fh;
1554 print $htm_fh "</table>\n";
1555 print $htm_fh "</td><td>\n";
1556 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0)) {
1557 # calculate the graph
1562 my $graph = GD::Graph::pie->new(300, 300);
1565 y_label => 'Volume' ,
1566 title => 'By Volume',
1568 my $gd = $graph->plot(\@data) or warn($graph->error);
1572 open(IMG, ">$chartdir/${temp}_volume.png") or die "Could not write $chartdir/${temp}_volume.png: $!\n";
1576 print $htm_fh "<img src=\"$chartrel/${temp}_volume.png\">";
1579 print $htm_fh "</td><td>\n";
1580 print $htm_fh "</td></tr></table>\n\n";
1583 ++${$row_sref} if $xls_fh;
1588 #######################################################################
1591 # @sorted_keys = top_n_sort($n,$href1,$href2,$href3);
1593 # Given a hash which has numerical values, return the sorted $n keys which
1594 # point to the top values. The second and third hashes are used as
1595 # tiebreakers. They all must have the same keys.
1597 # The idea behind this routine is that when you only want to see the
1598 # top n members of a set, rather than sorting the entire set and then
1599 # plucking off the top n, sort through the stack as you go, discarding
1600 # any member which is lower than your current n'th highest member.
1602 # This proves to be an order of magnitude faster for large hashes.
1603 # On 200,000 lines of mainlog it benchmarked 9 times faster.
1604 # On 700,000 lines of mainlog it benchmarked 13.8 times faster.
1606 # We assume the values are > 0.
1607 #######################################################################
1609 my($n,$href1,$href2,$href3) = @_;
1611 # PH's original sort was:
1613 # foreach $key (sort
1615 # $$m_count{$b} <=> $$m_count{$a} ||
1616 # $$m_data_gigs{$b} <=> $$m_data_gigs{$a} ||
1617 # $$m_data{$b} <=> $$m_data{$a} ||
1623 #We use a key of '_' to represent non-existant values, as null keys are valid.
1624 #'_' is not a valid domain, edomain, host, or email.
1625 my(@top_n_keys) = ('_') x $n;
1626 my($minimum_value1,$minimum_value2,$minimum_value3) = (0,0,0);
1628 my $n_minus_1 = $n - 1;
1629 my $n_minus_2 = $n - 2;
1631 # Create a dummy hash incase the user has not provided us with
1632 # tiebreaker hashes.
1634 $href2 = \%dummy_hash unless defined $href2;
1635 $href3 = \%dummy_hash unless defined $href3;
1637 # Pick out the top $n keys.
1638 my($key,$value1,$value2,$value3,$i,$comparison,$insert_position);
1639 while (($key,$value1) = each %$href1) {
1641 #print STDERR "key $key ($value1,",$href2->{$key},",",$href3->{$key},") <=> ($minimum_value1,$minimum_value2,$minimum_value3)\n";
1643 # Check to see that the new value is bigger than the lowest of the
1644 # top n keys that we're keeping. We test the main key first, because
1645 # for the majority of cases we can skip creating dummy hash values
1646 # should the user have not provided real tie-breaking hashes.
1647 next unless $value1 >= $minimum_value1;
1649 # Create a dummy hash entry for the key if required.
1650 # Note that setting the dummy_hash value sets it for both href2 &
1651 # href3. Also note that currently we are guarenteed to have a real
1652 # value for href3 if a real value for href2 exists so don't need to
1653 # test for it as well.
1654 $dummy_hash{$key} = 0 unless exists $href2->{$key};
1656 $comparison = $value1 <=> $minimum_value1 ||
1657 $href2->{$key} <=> $minimum_value2 ||
1658 $href3->{$key} <=> $minimum_value3 ||
1659 $top_n_key cmp $key;
1660 next unless ($comparison == 1);
1662 # As we will be using these values a few times, extract them into scalars.
1663 $value2 = $href2->{$key};
1664 $value3 = $href3->{$key};
1666 # This key is bigger than the bottom n key, so the lowest position we
1667 # will insert it into is $n minus 1 (the bottom of the list).
1668 $insert_position = $n_minus_1;
1670 # Now go through the list, stopping when we find a key that we're
1671 # bigger than, or we come to the penultimate position - we've
1672 # already tested bigger than the last.
1674 # Note: we go top down as the list starts off empty.
1675 # Note: stepping through the list in this way benchmarks nearly
1676 # three times faster than doing a sort() on the reduced list.
1677 # I assume this is because the list is already in order, and
1678 # we get a performance boost from not having to do hash lookups
1680 for ($i = 0; $i < $n_minus_1; $i++) {
1681 $top_n_key = $top_n_keys[$i];
1682 if ( ($top_n_key eq '_') ||
1683 ( ($value1 <=> $href1->{$top_n_key} ||
1684 $value2 <=> $href2->{$top_n_key} ||
1685 $value3 <=> $href3->{$top_n_key} ||
1686 $top_n_key cmp $key) == 1
1689 $insert_position = $i;
1694 # Remove the last element, then insert the new one.
1695 $#top_n_keys = $n_minus_2;
1696 splice(@top_n_keys,$insert_position,0,$key);
1698 # Extract our new minimum values.
1699 $top_n_key = $top_n_keys[$n_minus_1];
1700 if ($top_n_key ne '_') {
1701 $minimum_value1 = $href1->{$top_n_key};
1702 $minimum_value2 = $href2->{$top_n_key};
1703 $minimum_value3 = $href3->{$top_n_key};
1707 # Return the top n list, grepping out non-existant values, just in case
1708 # we didn't have that many values.
1709 return(grep(!/^_$/,@top_n_keys));
1714 #######################################################################
1717 # $header = html_header($title);
1719 # Print our HTML header and start the <body> block.
1720 #######################################################################
1723 my $text = << "EoText";
1724 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
1727 <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-15">
1728 <title>$title</title>
1730 <body bgcolor="white">
1738 #######################################################################
1743 # Display usage instructions and exit.
1744 #######################################################################
1748 eximstats Version $VERSION
1751 eximstats [Output] [Options] mainlog1 mainlog2 ...
1752 eximstats -merge -html [Options] report.1.html ... > weekly_rep.html
1755 eximstats -html=eximstats.html mainlog1 mainlog2 ...
1756 eximstats mainlog1 mainlog2 ... > report.txt
1758 Parses exim mainlog or syslog files and generates a statistical analysis
1759 of the messages processed.
1761 Valid output types are:
1762 -txt[=<file>] plain text (default unless no other type is specified)
1765 With no type and file given, defaults to -txt and STDOUT.
1768 -h<number> histogram divisions per hour. The default is 1, and
1769 0 suppresses histograms. Other valid values are:
1770 2, 3, 5, 10, 15, 20, 30 or 60.
1771 -ne don't display error information
1772 -nr don't display relaying information
1773 -nr/pattern/ don't display relaying information that matches
1774 -nt don't display transport information
1775 -nt/pattern/ don't display transport information that matches
1776 -nvr don't do volume rounding. Display in bytes, not KB/MB/GB.
1777 -t<number> display top <number> sources/destinations
1778 default is 50, 0 suppresses top listing
1779 -tnl omit local sources/destinations in top listing
1780 -t_remote_users show top user sources/destinations from non-local domains
1781 -q<list> list of times for queuing information. -q0 suppresses.
1782 -show_rt<list> Show the receipt times for all the messages.
1783 -show_dt<list> Show the delivery times for all the messages.
1784 <list> is an optional list of times in seconds.
1787 -include_original_destination show both the final and original
1788 destinations in the results rather than just the final ones.
1790 -byhost show results by sending host (default unless bydomain or
1791 byemail is specified)
1792 -bydomain show results by sending domain.
1793 -byemail show results by sender's email address
1794 -byedomain show results by sender's email domain
1795 -bylocaldomain show results by local domain
1797 -pattern "Description" /pattern/
1798 Count lines matching specified patterns and show them in
1799 the results. It can be specified multiple times. Eg:
1800 -pattern 'Refused connections' '/refused connection/'
1802 -merge merge previously generated reports into a new report
1804 -charts Create charts (this requires the GD::Graph modules).
1805 Only valid with -html.
1806 -chartdir <dir> Create the charts' png files in the directory <dir>
1807 -chartrel <dir> Specify the relative directory for the "img src=" tags
1808 from where to include the charts in the html file
1809 -chartdir and -chartrel default to '.'
1811 -emptyok It is OK if there is no valid input, don't print an error.
1813 -d Debug mode - dump the eval'ed parser onto STDERR.
1822 #######################################################################
1823 # generate_parser();
1825 # $parser = generate_parser();
1827 # This subroutine generates the parsing routine which will be
1828 # used to parse the mainlog. We take the base operation, and remove bits not in use.
1829 # This improves performance depending on what bits you take out or add.
1831 # I've tested using study(), but this does not improve performance.
1833 # We store our parsing routing in a variable, and process it looking for #IFDEF (Expression)
1834 # or #IFNDEF (Expression) statements and corresponding #ENDIF (Expression) statements. If
1835 # the expression evaluates to true, then it is included/excluded accordingly.
1836 #######################################################################
1837 sub generate_parser {
1839 my($ip,$host,$email,$edomain,$domain,$thissize,$size,$old,$new);
1840 my($tod,$m_hour,$m_min,$id,$flag,$extra,$length);
1841 my($seconds,$queued,$rcpt_time,$local_domain);
1845 # Convert syslog lines to mainlog format.
1847 next unless s/^.*? exim\\b.*?: //;
1850 $length = length($_);
1851 next if ($length < 38);
1852 next unless /^(\\d{4}\\-\\d\\d-\\d\\d\\s(\\d\\d):(\\d\\d):\\d\\d( [-+]\\d\\d\\d\\d)?)( \\[\\d+\\])?/o;
1854 ($tod,$m_hour,$m_min) = ($1,$2,$3);
1856 # PH - watch for GMT offsets in the timestamp.
1859 next if ($length < 44);
1865 # PH - watch for PID added after the timestamp.
1867 $extra += length($5);
1868 next if ($length < 38 + $extra);
1871 $id = substr($_, 20 + $extra, 16);
1872 $flag = substr($_, 37 + $extra, 2);
1874 if ($flag !~ /^([<>=*-]+|SA)$/ && /rejected|refused|dropped/) {
1879 # Rejects can have no MSGID...
1880 if ($flag eq "Re" && $id !~ /^[-0-9a-zA-Z]+$/) {
1881 $id = "reject:" . ++$rej_id;
1886 # Watch for user specified patterns.
1887 my $user_pattern_index = 0;
1888 foreach (@user_patterns) {
1889 $user_pattern_totals[$user_pattern_index] = 0;
1890 $parser .= " if ($_) {\n";
1891 $parser .= " \$user_pattern_totals[$user_pattern_index]++;\n";
1892 $parser .= " \$user_pattern_interval_count[$user_pattern_index][(\$m_hour*60 + \$m_min)/$hist_interval]++;\n" if ($hist_opt > 0);
1894 $user_pattern_index++;
1898 next unless ($flag =~ /<=|=>|->|==|\\*\\*|Co|SA|Re/);
1900 #Strip away the timestamp, ID and flag to speed up later pattern matches.
1901 #The flags include Co (Completed), Re (Rejected), and SA (SpamAssassin).
1902 $_ = substr($_, 40 + $extra); # PH
1904 # Alias @message to the array of information about the message.
1905 # This minimises the number of calls to hash functions.
1906 $messages{$id} = [] unless exists $messages{$id};
1907 *message = $messages{$id};
1910 # JN - Skip over certain transports as specified via the "-nt/.../" command
1911 # line switch (where ... is a perl style regular expression). This is
1912 # required so that transports that skew stats such as SpamAssassin can be
1914 #IFDEF ($transport_pattern)
1915 if (/\\sT=(\\S+)/) {
1916 next if ($1 =~ /$transport_pattern/o) ;
1918 #ENDIF ($transport_pattern)
1922 # Do some pattern matches to get the host and IP address.
1923 # We expect lines to be of the form "H=[IpAddr]" or "H=Host [IpAddr]" or
1924 # "H=Host (UnverifiedHost) [IpAddr]" or "H=(UnverifiedHost) [IpAddr]".
1925 # We do 2 separate matches to keep the matches simple and fast.
1926 # Host is local unless otherwise specified.
1927 $ip = (/\\bH=.*?(\\[[^]]+\\])/) ? $1
1928 # 2008-03-31 06:25:22 Connection from [213.246.33.217]:39456 refused: too many connections from that IP address // .hs
1929 : (/Connection from (\[\S+\])/) ? $1
1930 # 2008-03-31 06:52:40 SMTP call from mail.cacoshrf.com (ccsd02.ccsd.local) [69.24.118.229]:4511 dropped: too many nonmail commands (last was "RSET") // .hs
1931 : (/SMTP call from .*?(\[\S+\])/) ? $1
1933 $host = (/\\bH=(\\S+)/) ? $1 : "local";
1935 $domain = "localdomain"; #Domain is localdomain unless otherwise specified.
1937 #IFDEF ($do_sender{Domain})
1938 if ($host =~ /^\\[/ || $host =~ /^[\\d\\.]+$/) {
1939 # Host is just an IP address.
1942 elsif ($host =~ /^(\\(?)[^\\.]+\\.([^\\.]+\\..*)/) {
1943 # Remove the host portion from the DNS name. We ensure that we end up
1944 # with at least xxx.yyy. $host can be "(x.y.z)" or "x.y.z".
1945 $domain = lc("$1.$2");
1946 $domain =~ s/^\\.//; #Remove preceding dot.
1948 #ENDIF ($do_sender{Domain})
1950 #IFDEF ($do_sender{Email})
1951 #IFDEF ($include_original_destination)
1952 # Catch both "a@b.com <c@d.com>" and "e@f.com"
1953 #$email = (/^(\S+) (<(\S*?)>)?/) ? $3 || $1 : "";
1954 $email = (/^(\S+ (<[^@>]+@?[^>]*>)?)/) ? $1 : "";
1956 #ENDIF ($include_original_destination)
1958 #IFNDEF ($include_original_destination)
1959 $email = (/^(\S+)/) ? $1 : "";
1960 #ENDIF ($include_original_destination)
1961 #ENDIF ($do_sender{Email})
1963 #IFDEF ($do_sender{Edomain})
1964 if (/^(<>|blackhole)/) {
1967 #IFDEF ($include_original_destination)
1968 elsif (/^(\S+ (<\S*?\\@(\S+?)>)?)/) {
1971 $edomain =~ s/@(\S+?)>/"@" . lc($1) . ">"/e;
1973 #ENDIF ($include_original_destination)
1974 #IFNDEF ($include_original_destination)
1975 elsif (/^\S*?\\@(\S+)/) {
1978 #ENDIF ($include_original_destination)
1983 #ENDIF ($do_sender{Edomain})
1985 if ($tod lt $begin) {
1988 elsif ($tod gt $end) {
1993 if ($flag eq "<=") {
1994 $thissize = (/\\sS=(\\d+)( |$)/) ? $1 : 0;
1995 $message[$SIZE] = $thissize;
1996 $message[$PROTOCOL] = (/ P=(\S+)/) ? $1 : undef;
1998 #IFDEF ($show_relay)
1999 if ($host ne "local") {
2000 # Save incoming information in case it becomes interesting
2001 # later, when delivery lines are read.
2002 my($from) = /^(\\S+)/;
2003 $message[$FROM_HOST] = "$host$ip";
2004 $message[$FROM_ADDRESS] = $from;
2006 #ENDIF ($show_relay)
2008 #IFDEF ($local_league_table || $include_remote_users)
2012 #IFDEF ($local_league_table && $include_remote_users)
2013 { #Store both local and remote users.
2014 #ENDIF ($local_league_table && $include_remote_users)
2016 #IFDEF ($local_league_table && ! $include_remote_users)
2017 if ($host eq "local") { #Store local users only.
2018 #ENDIF ($local_league_table && ! $include_remote_users)
2020 #IFDEF ($include_remote_users && ! $local_league_table)
2021 if ($host ne "local") { #Store remote users only.
2022 #ENDIF ($include_remote_users && ! $local_league_table)
2024 ++$received_count_user{$user};
2025 add_volume(\\$received_data_user{$user},\\$received_data_gigs_user{$user},$thissize);
2028 #ENDIF ($local_league_table || $include_remote_users)
2030 #IFDEF ($do_sender{Host})
2031 ++$received_count{Host}{$host};
2032 add_volume(\\$received_data{Host}{$host},\\$received_data_gigs{Host}{$host},$thissize);
2033 #ENDIF ($do_sender{Host})
2035 #IFDEF ($do_sender{Domain})
2037 ++$received_count{Domain}{$domain};
2038 add_volume(\\$received_data{Domain}{$domain},\\$received_data_gigs{Domain}{$domain},$thissize);
2040 #ENDIF ($do_sender{Domain})
2042 #IFDEF ($do_sender{Email})
2043 ++$received_count{Email}{$email};
2044 add_volume(\\$received_data{Email}{$email},\\$received_data_gigs{Email}{$email},$thissize);
2045 #ENDIF ($do_sender{Email})
2047 #IFDEF ($do_sender{Edomain})
2048 ++$received_count{Edomain}{$edomain};
2049 add_volume(\\$received_data{Edomain}{$edomain},\\$received_data_gigs{Edomain}{$edomain},$thissize);
2050 #ENDIF ($do_sender{Edomain})
2052 ++$total_received_count;
2053 add_volume(\\$total_received_data,\\$total_received_data_gigs,$thissize);
2055 #IFDEF ($#queue_times >= 0 || $#rcpt_times >= 0)
2056 $message[$ARRIVAL_TIME] = $tod;
2057 #ENDIF ($#queue_times >= 0 || $#rcpt_times >= 0)
2059 #IFDEF ($hist_opt > 0)
2060 $received_interval_count[($m_hour*60 + $m_min)/$hist_interval]++;
2061 #ENDIF ($hist_opt > 0)
2064 elsif ($flag eq "=>") {
2065 $size = $message[$SIZE] || 0;
2066 if ($host ne "local") {
2067 $message[$REMOTE_DELIVERED] = 1;
2070 #IFDEF ($show_relay)
2071 # Determine relaying address if either only one address listed,
2072 # or two the same. If they are different, it implies a forwarding
2073 # or aliasing, which is not relaying. Note that for multi-aliased
2074 # addresses, there may be a further address between the first
2077 if (defined $message[$FROM_HOST]) {
2078 if (/^(\\S+)(?:\\s+\\([^)]\\))?\\s+<([^>]+)>/) {
2079 ($old,$new) = ($1,$2);
2085 if ("\\L$new" eq "\\L$old") {
2086 ($old) = /^(\\S+)/ if $old eq "";
2087 my $key = "H=\\L$message[$FROM_HOST]\\E A=\\L$message[$FROM_ADDRESS]\\E => " .
2088 "H=\\L$host\\E$ip A=\\L$old\\E";
2089 if (!defined $relay_pattern || $key !~ /$relay_pattern/o) {
2090 $relayed{$key} = 0 if !defined $relayed{$key};
2098 #ENDIF ($show_relay)
2102 #IFDEF ($local_league_table || $include_remote_users)
2103 #IFDEF ($local_league_table && $include_remote_users)
2104 { #Store both local and remote users.
2105 #ENDIF ($local_league_table && $include_remote_users)
2107 #IFDEF ($local_league_table && ! $include_remote_users)
2108 if ($host eq "local") { #Store local users only.
2109 #ENDIF ($local_league_table && ! $include_remote_users)
2111 #IFDEF ($include_remote_users && ! $local_league_table)
2112 if ($host ne "local") { #Store remote users only.
2113 #ENDIF ($include_remote_users && ! $local_league_table)
2115 if (my($user) = split((/\\s</)? " <" : " ", $_)) {
2116 #IFDEF ($include_original_destination)
2118 #ENDIF ($include_original_destination)
2119 #IFNDEF ($include_original_destination)
2120 if ($user =~ /^[\\/|]/) {
2121 #ENDIF ($include_original_destination)
2122 #my($parent) = $_ =~ /(<[^@]+@?[^>]*>)/;
2123 my($parent) = $_ =~ / (<.+?>) /; #DT 1.54
2124 if (defined $parent) {
2125 $user = "$user $parent";
2126 #IFDEF ($do_local_domain)
2127 if ($parent =~ /\\@(.+)>/) {
2128 $local_domain = lc($1);
2129 ++$delivered_messages_local_domain{$local_domain};
2130 ++$delivered_addresses_local_domain{$local_domain};
2131 add_volume(\\$delivered_data_local_domain{$local_domain},\\$delivered_data_gigs_local_domain{$local_domain},$size);
2133 #ENDIF ($do_local_domain)
2136 ++$delivered_messages_user{$user};
2137 ++$delivered_addresses_user{$user};
2138 add_volume(\\$delivered_data_user{$user},\\$delivered_data_gigs_user{$user},$size);
2141 #ENDIF ($local_league_table || $include_remote_users)
2143 #IFDEF ($do_sender{Host})
2144 $delivered_messages{Host}{$host}++;
2145 $delivered_addresses{Host}{$host}++;
2146 add_volume(\\$delivered_data{Host}{$host},\\$delivered_data_gigs{Host}{$host},$size);
2147 #ENDIF ($do_sender{Host})
2148 #IFDEF ($do_sender{Domain})
2150 ++$delivered_messages{Domain}{$domain};
2151 ++$delivered_addresses{Domain}{$domain};
2152 add_volume(\\$delivered_data{Domain}{$domain},\\$delivered_data_gigs{Domain}{$domain},$size);
2154 #ENDIF ($do_sender{Domain})
2155 #IFDEF ($do_sender{Email})
2156 ++$delivered_messages{Email}{$email};
2157 ++$delivered_addresses{Email}{$email};
2158 add_volume(\\$delivered_data{Email}{$email},\\$delivered_data_gigs{Email}{$email},$size);
2159 #ENDIF ($do_sender{Email})
2160 #IFDEF ($do_sender{Edomain})
2161 ++$delivered_messages{Edomain}{$edomain};
2162 ++$delivered_addresses{Edomain}{$edomain};
2163 add_volume(\\$delivered_data{Edomain}{$edomain},\\$delivered_data_gigs{Edomain}{$edomain},$size);
2164 #ENDIF ($do_sender{Edomain})
2166 ++$total_delivered_messages;
2167 ++$total_delivered_addresses;
2168 add_volume(\\$total_delivered_data,\\$total_delivered_data_gigs,$size);
2170 #IFDEF ($show_transport)
2171 my $transport = (/\\sT=(\\S+)/) ? $1 : ":blackhole:";
2172 ++$transported_count{$transport};
2173 add_volume(\\$transported_data{$transport},\\$transported_data_gigs{$transport},$size);
2174 #ENDIF ($show_transport)
2176 #IFDEF ($hist_opt > 0)
2177 $delivered_interval_count[($m_hour*60 + $m_min)/$hist_interval]++;
2178 #ENDIF ($hist_opt > 0)
2180 #IFDEF ($#delivery_times > 0)
2182 $seconds = wdhms_seconds($1);
2183 for ($i = 0; $i <= $#delivery_times; $i++) {
2184 if ($seconds < $delivery_times[$i]) {
2186 ++$dt_remote_bin[$i] if $message[$REMOTE_DELIVERED];
2190 if ($i > $#delivery_times) {
2192 ++$dt_remote_overflow if $message[$REMOTE_DELIVERED];
2195 #ENDIF ($#delivery_times > 0)
2199 elsif ($flag eq "->") {
2201 #IFDEF ($local_league_table || $include_remote_users)
2202 #IFDEF ($local_league_table && $include_remote_users)
2203 { #Store both local and remote users.
2204 #ENDIF ($local_league_table && $include_remote_users)
2206 #IFDEF ($local_league_table && ! $include_remote_users)
2207 if ($host eq "local") { #Store local users only.
2208 #ENDIF ($local_league_table && ! $include_remote_users)
2210 #IFDEF ($include_remote_users && ! $local_league_table)
2211 if ($host ne "local") { #Store remote users only.
2212 #ENDIF ($include_remote_users && ! $local_league_table)
2214 if (my($user) = split((/\\s</)? " <" : " ", $_)) {
2215 #IFDEF ($include_original_destination)
2217 #ENDIF ($include_original_destination)
2218 #IFNDEF ($include_original_destination)
2219 if ($user =~ /^[\\/|]/) {
2220 #ENDIF ($include_original_destination)
2221 #my($parent) = $_ =~ /(<[^@]+@?[^>]*>)/;
2222 my($parent) = $_ =~ / (<.+?>) /; #DT 1.54
2223 $user = "$user $parent" if defined $parent;
2225 ++$delivered_addresses_user{$user};
2228 #ENDIF ($local_league_table || $include_remote_users)
2230 #IFDEF ($do_sender{Host})
2231 $delivered_addresses{Host}{$host}++;
2232 #ENDIF ($do_sender{Host})
2233 #IFDEF ($do_sender{Domain})
2235 ++$delivered_addresses{Domain}{$domain};
2237 #ENDIF ($do_sender{Domain})
2238 #IFDEF ($do_sender{Email})
2239 ++$delivered_addresses{Email}{$email};
2240 #ENDIF ($do_sender{Email})
2241 #IFDEF ($do_sender{Edomain})
2242 ++$delivered_addresses{Edomain}{$edomain};
2243 #ENDIF ($do_sender{Edomain})
2245 ++$total_delivered_addresses;
2248 elsif ($flag eq "==" && defined($message[$SIZE]) && !defined($message[$DELAYED])) {
2250 $message[$DELAYED] = 1;
2253 elsif ($flag eq "**") {
2254 if (defined ($message[$SIZE])) {
2255 unless (defined $message[$HAD_ERROR]) {
2257 $message[$HAD_ERROR] = 1;
2261 #IFDEF ($show_errors)
2262 ++$errors_count{$_};
2263 #ENDIF ($show_errors)
2267 elsif ($flag eq "Co") {
2269 #IFDEF ($#queue_times >= 0)
2270 $queued = queue_time($tod, $message[$ARRIVAL_TIME], $id);
2272 for ($i = 0; $i <= $#queue_times; $i++) {
2273 if ($queued < $queue_times[$i]) {
2275 ++$qt_remote_bin[$i] if $message[$REMOTE_DELIVERED];
2279 if ($i > $#queue_times) {
2281 ++$qt_remote_overflow if $message[$REMOTE_DELIVERED];
2283 #ENDIF ($#queue_times >= 0)
2285 #IFDEF ($#rcpt_times >= 0)
2287 $seconds = wdhms_seconds($1);
2288 #Calculate $queued if not previously calculated above.
2289 #IFNDEF ($#queue_times >= 0)
2290 $queued = queue_time($tod, $message[$ARRIVAL_TIME], $id);
2291 #ENDIF ($#queue_times >= 0)
2292 $rcpt_time = $seconds - $queued;
2295 if (defined $message[$PROTOCOL]) {
2296 $protocol = $message[$PROTOCOL];
2298 # Create the bin if its not already defined.
2299 unless (exists $rcpt_times_bin{$protocol}) {
2300 initialise_rcpt_times($protocol);
2305 for ($i = 0; $i <= $#rcpt_times; ++$i) {
2306 if ($rcpt_time < $rcpt_times[$i]) {
2307 ++$rcpt_times_bin{all}[$i];
2308 ++$rcpt_times_bin{$protocol}[$i] if defined $protocol;
2313 if ($i > $#rcpt_times) {
2314 ++$rcpt_times_overflow{all};
2315 ++$rcpt_times_overflow{$protocol} if defined $protocol;
2318 #ENDIF ($#rcpt_times >= 0)
2320 delete($messages{$id});
2322 elsif ($flag eq "SA") {
2323 $ip = (/From.*?(\\[[^]]+\\])/ || /\\((local)\\)/) ? $1 : "";
2324 #SpamAssassin message
2325 if (/Action: ((permanently|temporarily) rejected message|flagged as Spam but accepted): score=(\d+\.\d)/) {
2326 #add_volume(\\$spam_score,\\$spam_score_gigs,$3);
2327 ++$spam_count_by_ip{$ip};
2328 } elsif (/Action: scanned but message isn\'t spam: score=(-?\d+\.\d)/) {
2329 #add_volume(\\$ham_score,\\$ham_score_gigs,$1);
2330 ++$ham_count_by_ip{$ip};
2331 } elsif (/(Not running SA because SAEximRunCond expanded to false|check skipped due to message size)/) {
2332 ++$ham_count_by_ip{$ip};
2336 # Look for Reject messages or blackholed messages (deliveries
2337 # without a transport)
2338 if ($flag eq "Re" || ($flag eq "=>" && ! /\\sT=\\S+/)) {
2339 # Correct the IP address for rejects:
2340 # rejected EHLO from my.test.net [10.0.0.5]: syntactically invalid argument(s):
2341 # rejected EHLO from [10.0.0.6]: syntactically invalid argument(s):
2342 $ip = $1 if ($ip eq "local" && /^rejected [HE][HE]LO from .*?(\[.+?\]):/);
2343 if (/SpamAssassin/) {
2344 ++$rejected_count_by_reason{"Rejected by SpamAssassin"};
2345 ++$rejected_count_by_ip{$ip};
2348 /(temporarily rejected [A-Z]*) .*?(: .*?)(:|\s*$)/
2350 ++$temporarily_rejected_count_by_reason{"\u$1$2"};
2351 ++$temporarily_rejected_count_by_ip{$ip};
2354 /(temporarily refused connection)/
2356 ++$temporarily_rejected_count_by_reason{"\u$1"};
2357 ++$temporarily_rejected_count_by_ip{$ip};
2360 /(listed at [^ ]+)/ ||
2361 /(Forged IP detected in HELO)/ ||
2362 /(Invalid domain or IP given in HELO\/EHLO)/ ||
2363 /(unqualified recipient rejected)/ ||
2364 /(closed connection (after|in response) .*?)\s*$/ ||
2365 /(sender rejected)/ ||
2366 # 2005-09-23 15:07:49 1EInHJ-0007Ex-Au H=(a.b.c) [10.0.0.1] F=<> rejected after DATA: This message contains a virus: (Eicar-Test-Signature) please scan your system.
2367 # 2005-10-06 10:50:07 1ENRS3-0000Nr-Kt => blackhole (DATA ACL discarded recipients): This message contains a virus: (Worm.SomeFool.P) please scan your system.
2368 / rejected after DATA: (.*)/ ||
2369 /.DATA ACL discarded recipients.: (.*)/ ||
2370 /rejected after DATA: (unqualified address not permitted)/ ||
2371 /(VRFY rejected)/ ||
2372 # /(sender verify (defer|fail))/i ||
2373 /(too many recipients)/ ||
2374 /(refused relay.*?) to/ ||
2375 /(rejected by non-SMTP ACL: .*)/ ||
2376 /(rejected by local_scan.*)/ ||
2377 # SMTP call from %s dropped: too many syntax or protocol errors (last command was "%s"
2378 # SMTP call from %s dropped: too many nonmail commands
2379 /(dropped: too many ((nonmail|unrecognized) commands|syntax or protocol errors))/ ||
2381 # local_scan() function crashed with signal %d - message temporarily rejected
2382 # local_scan() function timed out - message temporarily rejected
2383 /(local_scan.. function .* - message temporarily rejected)/ ||
2384 # SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from %s
2385 /(SMTP protocol .*?(error|violation))/ ||
2388 ++$rejected_count_by_reason{"\u$1"};
2389 ++$rejected_count_by_ip{$ip};
2391 elsif (/rejected [HE][HE]LO from [^:]*: syntactically invalid argument/) {
2392 ++$rejected_count_by_reason{"Rejected HELO/EHLO: syntactically invalid argument"};
2393 ++$rejected_count_by_ip{$ip};
2395 elsif (/response to "RCPT TO.*? was: (.*)/) {
2396 ++$rejected_count_by_reason{"Response to RCPT TO was: $1"};
2397 ++$rejected_count_by_ip{$ip};
2400 /(lookup of host )\S+ (failed)/ ||
2402 # rejected from <%s>%s%s%s%s: message too big:
2403 /(rejected [A-Z]*) .*?(: .*?)(:|\s*$)/ ||
2404 # refused connection from %s (host_reject_connection)
2405 # refused connection from %s (tcp wrappers)
2406 /(refused connection )from.*? (\(.*)/ ||
2408 # error from remote mailer after RCPT TO:<a@b.c>: host a.b.c [10.0.0.1]: 450 <a@b.c>: Recipient address rejected: Greylisted for 60 seconds
2409 # error from remote mailer after MAIL FROM:<> SIZE=3468: host a.b.c [10.0.0.1]: 421 a.b.c has refused your connection because your server did not have a PTR record.
2410 /(error from remote mailer after .*?:).*(: .*?)(:|\s*$)/ ||
2412 # a.b.c F=<a@b.c> rejected after DATA: "@" or "." expected after "Undisclosed-Recipient": failing address in "To" header is: <Undisclosed-Recipient:;>
2413 /rejected after DATA: ("." or "." expected).*?(: failing address in .*? header)/ ||
2415 # connection from %s refused load average = %.2f
2416 /(Connection )from.*? (refused: load average)/ ||
2417 # connection from %s refused (IP options)
2418 # Connection from %s refused: too many connections
2419 # connection from %s refused
2420 /([Cc]onnection )from.*? (refused.*)/ ||
2421 # [10.0.0.1]: connection refused
2422 /: (Connection refused)()/
2424 ++$rejected_count_by_reason{"\u$1$2"};
2425 ++$rejected_count_by_ip{$ip};
2428 # 2008-03-31 06:25:22 H=mail.densitron.com [216.70.140.224]:45386 temporarily rejected connection in "connect" ACL: too fast reconnects // .hs
2429 # 2008-03-31 06:25:22 H=mail.densitron.com [216.70.140.224]:45386 temporarily rejected connection in "connect" ACL // .hs
2430 /(temporarily rejected connection in .*?ACL:?.*)/
2432 ++$temporarily_rejected_count_by_ip{$ip};
2433 ++$temporarily_rejected_count_by_reason{"\u$1"};
2436 ++$rejected_count_by_reason{Unknown};
2437 ++$rejected_count_by_ip{$ip};
2438 print STDERR "Unknown rejection: $_" if $debug;
2443 # We now do a 'C preprocessor style operation on our parser
2444 # to remove bits not in use.
2445 my(%defines_in_operation,$removing_lines,$processed_parser);
2446 foreach (split (/\n/,$parser)) {
2447 if ((/^\s*#\s*IFDEF\s*\((.*?)\)/i && ! eval $1) ||
2448 (/^\s*#\s*IFNDEF\s*\((.*?)\)/i && eval $1) ) {
2449 $defines_in_operation{$1} = 1;
2450 $removing_lines = 1;
2453 # Convert constants.
2454 while (/(\$[A-Z][A-Z_]*)\b/) {
2455 my $constant = eval $1;
2456 s/(\$[A-Z][A-Z_]*)\b/$constant/;
2459 $processed_parser .= $_."\n" unless $removing_lines;
2461 if (/^\s*#\s*ENDIF\s*\((.*?)\)/i) {
2462 delete $defines_in_operation{$1};
2463 unless (keys %defines_in_operation) {
2464 $removing_lines = 0;
2468 print STDERR "# START OF PARSER:$processed_parser\n# END OF PARSER\n\n" if $debug;
2470 return $processed_parser;
2475 #######################################################################
2478 # parse($parser,\*FILEHANDLE);
2480 # This subroutine accepts a parser and a filehandle from main and parses each
2481 # line. We store the results into global variables.
2482 #######################################################################
2484 my($parser,$fh) = @_;
2486 if ($merge_reports) {
2487 parse_old_eximstat_reports($fh);
2498 #######################################################################
2503 # Print our headers and contents.
2504 #######################################################################
2508 my $title = "Exim statistics from $begin to $end";
2510 print $txt_fh "\n$title\n" if $txt_fh;
2512 print $htm_fh html_header($title);
2513 print $htm_fh "<ul>\n";
2514 print $htm_fh "<li><a href=\"#Grandtotal\">Grand total summary</a>\n";
2515 print $htm_fh "<li><a href=\"#Patterns\">User Specified Patterns</a>\n" if @user_patterns;
2516 print $htm_fh "<li><a href=\"#Transport\">Deliveries by Transport</a>\n" if $show_transport;
2518 print $htm_fh "<li><a href=\"#Messages received\">Messages received per hour</a>\n";
2519 print $htm_fh "<li><a href=\"#Deliveries\">Deliveries per hour</a>\n";
2522 if ($#queue_times >= 0) {
2523 print $htm_fh "<li><a href=\"#Time spent on the queue all messages\">Time spent on the queue: all messages</a>\n";
2524 print $htm_fh "<li><a href=\"#Time spent on the queue messages with at least one remote delivery\">Time spent on the queue: messages with at least one remote delivery</a>\n";
2527 if ($#delivery_times >= 0) {
2528 print $htm_fh "<li><a href=\"#Delivery times all messages\">Delivery times: all messages</a>\n";
2529 print $htm_fh "<li><a href=\"#Delivery times messages with at least one remote delivery\">Delivery times: messages with at least one remote delivery</a>\n";
2532 if ($#rcpt_times >= 0) {
2533 print $htm_fh "<li><a href=\"#Receipt times all messages\">Receipt times</a>\n";
2536 print $htm_fh "<li><a href=\"#Relayed messages\">Relayed messages</a>\n" if $show_relay;
2538 print $htm_fh "<li><a href=\"#Mail rejection reason count\">Top $topcount mail rejection reasons by message count</a>\n" if %rejected_count_by_reason;
2539 foreach ('Host','Domain','Email','Edomain') {
2540 next unless $do_sender{$_};
2541 print $htm_fh "<li><a href=\"#Sending \l$_ count\">Top $topcount sending \l${_}s by message count</a>\n";
2542 print $htm_fh "<li><a href=\"#Sending \l$_ volume\">Top $topcount sending \l${_}s by volume</a>\n";
2544 if (($local_league_table || $include_remote_users) && %received_count_user) {
2545 print $htm_fh "<li><a href=\"#Local sender count\">Top $topcount local senders by message count</a>\n";
2546 print $htm_fh "<li><a href=\"#Local sender volume\">Top $topcount local senders by volume</a>\n";
2548 foreach ('Host','Domain','Email','Edomain') {
2549 next unless $do_sender{$_};
2550 print $htm_fh "<li><a href=\"#$_ destination count\">Top $topcount \l$_ destinations by message count</a>\n";
2551 print $htm_fh "<li><a href=\"#$_ destination volume\">Top $topcount \l$_ destinations by volume</a>\n";
2553 if (($local_league_table || $include_remote_users) && %delivered_messages_user) {
2554 print $htm_fh "<li><a href=\"#Local destination count\">Top $topcount local destinations by message count</a>\n";
2555 print $htm_fh "<li><a href=\"#Local destination volume\">Top $topcount local destinations by volume</a>\n";
2557 if (($local_league_table || $include_remote_users) && %delivered_messages_local_domain) {
2558 print $htm_fh "<li><a href=\"#Local domain destination count\">Top $topcount local domain destinations by message count</a>\n";
2559 print $htm_fh "<li><a href=\"#Local domain destination volume\">Top $topcount local domain destinations by volume</a>\n";
2562 print $htm_fh "<li><a href=\"#Rejected ip count\">Top $topcount rejected ips by message count</a>\n" if %rejected_count_by_ip;
2563 print $htm_fh "<li><a href=\"#Temporarily rejected ip count\">Top $topcount temporarily rejected ips by message count</a>\n" if %temporarily_rejected_count_by_ip;
2564 print $htm_fh "<li><a href=\"#Non-rejected spamming ip count\">Top $topcount non-rejected spamming ips by message count</a>\n" if %spam_count_by_ip;
2567 print $htm_fh "<li><a href=\"#errors\">List of errors</a>\n" if %errors_count;
2568 print $htm_fh "</ul>\n<hr>\n";
2572 $ws_global->write($row++, $col+0, "Exim Statistics", $f_header1);
2573 &set_worksheet_line($ws_global, $row, $col, ["from:", $begin, "to:", $end], $f_default);
2579 #######################################################################
2580 # print_grandtotals();
2582 # print_grandtotals();
2584 # Print the grand totals.
2585 #######################################################################
2586 sub print_grandtotals {
2588 # Get the sender by headings and results. This is complicated as we can have
2589 # different numbers of columns.
2590 my($sender_txt_header,$sender_txt_format,$sender_html_format);
2591 my(@received_totals,@delivered_totals);
2592 my($row_tablehead, $row_max);
2593 my(@col_headers) = ('TOTAL', 'Volume', 'Messages', 'Addresses');
2595 foreach ('Host','Domain','Email','Edomain') {
2596 next unless $do_sender{$_};
2597 if ($merge_reports) {
2598 push(@received_totals, get_report_total($report_totals{Received},"${_}s"));
2599 push(@delivered_totals,get_report_total($report_totals{Delivered},"${_}s"));
2602 push(@received_totals,scalar(keys %{$received_data{$_}}));
2603 push(@delivered_totals,scalar(keys %{$delivered_data{$_}}));
2605 $sender_txt_header .= " " x ($COLUMN_WIDTHS - length($_)) . $_ . 's';
2606 $sender_html_format .= "<td align=\"right\">%s</td>";
2607 $sender_txt_format .= " " x ($COLUMN_WIDTHS - 5) . "%6s";
2608 push(@col_headers,"${_}s");
2611 my $txt_format1 = " %-16s %9s %6d %6s $sender_txt_format";
2612 my $txt_format2 = " %6d %4.1f%% %6d %4.1f%%",
2613 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%s</td><td align=\"right\">%s</td><td align=\"right\">%s</td>$sender_html_format";
2614 my $htm_format2 = "<td align=\"right\">%d</td><td align=\"right\">%4.1f%%</td><td align=\"right\">%d</td><td align=\"right\">%4.1f%%</td>";
2617 my $sender_spaces = " " x length($sender_txt_header);
2619 print $txt_fh "Grand total summary\n";
2620 print $txt_fh "-------------------\n";
2621 print $txt_fh " $sender_spaces At least one address\n";
2622 print $txt_fh " TOTAL Volume Messages Addresses $sender_txt_header Delayed Failed\n";
2625 print $htm_fh "<a name=\"Grandtotal\"></a>\n";
2626 print $htm_fh "<h2>Grand total summary</h2>\n";
2627 print $htm_fh "<table border=1>\n";
2628 print $htm_fh "<tr><th>" . join('</th><th>',@col_headers) . "</th><th colspan=2>At least one addr<br>Delayed</th><th colspan=2>At least one addr<br>Failed</th>\n";
2631 $ws_global->write($row++, 0, "Grand total summary", $f_header2);
2632 $ws_global->write($row, 0, \@col_headers, $f_header2);
2633 $ws_global->merge_range($row, scalar(@col_headers), $row, scalar(@col_headers)+1, "At least one addr Delayed", $f_header2_m);
2634 $ws_global->merge_range($row, scalar(@col_headers)+2, $row, scalar(@col_headers)+3, "At least one addr Failed", $f_header2_m);
2635 #$ws_global->write(++$row, scalar(@col_headers), ['Total','Percent','Total','Percent'], $f_header2);
2639 my($volume,$failed_count);
2640 if ($merge_reports) {
2641 $volume = volume_rounded($report_totals{Received}{Volume}, $report_totals{Received}{'Volume-gigs'});
2642 $total_received_count = get_report_total($report_totals{Received},'Messages');
2643 $failed_count = get_report_total($report_totals{Received},'Failed');
2644 $delayed_count = get_report_total($report_totals{Received},'Delayed');
2647 $volume = volume_rounded($total_received_data, $total_received_data_gigs);
2648 $failed_count = $message_errors;
2655 $volume,$total_received_count,'',
2658 ($total_received_count) ? ($delayed_count*100/$total_received_count) : 0,
2660 ($total_received_count) ? ($failed_count*100/$total_received_count) : 0
2663 printf $txt_fh ("$txt_format1$txt_format2\n", 'Received', @content) if $txt_fh;
2664 printf $htm_fh ("$htm_format1$htm_format2\n", 'Received', @content) if $htm_fh;
2666 $ws_global->write(++$row, 0, 'Received', $f_default);
2667 for (my $i=0; $i < scalar(@content); $i++) {
2668 if ($i == 4 || $i == 6) {
2669 $ws_global->write($row, $i+1, $content[$i]/100, $f_percent);
2672 $ws_global->write($row, $i+1, $content[$i], $f_default);
2678 if ($merge_reports) {
2679 $volume = volume_rounded($report_totals{Delivered}{Volume}, $report_totals{Delivered}{'Volume-gigs'});
2680 $total_delivered_messages = get_report_total($report_totals{Delivered},'Messages');
2681 $total_delivered_addresses = get_report_total($report_totals{Delivered},'Addresses');
2684 $volume = volume_rounded($total_delivered_data, $total_delivered_data_gigs);
2687 my @content=($volume, $total_delivered_messages, $total_delivered_addresses, @delivered_totals);
2688 printf $txt_fh ("$txt_format1\n", 'Delivered', @content) if $txt_fh;
2689 printf $htm_fh ("$htm_format1\n", 'Delivered', @content) if $htm_fh;
2692 $ws_global->write(++$row, 0, 'Delivered', $f_default);
2693 for (my $i=0; $i < scalar(@content); $i++) {
2694 $ws_global->write($row, $i+1, $content[$i], $f_default);
2698 if ($merge_reports) {
2699 foreach ('Rejects', 'Temp Rejects', 'Ham', 'Spam') {
2700 my $messages = get_report_total($report_totals{$_},'Messages');
2701 my $addresses = get_report_total($report_totals{$_},'Addresses');
2703 @content = ($_, '', $messages, '');
2704 push(@content,get_report_total($report_totals{$_},'Hosts')) if $do_sender{Host};
2705 #These rows do not have entries for the following columns (if specified)
2706 foreach ('Domain','Email','Edomain') {
2707 push(@content,'') if $do_sender{$_};
2710 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2711 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2712 $ws_global->write(++$row, 0, \@content) if $xls_fh;
2717 foreach my $total_aref (['Rejects',\%rejected_count_by_ip],
2718 ['Temp Rejects',\%temporarily_rejected_count_by_ip],
2719 ['Ham',\%ham_count_by_ip],
2720 ['Spam',\%spam_count_by_ip]) {
2721 #Count the number of messages of this type.
2723 map {$messages += $_} values %{$total_aref->[1]};
2725 if ($messages > 0) {
2726 @content = ($total_aref->[0], '', $messages, '');
2728 #Count the number of distict IPs for the Hosts column.
2729 push(@content,scalar(keys %{$total_aref->[1]})) if $do_sender{Host};
2731 #These rows do not have entries for the following columns (if specified)
2732 foreach ('Domain','Email','Edomain') {
2733 push(@content,'') if $do_sender{$_};
2736 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2737 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2738 $ws_global->write(++$row, 0, \@content) if $xls_fh;
2743 printf $txt_fh "\n" if $txt_fh;
2744 printf $htm_fh "</table>\n" if $htm_fh;
2749 #######################################################################
2750 # print_user_patterns()
2752 # print_user_patterns();
2754 # Print the counts of user specified patterns.
2755 #######################################################################
2756 sub print_user_patterns {
2757 my $txt_format1 = " %-18s %6d";
2758 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%d</td>";
2761 print $txt_fh "User Specified Patterns\n";
2762 print $txt_fh "-----------------------";
2763 print $txt_fh "\n Total\n";
2766 print $htm_fh "<hr><a name=\"Patterns\"></a><h2>User Specified Patterns</h2>\n";
2767 print $htm_fh "<table border=0 width=\"100%\">\n";
2768 print $htm_fh "<tr><td>\n";
2769 print $htm_fh "<table border=1>\n";
2770 print $htm_fh "<tr><th> </th><th>Total</th>\n";
2773 $ws_global->write($row++, $col, "User Specified Patterns", $f_header2);
2774 &set_worksheet_line($ws_global, $row++, 1, ["Total"], $f_headertab);
2779 if ($merge_reports) {
2780 # We are getting our data from previous reports.
2781 foreach $key (@user_descriptions) {
2782 my $count = get_report_total($report_totals{patterns}{$key},'Total');
2783 printf $txt_fh ("$txt_format1\n",$key,$count) if $txt_fh;
2784 printf $htm_fh ("$htm_format1\n",$key,$count) if $htm_fh;
2787 &set_worksheet_line($ws_global, $row++, 0, [$key,$count], $f_default);
2792 # We are getting our data from mainlog files.
2793 my $user_pattern_index = 0;
2794 foreach $key (@user_descriptions) {
2795 printf $txt_fh ("$txt_format1\n",$key,$user_pattern_totals[$user_pattern_index]) if $txt_fh;
2796 printf $htm_fh ("$htm_format1\n",$key,$user_pattern_totals[$user_pattern_index]) if $htm_fh;
2797 $ws_global->write($row++, 0, [$key,$user_pattern_totals[$user_pattern_index]]) if $xls_fh;
2798 $user_pattern_index++;
2801 print $txt_fh "\n" if $txt_fh;
2802 print $htm_fh "</table>\n\n" if $htm_fh;
2808 if ($hist_opt > 0) {
2809 my $user_pattern_index = 0;
2810 foreach $key (@user_descriptions) {
2811 print_histogram($key, 'occurence', @{$user_pattern_interval_count[$user_pattern_index]});
2812 $user_pattern_index++;
2817 #######################################################################
2822 # Print statistics about rejected mail.
2823 #######################################################################
2825 my($format1,$reason);
2827 my $txt_format1 = " %-40s %6d";
2828 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%d</td>";
2831 print $txt_fh "Rejected mail by reason\n";
2832 print $txt_fh "-----------------------";
2833 print $txt_fh "\n Total\n";
2836 print $htm_fh "<hr><a name=\"patterns\"></a><h2>Rejected mail by reason</h2>\n";
2837 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
2838 print $htm_fh "<tr><th> </th><th>Total</th>\n";
2841 $ws_global->write($row++, $col, "Rejected mail by reason", $f_header2);
2842 &set_worksheet_line($ws_global, $row++, 1, ["Total"], $f_headertab);
2846 my $href = ($merge_reports) ? $report_totals{rejected_mail_by_reason} : \%rejected_count_by_reason;
2847 my(@chartdatanames, @chartdatavals_count);
2849 foreach $reason (top_n_sort($topcount, $href, undef, undef)) {
2850 printf $txt_fh ("$txt_format1\n",$reason,$href->{$reason}) if $txt_fh;
2851 printf $htm_fh ("$htm_format1\n",$reason,$href->{$reason}) if $htm_fh;
2852 set_worksheet_line($ws_global, $row++, 0, [$reason,$href->{$reason}], $f_default) if $xls_fh;
2853 push(@chartdatanames, $reason);
2854 push(@chartdatavals_count, $href->{$reason});
2858 print $txt_fh "\n" if $txt_fh;
2861 print $htm_fh "</tr></table></td><td>";
2862 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_count > 0)) {
2863 # calculate the graph
2866 \@chartdatavals_count
2868 my $graph = GD::Graph::pie->new(200, 200);
2870 x_label => 'Rejection Reasons',
2871 y_label => 'Messages',
2872 title => 'By count',
2874 my $gd = $graph->plot(\@data) or warn($graph->error);
2876 open(IMG, ">$chartdir/rejections_count.png") or die "Could not write $chartdir/rejections_count.png: $!\n";
2880 print $htm_fh "<img src=\"$chartrel/rejections_count.png\">";
2883 print $htm_fh "</td></tr></table>\n\n";
2891 #######################################################################
2892 # print_transport();
2894 # print_transport();
2896 # Print totals by transport.
2897 #######################################################################
2898 sub print_transport {
2899 my(@chartdatanames);
2900 my(@chartdatavals_count);
2901 my(@chartdatavals_vol);
2902 no integer; #Lose this for charting the data.
2904 my $txt_format1 = " %-18s %6s %6d";
2905 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%s</td><td align=\"right\">%d</td>";
2908 print $txt_fh "Deliveries by transport\n";
2909 print $txt_fh "-----------------------";
2910 print $txt_fh "\n Volume Messages\n";
2913 print $htm_fh "<hr><a name=\"Transport\"></a><h2>Deliveries by Transport</h2>\n";
2914 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
2915 print $htm_fh "<tr><th> </th><th>Volume</th><th>Messages</th>\n";
2918 $ws_global->write(++$row, $col, "Deliveries by transport", $f_header2);
2919 $ws_global->write(++$row, 1, ["Volume", "Messages"], $f_headertab);
2923 if ($merge_reports) {
2924 # We are getting our data from previous reports.
2925 foreach $key (sort keys %{$report_totals{transport}}) {
2926 my $count = get_report_total($report_totals{transport}{$key},'Messages');
2927 my @content=($key, volume_rounded($report_totals{transport}{$key}{Volume},
2928 $report_totals{transport}{$key}{'Volume-gigs'}), $count);
2929 push(@chartdatanames, $key);
2930 push(@chartdatavals_count, $count);
2931 push(@chartdatavals_vol, $report_totals{transport}{$key}{'Volume-gigs'}*$gig + $report_totals{transport}{$key}{Volume} );
2932 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2933 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2934 $ws_global->write(++$row, 0, \@content) if $xls_fh;
2938 # We are getting our data from mainlog files.
2939 foreach $key (sort keys %transported_data) {
2940 my @content=($key, volume_rounded($transported_data{$key},$transported_data_gigs{$key}),
2941 $transported_count{$key});
2942 push(@chartdatanames, $key);
2943 push(@chartdatavals_count, $transported_count{$key});
2944 push(@chartdatavals_vol, $transported_data_gigs{$key}*$gig + $transported_data{$key});
2945 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2946 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2947 $ws_global->write(++$row, 0, \@content) if $xls_fh;
2950 print $txt_fh "\n" if $txt_fh;
2952 print $htm_fh "</tr></table></td><td>";
2954 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_count > 0))
2956 # calculate the graph
2959 \@chartdatavals_count
2961 my $graph = GD::Graph::pie->new(200, 200);
2963 x_label => 'Transport',
2964 y_label => 'Messages',
2965 title => 'By count',
2967 my $gd = $graph->plot(\@data) or warn($graph->error);
2969 open(IMG, ">$chartdir/transports_count.png") or die "Could not write $chartdir/transports_count.png: $!\n";
2973 print $htm_fh "<img src=\"$chartrel/transports_count.png\">";
2976 print $htm_fh "</td><td>";
2978 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_vol > 0)) {
2983 my $graph = GD::Graph::pie->new(200, 200);
2985 title => 'By volume',
2987 my $gd = $graph->plot(\@data) or warn($graph->error);
2989 open(IMG, ">$chartdir/transports_vol.png") or die "Could not write $chartdir/transports_vol.png: $!\n";
2993 print $htm_fh "<img src=\"$chartrel/transports_vol.png\">";
2997 print $htm_fh "</td></tr></table>\n\n";
3003 #######################################################################
3008 # Print our totals by relay.
3009 #######################################################################
3011 my $row_print_relay=1;
3012 my $temp = "Relayed messages";
3013 print $htm_fh "<hr><a name=\"$temp\"></a><h2>$temp</h2>\n" if $htm_fh;
3014 if (scalar(keys %relayed) > 0 || $relayed_unshown > 0) {
3017 my $txt_format = "%7d %s\n => %s\n";
3018 my $htm_format = "<tr><td align=\"right\">%d</td><td>%s</td><td>%s</td>\n";
3020 printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
3022 print $htm_fh "<table border=1>\n";
3023 print $htm_fh "<tr><th>Count</th><th>From</th><th>To</th>\n";
3026 $ws_relayed->write($row_print_relay++, $col, $temp, $f_header2);
3027 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, ["Count", "From", "To"], $f_headertab);
3032 foreach $key (sort keys %relayed) {
3033 my $count = $relayed{$key};
3036 my($one,$two) = split(/=> /, $key);
3037 my @content=($count, $one, $two);
3038 printf $txt_fh ($txt_format, @content) if $txt_fh;
3039 printf $htm_fh ($htm_format, @content) if $htm_fh;
3042 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, \@content);
3047 print $htm_fh "</table>\n<p>\n" if $htm_fh;
3048 print $txt_fh "${spacing}Total: $shown (plus $relayed_unshown unshown)\n\n" if $txt_fh;
3049 print $htm_fh "${spacing}Total: $shown (plus $relayed_unshown unshown)\n\n" if $htm_fh;
3052 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, [$shown, "Sum of shown" ]);
3053 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, [$relayed_unshown, "unshown"]);
3058 print $txt_fh "No relayed messages\n-------------------\n\n" if $txt_fh;
3059 print $htm_fh "No relayed messages\n\n" if $htm_fh;
3069 #######################################################################
3074 # Print our errors. In HTML, we display them as a list rather than a table -
3075 # Netscape doesn't like large tables!
3076 #######################################################################
3078 my $total_errors = 0;
3081 if (scalar(keys %errors_count) != 0) {
3082 my $temp = "List of errors";
3083 my $htm_format = "<li>%d - %s\n";
3085 printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
3087 print $htm_fh "<hr><a name=\"errors\"></a><h2>$temp</h2>\n";
3088 print $htm_fh "<ul><li><b>Count - Error</b>\n";
3092 $ws_errors->write($row++, 0, $temp, $f_header2);
3093 &set_worksheet_line($ws_errors, $row++, 0, ["Count", "Error"], $f_headertab);
3098 foreach $key (sort keys %errors_count) {
3101 $text =~ s/\s\s+/ /g; #Convert multiple spaces to a single space.
3102 $total_errors += $errors_count{$key};
3105 printf $txt_fh ("%5d ", $errors_count{$key});
3106 my $text_remaining = $text;
3107 while (length($text_remaining) > 65) {
3108 my($first,$rest) = $text_remaining =~ /(.{50}\S*)\s+(.+)/;
3110 printf $txt_fh ("%s\n\t ", $first);
3111 $text_remaining = $rest;
3113 printf $txt_fh ("%s\n\n", $text_remaining);
3118 #Translate HTML tag characters. Sergey Sholokh.
3119 $text =~ s/\</\<\;/g;
3120 $text =~ s/\>/\>\;/g;
3122 printf $htm_fh ($htm_format,$errors_count{$key},$text);
3126 &set_worksheet_line($ws_errors, $row++, 0, [$errors_count{$key},$text]);
3130 $temp = "Errors encountered: $total_errors";
3133 print $txt_fh $temp, "\n";
3134 print $txt_fh "-" x length($temp),"\n";
3137 print $htm_fh "</ul>\n<p>\n";
3138 print $htm_fh $temp, "\n";
3142 &set_worksheet_line($ws_errors, $row++, 0, [$total_errors, "Sum of Errors encountered"]);
3149 #######################################################################
3150 # parse_old_eximstat_reports();
3152 # parse_old_eximstat_reports($fh);
3154 # Parse old eximstat output so we can merge daily stats to weekly stats and weekly to monthly etc.
3156 # To test that the merging still works after changes, do something like the following.
3157 # All the diffs should produce no output.
3159 # options='-bydomain -byemail -byhost -byedomain'
3160 # options="$options -show_rt1,2,4 -show_dt 1,2,4"
3161 # options="$options -pattern 'Completed Messages' /Completed/"
3162 # options="$options -pattern 'Received Messages' /<=/"
3164 # ./eximstats $options mainlog > mainlog.txt
3165 # ./eximstats $options -merge mainlog.txt > mainlog.2.txt
3166 # diff mainlog.txt mainlog.2.txt
3168 # ./eximstats $options -html mainlog > mainlog.html
3169 # ./eximstats $options -merge -html mainlog.txt > mainlog.2.html
3170 # diff mainlog.html mainlog.2.html
3172 # ./eximstats $options -merge mainlog.html > mainlog.3.txt
3173 # diff mainlog.txt mainlog.3.txt
3175 # ./eximstats $options -merge -html mainlog.html > mainlog.3.html
3176 # diff mainlog.html mainlog.3.html
3178 # ./eximstats $options -nvr mainlog > mainlog.nvr.txt
3179 # ./eximstats $options -merge mainlog.nvr.txt > mainlog.4.txt
3180 # diff mainlog.txt mainlog.4.txt
3182 # # double_mainlog.txt should have twice the values that mainlog.txt has.
3183 # ./eximstats $options mainlog mainlog > double_mainlog.txt
3184 #######################################################################
3185 sub parse_old_eximstat_reports {
3188 my(%league_table_value_entered, %league_table_value_was_zero, %table_order);
3190 my(%user_pattern_index);
3191 my $user_pattern_index = 0;
3192 map {$user_pattern_index{$_} = $user_pattern_index++} @user_descriptions;
3193 my $user_pattern_keys = join('|', @user_descriptions);
3196 PARSE_OLD_REPORT_LINE:
3197 if (/Exim statistics from ([\d\-]+ [\d:]+(\s+[\+\-]\d+)?) to ([\d\-]+ [\d:]+(\s+[\+\-]\d+)?)/) {
3198 $begin = $1 if ($1 lt $begin);
3199 $end = $3 if ($3 gt $end);
3201 elsif (/Grand total summary/) {
3202 # Fill in $report_totals{Received|Delivered}{Volume|Messages|Addresses|Hosts|Domains|...|Delayed|DelayedPercent|Failed|FailedPercent}
3203 my(@fields, @delivered_fields);
3204 my $doing_table = 0;
3206 $_ = html2txt($_); #Convert general HTML markup to text.
3207 s/At least one addr//g; #Another part of the HTML output we don't want.
3209 # TOTAL Volume Messages Addresses Hosts Domains Delayed Failed
3210 # Received 26MB 237 177 23 8 3.4% 28 11.8%
3211 # Delivered 13MB 233 250 99 88
3212 if (/TOTAL\s+(.*?)\s*$/) {
3214 @delivered_fields = split(/\s+/,$1);
3216 #Delayed and Failed have two columns each, so add the extra field names in.
3217 splice(@delivered_fields,-1,1,'DelayedPercent','Failed','FailedPercent');
3219 # Addresses only figure in the Delivered row, so remove them from the
3221 @fields = grep !/Addresses/, @delivered_fields;
3223 elsif (/(Received)\s+(.*?)\s*$/) {
3224 print STDERR "Parsing $_" if $debug;
3225 add_to_totals($report_totals{$1},\@fields,$2);
3227 elsif (/(Delivered)\s+(.*?)\s*$/) {
3228 print STDERR "Parsing $_" if $debug;
3229 add_to_totals($report_totals{$1},\@delivered_fields,$2);
3231 # If we're merging an old report which doesn't include addresses,
3232 # then use the Messages field instead.
3233 unless (grep(/Addresses/, @delivered_fields)) {
3235 line_to_hash(\%tmp,\@delivered_fields,$data);
3236 add_to_totals($report_totals{Delivered},['Addresses'],$tmp{Messages});
3239 elsif (/(Temp Rejects|Rejects|Ham|Spam)\s+(.*?)\s*$/) {
3240 print STDERR "Parsing $_" if $debug;
3241 add_to_totals($report_totals{$1},['Messages','Hosts'],$2);
3244 last if $doing_table;
3249 elsif (/User Specified Patterns/i) {
3250 #User Specified Patterns
3251 #-----------------------
3255 while (<$fh>) { last if (/Total/); } #Wait until we get the table headers.
3257 print STDERR "Parsing $_" if $debug;
3258 $_ = html2txt($_); #Convert general HTML markup to text.
3259 if (/^\s*(.*?)\s+(\d+)\s*$/) {
3260 $report_totals{patterns}{$1} = {} unless (defined $report_totals{patterns}{$1});
3261 add_to_totals($report_totals{patterns}{$1},['Total'],$2);
3263 last if (/^\s*$/); #Finished if we have a blank line.
3267 elsif (/(^|<h2>)($user_pattern_keys) per /o) {
3268 # Parse User defined pattern histograms if they exist.
3269 parse_histogram($fh, $user_pattern_interval_count[$user_pattern_index{$2}] );
3273 elsif (/Deliveries by transport/i) {
3274 #Deliveries by transport
3275 #-----------------------
3277 # :blackhole: 70KB 51
3278 # address_pipe 655KB 1
3281 while (<$fh>) { last if (/Volume/); } #Wait until we get the table headers.
3283 print STDERR "Parsing $_" if $debug;
3284 $_ = html2txt($_); #Convert general HTML markup to text.
3285 if (/(\S+)\s+(\d+\S*\s+\d+)/) {
3286 $report_totals{transport}{$1} = {} unless (defined $report_totals{transport}{$1});
3287 add_to_totals($report_totals{transport}{$1},['Volume','Messages'],$2);
3289 last if (/^\s*$/); #Finished if we have a blank line.
3292 elsif (/Messages received per/) {
3293 parse_histogram($fh, \@received_interval_count);
3295 elsif (/Deliveries per/) {
3296 parse_histogram($fh, \@delivered_interval_count);
3299 #elsif (/Time spent on the queue: (all messages|messages with at least one remote delivery)/) {
3300 elsif (/(Time spent on the queue|Delivery times|Receipt times): ((\S+) messages|messages with at least one remote delivery)((<[^>]*>)*\s*)$/) {
3301 #Time spent on the queue: all messages
3302 #-------------------------------------
3304 #Under 1m 217 91.9% 91.9%
3310 # Set a pointer to the queue bin so we can use the same code
3311 # block for both all messages and remote deliveries.
3312 #my $bin_aref = ($1 eq 'all messages') ? \@qt_all_bin : \@qt_remote_bin;
3313 my($bin_aref, $times_aref, $overflow_sref);
3314 if ($1 eq 'Time spent on the queue') {
3315 $times_aref = \@queue_times;
3316 if ($2 eq 'all messages') {
3317 $bin_aref = \@qt_all_bin;
3318 $overflow_sref = \$qt_all_overflow;
3321 $bin_aref = \@qt_remote_bin;
3322 $overflow_sref = \$qt_remote_overflow;
3325 elsif ($1 eq 'Delivery times') {
3326 $times_aref = \@delivery_times;
3327 if ($2 eq 'all messages') {
3328 $bin_aref = \@dt_all_bin;
3329 $overflow_sref = \$dt_all_overflow;
3332 $bin_aref = \@dt_remote_bin;
3333 $overflow_sref = \$dt_remote_overflow;
3337 unless (exists $rcpt_times_bin{$3}) {
3338 initialise_rcpt_times($3);
3340 $bin_aref = $rcpt_times_bin{$3};
3341 $times_aref = \@rcpt_times;
3342 $overflow_sref = \$rcpt_times_overflow{$3};
3346 my ($blank_lines, $reached_table) = (0,0);
3348 $_ = html2txt($_); #Convert general HTML markup to text.
3349 # The table is preceded by one blank line, and has one blank line
3350 # following it. As the table may be empty, the best way to determine
3351 # that we've finished it is to look for the second blank line.
3352 ++$blank_lines if /^\s*$/;
3353 last if ($blank_lines >=2); #Finished the table ?
3354 $reached_table = 1 if (/\d/);
3355 next unless $reached_table;
3356 my $previous_seconds_on_queue = 0;
3357 if (/^\s*(Under|Over|)\s+(\d+[smhdw])\s+(\d+)/) {
3358 print STDERR "Parsing $_" if $debug;
3359 my($modifier,$formated_time,$count) = ($1,$2,$3);
3360 my $seconds = unformat_time($formated_time);
3361 my $time_on_queue = ($seconds + $previous_seconds_on_queue) / 2;
3362 $previous_seconds_on_queue = $seconds;
3363 $time_on_queue = $seconds * 2 if ($modifier eq 'Over');
3365 for ($i = 0; $i <= $#$times_aref; $i++) {
3366 if ($time_on_queue < $times_aref->[$i]) {
3367 $$bin_aref[$i] += $count;
3371 $$overflow_sref += $count if ($i > $#$times_aref);
3377 elsif (/Relayed messages/) {
3381 # 1 addr.domain.com [1.2.3.4] a.user@domain.com
3382 # => addr2.domain2.com [5.6.7.8] a2.user2@domain2.com
3384 #<tr><td align="right">1</td><td>addr.domain.com [1.2.3.4] a.user@domain.com </td><td>addr2.domain2.com [5.6.7.8] a2.user2@domain2.com</td>
3386 my $reached_table = 0;
3389 unless ($reached_table) {
3390 last if (/No relayed messages/);
3391 $reached_table = 1 if (/^\s*\d/ || />\d+</);
3392 next unless $reached_table;
3394 if (/>(\d+)<.td><td>(.*?) ?<.td><td>(.*?)</) {
3395 update_relayed($1,$2,$3);
3397 elsif (/^\s*(\d+)\s+(.*?)\s*$/) {
3398 ($count,$sender) = ($1,$2);
3400 elsif (/=>\s+(.*?)\s*$/) {
3401 update_relayed($count,$sender,$1);
3404 last; #Finished the table ?
3409 elsif (/Top (.*?) by (message count|volume)/) {
3410 #Top 50 sending hosts by message count
3411 #-------------------------------------
3414 # Could also have average values for HTML output.
3415 # 48 1468KB 30KB local
3417 my($category,$by_count_or_volume) = ($1,$2);
3419 #As we show 2 views of each table (by count and by volume),
3420 #most (but not all) entries will appear in both tables.
3421 #Set up a hash to record which entries we have already seen
3422 #and one to record which ones we are seeing for the first time.
3423 if ($by_count_or_volume =~ /count/) {
3424 undef %league_table_value_entered;
3425 undef %league_table_value_was_zero;
3429 #As this section processes multiple different table categories,
3430 #set up pointers to the hashes to be updated.
3431 my($messages_href,$addresses_href,$data_href,$data_gigs_href);
3432 if ($category =~ /local sender/) {
3433 $messages_href = \%received_count_user;
3434 $addresses_href = undef;
3435 $data_href = \%received_data_user;
3436 $data_gigs_href = \%received_data_gigs_user;
3438 elsif ($category =~ /sending (\S+?)s?\b/) {
3439 #Top 50 sending (host|domain|email|edomain)s
3440 #Top sending (host|domain|email|edomain)
3441 $messages_href = \%{$received_count{"\u$1"}};
3442 $data_href = \%{$received_data{"\u$1"}};
3443 $data_gigs_href = \%{$received_data_gigs{"\u$1"}};
3445 elsif ($category =~ /local destination/) {
3446 $messages_href = \%delivered_messages_user;
3447 $addresses_href = \%delivered_addresses_user;
3448 $data_href = \%delivered_data_user;
3449 $data_gigs_href = \%delivered_data_gigs_user;
3451 elsif ($category =~ /local domain destination/) {
3452 $messages_href = \%delivered_messages_local_domain;
3453 $addresses_href = \%delivered_addresses_local_domain;
3454 $data_href = \%delivered_data_local_domain;
3455 $data_gigs_href = \%delivered_data_gigs_local_domain;
3457 elsif ($category =~ /(\S+) destination/) {
3458 #Top 50 (host|domain|email|edomain) destinations
3459 #Top (host|domain|email|edomain) destination
3460 $messages_href = \%{$delivered_messages{"\u$1"}};
3461 $addresses_href = \%{$delivered_addresses{"\u$1"}};
3462 $data_href = \%{$delivered_data{"\u$1"}};
3463 $data_gigs_href = \%{$delivered_data_gigs{"\u$1"}};
3465 elsif ($category =~ /temporarily rejected ips/) {
3466 $messages_href = \%temporarily_rejected_count_by_ip;
3468 elsif ($category =~ /rejected ips/) {
3469 $messages_href = \%rejected_count_by_ip;
3471 elsif ($category =~ /non-rejected spamming ips/) {
3472 $messages_href = \%spam_count_by_ip;
3474 elsif ($category =~ /mail temporary rejection reasons/) {
3475 $messages_href = \%temporarily_rejected_count_by_reason;
3477 elsif ($category =~ /mail rejection reasons/) {
3478 $messages_href = \%rejected_count_by_reason;
3481 my $reached_table = 0;
3484 # Watch out for empty tables.
3485 goto PARSE_OLD_REPORT_LINE if (/<h2>/ or (/^\s*[a-zA-Z]/ && !/^\s*Messages/));
3487 $_ = html2txt($_); #Convert general HTML markup to text.
3489 # Messages Addresses Bytes Average
3490 if (/^\s*Messages/) {
3491 my $pattern = '^\s*(\d+)';
3492 $pattern .= (/Addresses/) ? '\s+(\d+)' : '()';
3493 $pattern .= (/Bytes/) ? '\s+([\dKMGB]+)' : '()';
3494 $pattern .= (/Average/) ? '\s+[\dKMGB]+' : '';
3495 $pattern .= '\s+(.*?)\s*$';
3496 $row_re = qr/$pattern/;
3500 next unless $reached_table;
3502 my($messages, $addresses, $rounded_volume, $entry);
3505 ($messages, $addresses, $rounded_volume, $entry) = ($1, $2, $3, $4);
3508 #Else we have finished the table and we may need to do some
3509 #kludging to retain the order of the entries.
3511 if ($by_count_or_volume =~ /volume/) {
3512 #Add a few bytes to appropriate entries to preserve the order.
3513 foreach $rounded_volume (keys %table_order) {
3514 #For each rounded volume, we want to create a list which has things
3515 #ordered from the volume table at the front, and additional things
3516 #from the count table ordered at the back.
3517 @{$table_order{$rounded_volume}{volume}} = () unless defined $table_order{$rounded_volume}{volume};
3518 @{$table_order{$rounded_volume}{'message count'}} = () unless defined $table_order{$rounded_volume}{'message count'};
3520 map {$mark{$_} = 1} @{$table_order{$rounded_volume}{volume}};
3521 @order = @{$table_order{$rounded_volume}{volume}};
3522 map {push(@order,$_)} grep(!$mark{$_},@{$table_order{$rounded_volume}{'message count'}});
3524 my $bonus_bytes = $#order;
3525 $bonus_bytes = 511 if ($bonus_bytes > 511); #Don't go over the half-K boundary!
3526 while (@order and ($bonus_bytes > 0)) {
3527 my $entry = shift(@order);
3528 if ($league_table_value_was_zero{$entry}) {
3529 $$data_href{$entry} += $bonus_bytes;
3530 print STDERR "$category by $by_count_or_volume: added $bonus_bytes bonus bytes to $entry\n" if $debug;
3539 # Store a new table entry.
3541 # Add the entry into the %table_order hash if it has a rounded
3542 # volume (KB/MB/GB).
3543 push(@{$table_order{$rounded_volume}{$by_count_or_volume}},$entry) if ($rounded_volume =~ /\D/);
3545 unless ($league_table_value_entered{$entry}) {
3546 $league_table_value_entered{$entry} = 1;
3547 unless ($$messages_href{$entry}) {
3548 $$messages_href{$entry} = 0;
3549 $$addresses_href{$entry} = 0;
3550 $$data_href{$entry} = 0;
3551 $$data_gigs_href{$entry} = 0;
3552 $league_table_value_was_zero{$entry} = 1;
3555 $$messages_href{$entry} += $messages;
3557 # When adding the addresses, be aware that we could be merging
3558 # an old report which does not include addresses. In this case,
3559 # we add the messages instead.
3560 $$addresses_href{$entry} += ($addresses) ? $addresses : $messages;
3562 #Add the rounded value to the data and data_gigs hashes.
3563 un_round($rounded_volume,\$$data_href{$entry},\$$data_gigs_href{$entry}) if $rounded_volume;
3564 print STDERR "$category by $by_count_or_volume: added $messages,$rounded_volume to $entry\n" if $debug;
3569 elsif (/List of errors/) {
3573 # 1 07904931641@one2one.net R=external T=smtp: SMTP error
3574 # from remote mailer after RCPT TO:<07904931641@one2one.net>:
3575 # host mail.one2one.net [193.133.192.24]: 550 User unknown
3577 #<li>1 - ally.dufc@dunbar.org.uk R=external T=smtp: SMTP error from remote mailer after RCPT TO:<ally.dufc@dunbar.org.uk>: host mail.dunbar.org.uk [216.167.89.88]: 550 Unknown local part ally.dufc in <ally.dufc@dunbar.org.uk>
3580 my $reached_table = 0;
3581 my($count,$error,$blanks);
3583 $reached_table = 1 if (/^( *|<li>)(\d+)/);
3584 next unless $reached_table;
3586 s/^<li>(\d+) -/$1/; #Convert an HTML line to a text line.
3587 $_ = html2txt($_); #Convert general HTML markup to text.
3590 $error .= ' ' . $1; #Join a multiline error.
3592 elsif (/^\s*(\d+)\s+(.*)/) {
3594 #Finished with a previous multiline error so save it.
3595 $errors_count{$error} = 0 unless $errors_count{$error};
3596 $errors_count{$error} += $count;
3598 ($count,$error) = ($1,$2);
3600 elsif (/Errors encountered/) {
3602 #Finished the section, so save our stored last error.
3603 $errors_count{$error} = 0 unless $errors_count{$error};
3604 $errors_count{$error} += $count;
3614 #######################################################################
3615 # parse_histogram($fh, \@delivered_interval_count);
3616 # Parse a histogram into the provided array of counters.
3617 #######################################################################
3618 sub parse_histogram {
3619 my($fh, $counters_aref) = @_;
3621 # Messages received per hour (each dot is 2 messages)
3622 #---------------------------------------------------
3624 #00-01 106 .....................................................
3625 #01-02 103 ...................................................
3627 my $reached_table = 0;
3629 $reached_table = 1 if (/^00/);
3630 next unless $reached_table;
3631 print STDERR "Parsing $_" if $debug;
3632 if (/^(\d+):(\d+)\s+(\d+)/) { #hh:mm start time format ?
3633 $$counters_aref[($1*60 + $2)/$hist_interval] += $3 if $hist_opt;
3635 elsif (/^(\d+)-(\d+)\s+(\d+)/) { #hh-hh start-end time format ?
3636 $$counters_aref[($1*60)/$hist_interval] += $3 if $hist_opt;
3638 else { #Finished the table ?
3645 #######################################################################
3648 # update_relayed($count,$sender,$recipient);
3650 # Adds an entry into the %relayed hash. Currently only used when
3652 #######################################################################
3653 sub update_relayed {
3654 my($count,$sender,$recipient) = @_;
3656 #When generating the key, put in the 'H=' and 'A=' which can be used
3658 my $key = "H=$sender => H=$recipient";
3659 $key =~ s/ ([^=\s]+\@\S+|<>)/ A=$1/g;
3660 if (!defined $relay_pattern || $key !~ /$relay_pattern/o) {
3661 $relayed{$key} = 0 if !defined $relayed{$key};
3662 $relayed{$key} += $count;
3665 $relayed_unshown += $count;
3670 #######################################################################
3673 # add_to_totals(\%totals,\@keys,$values);
3675 # Given a line of space seperated values, add them into the provided hash using @keys
3678 # If the value contains a '%', then the value is set rather than added. Otherwise, we
3679 # convert the value to bytes and gigs. The gigs get added to I<Key>-gigs.
3680 #######################################################################
3682 my($totals_href,$keys_aref,$values) = @_;
3683 my(@values) = split(/\s+/,$values);
3685 for(my $i = 0; $i < @values && $i < @$keys_aref; ++$i) {
3686 my $key = $keys_aref->[$i];
3687 if ($values[$i] =~ /%/) {
3688 $$totals_href{$key} = $values[$i];
3691 $$totals_href{$key} = 0 unless ($$totals_href{$key});
3692 $$totals_href{"$key-gigs"} = 0 unless ($$totals_href{"$key-gigs"});
3693 un_round($values[$i], \$$totals_href{$key}, \$$totals_href{"$key-gigs"});
3694 print STDERR "Added $values[$i] to $key - $$totals_href{$key} , " . $$totals_href{"$key-gigs"} . "GB.\n" if $debug;
3700 #######################################################################
3703 # line_to_hash(\%hash,\@keys,$line);
3705 # Given a line of space seperated values, set them into the provided hash
3706 # using @keys as the hash keys.
3707 #######################################################################
3709 my($href,$keys_aref,$values) = @_;
3710 my(@values) = split(/\s+/,$values);
3711 for(my $i = 0; $i < @values && $i < @$keys_aref; ++$i) {
3712 $$href{$keys_aref->[$i]} = $values[$i];
3717 #######################################################################
3718 # get_report_total();
3720 # $total = get_report_total(\%hash,$key);
3722 # If %hash contains values split into Units and Gigs, we calculate and return
3724 # $hash{$key} + 1024*1024*1024 * $hash{"${key}-gigs"}
3725 #######################################################################
3726 sub get_report_total {
3728 my($hash_ref,$key) = @_;
3729 if ($$hash_ref{"${key}-gigs"}) {
3730 return $$hash_ref{$key} + $gig * $$hash_ref{"${key}-gigs"};
3732 return $$hash_ref{$key} || 0;
3735 #######################################################################
3738 # $text_line = html2txt($html_line);
3740 # Convert a line from html to text. Currently we just convert HTML tags to spaces
3741 # and convert >, <, and tags back.
3742 #######################################################################
3746 # Convert HTML tags to spacing. Note that the reports may contain <Userid> and
3747 # <Userid@Domain> words, so explicitly specify the HTML tags we will remove
3748 # (the ones used by this program). If someone is careless enough to have their
3749 # Userid the same as an HTML tag, there's not much we can do about it.
3750 s/<\/?(html|head|title|body|h\d|ul|li|a\s+|table|tr|td|th|pre|hr|p|br)\b.*?>/ /g;
3752 s/\<\;/\</og; #Convert '<' to '<'.
3753 s/\>\;/\>/og; #Convert '>' to '>'.
3754 s/\ \;/ /og; #Convert ' ' to ' '.
3758 #######################################################################
3761 # $arg = get_next_arg();
3763 # Because eximstats arguments are often passed as variables,
3764 # we can't rely on shell parsing to deal with quotes. This
3765 # subroutine returns $ARGV[1] and does a shift. If $ARGV[1]
3766 # starts with a quote (' or "), and doesn't end in one, then
3767 # we append the next argument to it and shift again. We repeat
3768 # until we've got all of the argument.
3770 # This isn't perfect as all white space gets reduced to one space,
3771 # but it's as good as we can get! If it's esential that spacing
3772 # be preserved precisely, then you get that by not using shell
3774 #######################################################################
3777 my $matched_pattern = 0;
3779 $arg .= ' ' if $arg;
3780 $arg .= $ARGV[1]; shift(@ARGV);
3781 if ($arg !~ /^['"]/) {
3782 $matched_pattern = 1;
3785 if ($arg =~ s/^(['"])(.*)\1$/$2/) {
3786 $matched_pattern = 1;
3790 die "Mismatched argument quotes - <$arg>.\n" unless $matched_pattern;
3794 #######################################################################
3795 # set_worksheet_line($ws_global, $startrow, $startcol, \@content, $format);
3797 # set values to a sequence of cells in a row.
3799 #######################################################################
3800 sub set_worksheet_line {
3801 my ($worksheet, $row, $col, $content, $format) = @_;
3803 foreach my $token (@$content)
3805 $worksheet->write($row, $col++, $token, $format );
3810 #######################################################################
3811 # @rcpt_times = parse_time_list($string);
3813 # Parse a comma seperated list of time values in seconds given by
3814 # the user and fill an array.
3816 # Return a default list if $string is undefined.
3817 # Return () if $string eq '0'.
3818 #######################################################################
3819 sub parse_time_list {
3821 if (! defined $string) {
3822 return(60, 5*60, 15*60, 30*60, 60*60, 3*60*60, 6*60*60, 12*60*60, 24*60*60);
3824 my(@times) = split(/,/, $string);
3825 foreach my $q (@times) { $q = eval($q) + 0 }
3826 @times = sort { $a <=> $b } @times;
3827 @times = () if ($#times == 0 && $times[0] == 0);
3832 #######################################################################
3833 # initialise_rcpt_times($protocol);
3834 # Initialise an array of rcpt_times to 0 for the specified protocol.
3835 #######################################################################
3836 sub initialise_rcpt_times {
3838 for (my $i = 0; $i <= $#rcpt_times; ++$i) {
3839 $rcpt_times_bin{$protocol}[$i] = 0;
3841 $rcpt_times_overflow{$protocol} = 0;
3845 ##################################################
3847 ##################################################
3850 $last_timestamp = '';
3854 $show_transport = 1;
3856 $local_league_table = 1;
3857 $include_remote_users = 0;
3858 $include_original_destination = 0;
3860 $volume_rounding = 1;
3861 $localtime_offset = calculate_localtime_offset(); # PH/FANF
3864 $charts_option_specified = 0;
3868 @queue_times = parse_time_list();
3870 @delivery_times = ();
3873 $offset_seconds = 0;
3879 my(%output_files); # What output files have been specified?
3883 while (@ARGV > 0 && substr($ARGV[0], 0, 1) eq '-') {
3884 if ($ARGV[0] =~ /^\-h(\d+)$/) { $hist_opt = $1 }
3885 elsif ($ARGV[0] =~ /^\-ne$/) { $show_errors = 0 }
3886 elsif ($ARGV[0] =~ /^\-nr(.?)(.*)\1$/) {
3887 if ($1 eq "") { $show_relay = 0 } else { $relay_pattern = $2 }
3889 elsif ($ARGV[0] =~ /^\-q([,\d\+\-\*\/]+)$/) { @queue_times = parse_time_list($1) }
3890 elsif ($ARGV[0] =~ /^-nt$/) { $show_transport = 0 }
3891 elsif ($ARGV[0] =~ /^\-nt(.?)(.*)\1$/)
3893 if ($1 eq "") { $show_transport = 0 } else { $transport_pattern = $2 }
3895 elsif ($ARGV[0] =~ /^-t(\d+)$/) { $topcount = $1 }
3896 elsif ($ARGV[0] =~ /^-tnl$/) { $local_league_table = 0 }
3897 elsif ($ARGV[0] =~ /^-txt=?(\S*)$/) { $txt_fh = get_filehandle($1,\%output_files) }
3898 elsif ($ARGV[0] =~ /^-html=?(\S*)$/) { $htm_fh = get_filehandle($1,\%output_files) }
3899 elsif ($ARGV[0] =~ /^-xls=?(\S*)$/) {
3900 if ($HAVE_Spreadsheet_WriteExcel) {
3901 $xls_fh = get_filehandle($1,\%output_files);
3904 warn "WARNING: CPAN Module Spreadsheet::WriteExcel not installed. Obtain from www.cpan.org\n";
3907 elsif ($ARGV[0] =~ /^-merge$/) { $merge_reports = 1 }
3908 elsif ($ARGV[0] =~ /^-charts$/) {
3910 warn "WARNING: CPAN Module GD::Graph::pie not installed. Obtain from www.cpan.org\n" unless $HAVE_GD_Graph_pie;
3911 warn "WARNING: CPAN Module GD::Graph::linespoints not installed. Obtain from www.cpan.org\n" unless $HAVE_GD_Graph_linespoints;
3913 elsif ($ARGV[0] =~ /^-chartdir$/) { $chartdir = $ARGV[1]; shift; $charts_option_specified = 1; }
3914 elsif ($ARGV[0] =~ /^-chartrel$/) { $chartrel = $ARGV[1]; shift; $charts_option_specified = 1; }
3915 elsif ($ARGV[0] =~ /^-include_original_destination$/) { $include_original_destination = 1 }
3916 elsif ($ARGV[0] =~ /^-cache$/) { } #Not currently used.
3917 elsif ($ARGV[0] =~ /^-byhost$/) { $do_sender{Host} = 1 }
3918 elsif ($ARGV[0] =~ /^-bydomain$/) { $do_sender{Domain} = 1 }
3919 elsif ($ARGV[0] =~ /^-byemail$/) { $do_sender{Email} = 1 }
3920 elsif ($ARGV[0] =~ /^-byemaildomain$/) { $do_sender{Edomain} = 1 }
3921 elsif ($ARGV[0] =~ /^-byedomain$/) { $do_sender{Edomain} = 1 }
3922 elsif ($ARGV[0] =~ /^-bylocaldomain$/) { $do_local_domain = 1 }
3923 elsif ($ARGV[0] =~ /^-emptyok$/) { $emptyOK = 1 }
3924 elsif ($ARGV[0] =~ /^-nvr$/) { $volume_rounding = 0 }
3925 elsif ($ARGV[0] =~ /^-show_rt([,\d\+\-\*\/]+)?$/) { @rcpt_times = parse_time_list($1) }
3926 elsif ($ARGV[0] =~ /^-show_dt([,\d\+\-\*\/]+)?$/) { @delivery_times = parse_time_list($1) }
3927 elsif ($ARGV[0] =~ /^-d$/) { $debug = 1 }
3928 elsif ($ARGV[0] =~ /^--?h(elp)?$/){ help() }
3929 elsif ($ARGV[0] =~ /^-t_remote_users$/) { $include_remote_users = 1 }
3930 elsif ($ARGV[0] =~ /^-pattern$/)
3932 push(@user_descriptions,get_next_arg());
3933 push(@user_patterns,get_next_arg());
3935 elsif ($ARGV[0] =~ /^-utc$/)
3937 # We don't need this value if the log is in UTC.
3938 $localtime_offset = undef;
3942 print STDERR "Eximstats: Unknown or malformed option $ARGV[0]\n";
3948 # keep old default behaviour
3949 if (! ($xls_fh or $htm_fh or $txt_fh)) {
3953 # Check that all the charts options are specified.
3954 warn "-charts option not specified. Use -help for help.\n" if ($charts_option_specified && ! $charts);
3956 # Default to display tables by sending Host.
3957 $do_sender{Host} = 1 unless ($do_sender{Domain} || $do_sender{Email} || $do_sender{Edomain});
3959 # prepare xls Excel Workbook
3960 if (defined $xls_fh) {
3962 # Create a new Excel workbook
3963 $workbook = Spreadsheet::WriteExcel->new($xls_fh);
3966 $ws_global = $workbook->addworksheet('Exim Statistik');
3967 # show $ws_global as initial sheet
3968 $ws_global->set_first_sheet();
3969 $ws_global->activate();
3972 $ws_relayed = $workbook->addworksheet('Relayed Messages');
3973 $ws_relayed->set_column(1, 2, 80);
3976 $ws_errors = $workbook->addworksheet('Errors');
3981 $ws_global->set_column(0, 2, 20); # Columns B-D width set to 30
3982 $ws_global->set_column(3, 3, 15); # Columns B-D width set to 30
3983 $ws_global->set_column(4, 4, 25); # Columns B-D width set to 30
3986 $f_default = $workbook->add_format();
3988 $f_header1 = $workbook->add_format();
3989 $f_header1->set_bold();
3990 #$f_header1->set_color('red');
3991 $f_header1->set_size('15');
3992 $f_header1->set_valign();
3993 # $f_header1->set_align('center');
3994 # $ws_global->write($row++, 2, "Testing Headers 1", $f_header1);
3996 $f_header2 = $workbook->add_format();
3997 $f_header2->set_bold();
3998 $f_header2->set_size('12');
3999 $f_header2->set_valign();
4000 # $ws_global->write($row++, 2, "Testing Headers 2", $f_header2);
4002 # Create another header2 for use in merged cells.
4003 $f_header2_m = $workbook->add_format();
4004 $f_header2_m->set_bold();
4005 $f_header2_m->set_size('8');
4006 $f_header2_m->set_valign();
4007 $f_header2_m->set_align('center');
4009 $f_percent = $workbook->add_format();
4010 $f_percent->set_num_format('0.0%');
4012 $f_headertab = $workbook->add_format();
4013 $f_headertab->set_bold();
4014 $f_headertab->set_valign();
4015 # $ws_global->write($row++, 2, "Testing Headers tab", $f_headertab);
4020 # Initialise the queue/delivery/rcpt time counters.
4021 for (my $i = 0; $i <= $#queue_times; $i++) {
4022 $qt_all_bin[$i] = 0;
4023 $qt_remote_bin[$i] = 0;
4025 for (my $i = 0; $i <= $#delivery_times; $i++) {
4026 $dt_all_bin[$i] = 0;
4027 $dt_remote_bin[$i] = 0;
4029 initialise_rcpt_times('all');
4032 # Compute the number of slots for the histogram
4035 if ($hist_opt > 60 || 60 % $hist_opt != 0)
4037 print STDERR "Eximstats: -h must specify a factor of 60\n";
4040 $hist_interval = 60/$hist_opt; #Interval in minutes.
4041 $hist_number = (24*60)/$hist_interval; #Number of intervals per day.
4042 @received_interval_count = (0) x $hist_number;
4043 @delivered_interval_count = (0) x $hist_number;
4044 my $user_pattern_index = 0;
4045 for (my $user_pattern_index = 0; $user_pattern_index <= $#user_patterns; ++$user_pattern_index) {
4046 @{$user_pattern_interval_count[$user_pattern_index]} = (0) x $hist_number;
4048 @dt_all_bin = (0) x $hist_number;
4049 @dt_remote_bin = (0) x $hist_number;
4052 #$queue_unknown = 0;
4054 $total_received_data = 0;
4055 $total_received_data_gigs = 0;
4056 $total_received_count = 0;
4058 $total_delivered_data = 0;
4059 $total_delivered_data_gigs = 0;
4060 $total_delivered_messages = 0;
4061 $total_delivered_addresses = 0;
4063 $qt_all_overflow = 0;
4064 $qt_remote_overflow = 0;
4065 $dt_all_overflow = 0;
4066 $dt_remote_overflow = 0;
4068 $relayed_unshown = 0;
4069 $message_errors = 0;
4070 $begin = "9999-99-99 99:99:99";
4071 $end = "0000-00-00 00:00:00";
4073 foreach $section ('Received','Delivered','Temp Rejects', 'Rejects','Ham','Spam') {
4074 foreach $type ('Volume','Messages','Delayed','Failed','Hosts','Domains','Emails','Edomains') {
4075 $report_totals{$section}{$type} = 0;
4079 # Generate our parser.
4080 my $parser = generate_parser();
4085 # Scan the input files and collect the data
4086 foreach my $file (@ARGV) {
4087 if ($file =~ /\.gz/) {
4088 unless (open(FILE,"gunzip -c $file |")) {
4089 print STDERR "Failed to gunzip -c $file: $!";
4093 elsif ($file =~ /\.Z/) {
4094 unless (open(FILE,"uncompress -c $file |")) {
4095 print STDERR "Failed to uncompress -c $file: $!";
4100 unless (open(FILE,$file)) {
4101 print STDERR "Failed to read $file: $!";
4105 #Now parse the filehandle, updating the global variables.
4106 parse($parser,\*FILE);
4111 #No files provided. Parse STDIN, updating the global variables.
4112 parse($parser,\*STDIN);
4116 if ($begin eq "9999-99-99 99:99:99" && ! $emptyOK) {
4117 print STDERR "**** No valid log lines read\n";
4121 # Output our results.
4123 print_grandtotals();
4125 # Print counts of user specified patterns if required.
4126 print_user_patterns() if @user_patterns;
4128 # Print rejection reasons.
4131 # Print totals by transport if required.
4132 print_transport() if $show_transport;
4134 # Print the deliveries per interval as a histogram, unless configured not to.
4135 # First find the maximum in one interval and scale accordingly.
4136 if ($hist_opt > 0) {
4137 print_histogram("Messages received", 'message', @received_interval_count);
4138 print_histogram("Deliveries", 'delivery', @delivered_interval_count);
4141 # Print times on queue if required.
4142 if ($#queue_times >= 0) {
4143 print_duration_table("Time spent on the queue", "all messages", \@queue_times, \@qt_all_bin,$qt_all_overflow);
4144 print_duration_table("Time spent on the queue", "messages with at least one remote delivery", \@queue_times, \@qt_remote_bin,$qt_remote_overflow);
4147 # Print delivery times if required.
4148 if ($#delivery_times >= 0) {
4149 print_duration_table("Delivery times", "all messages", \@delivery_times, \@dt_all_bin,$dt_all_overflow);
4150 print_duration_table("Delivery times", "messages with at least one remote delivery", \@delivery_times, \@dt_remote_bin,$dt_remote_overflow);
4153 # Print rcpt times if required.
4154 if ($#rcpt_times >= 0) {
4155 foreach my $protocol ('all', grep(!/^all$/, sort keys %rcpt_times_bin)) {
4156 print_duration_table("Receipt times", "$protocol messages", \@rcpt_times, $rcpt_times_bin{$protocol}, $rcpt_times_overflow{$protocol});
4160 # Print relay information if required.
4161 print_relay() if $show_relay;
4163 # Print the league tables, if topcount isn't zero.
4164 if ($topcount > 0) {
4165 my($ws_rej, $ws_top50, $ws_rej_row, $ws_top50_row, $ws_temp_rej, $ws_temp_rej_row);
4166 $ws_rej_row = $ws_temp_rej_row = $ws_top50_row = 0;
4168 $ws_top50 = $workbook->addworksheet('Deliveries');
4169 $ws_rej = $workbook->addworksheet('Rejections') if (%rejected_count_by_reason || %rejected_count_by_ip || %spam_count_by_ip);
4170 $ws_temp_rej = $workbook->addworksheet('Temporary Rejections') if (%temporarily_rejected_count_by_reason || %temporarily_rejected_count_by_ip);
4173 print_league_table("mail rejection reason", \%rejected_count_by_reason, undef, undef, undef, $ws_rej, \$ws_rej_row) if %rejected_count_by_reason;
4174 print_league_table("mail temporary rejection reason", \%temporarily_rejected_count_by_reason, undef, undef, undef, $ws_temp_rej, \$ws_temp_rej_row) if %temporarily_rejected_count_by_reason;
4176 foreach ('Host','Domain','Email','Edomain') {
4177 next unless $do_sender{$_};
4178 print_league_table("sending \l$_", $received_count{$_}, undef, $received_data{$_},$received_data_gigs{$_}, $ws_top50, \$ws_top50_row);
4181 print_league_table("local sender", \%received_count_user, undef,
4182 \%received_data_user,\%received_data_gigs_user, $ws_top50, \$ws_top50_row) if (($local_league_table || $include_remote_users) && %received_count_user);
4183 foreach ('Host','Domain','Email','Edomain') {
4184 next unless $do_sender{$_};
4185 print_league_table("\l$_ destination", $delivered_messages{$_}, $delivered_addresses{$_}, $delivered_data{$_},$delivered_data_gigs{$_}, $ws_top50, \$ws_top50_row);
4187 print_league_table("local destination", \%delivered_messages_user, \%delivered_addresses_user, \%delivered_data_user,\%delivered_data_gigs_user, $ws_top50, \$ws_top50_row) if (($local_league_table || $include_remote_users) && %delivered_messages_user);
4188 print_league_table("local domain destination", \%delivered_messages_local_domain, \%delivered_addresses_local_domain, \%delivered_data_local_domain,\%delivered_data_gigs_local_domain, $ws_top50, \$ws_top50_row) if (($local_league_table || $include_remote_users) && %delivered_messages_local_domain);
4190 print_league_table("rejected ip", \%rejected_count_by_ip, undef, undef, undef, $ws_rej, \$ws_rej_row) if %rejected_count_by_ip;
4191 print_league_table("temporarily rejected ip", \%temporarily_rejected_count_by_ip, undef, undef, undef, $ws_rej, \$ws_rej_row) if %temporarily_rejected_count_by_ip;
4192 print_league_table("non-rejected spamming ip", \%spam_count_by_ip, undef, undef, undef, $ws_rej, \$ws_rej_row) if %spam_count_by_ip;
4196 # Print the error statistics if required.
4197 print_errors() if $show_errors;
4199 print $htm_fh "</body>\n</html>\n" if $htm_fh;
4202 $txt_fh->close if $txt_fh && ref $txt_fh;
4203 $htm_fh->close if $htm_fh;
4206 # close Excel Workbook
4207 $ws_global->set_first_sheet();
4208 # FIXME: whyever - activate does not work :-/
4209 $ws_global->activate();