Grammar changes in docs
[exim.git] / src / src / eximstats.src
CommitLineData
8c4f17b3 1#!PERL_COMMAND
059ec3d9 2
9242a7e8 3# Copyright (c) 2001-2017 University of Cambridge.
059ec3d9
PH
4# See the file NOTICE for conditions of use and distribution.
5
6# Perl script to generate statistics from one or more Exim log files.
7
8# Usage: eximstats [<options>] <log file> <log file> ...
9
10# 1996-05-21: Ignore lines not starting with valid date/time, just in case
11# these get into a log file.
12# 1996-11-19: Add the -h option to control the size of the histogram,
13# and optionally turn it off.
14# Use some Perl 5 things; it should be everywhere by now.
15# Add the Perl -w option and rewrite so no warnings are given.
16# Add the -t option to control the length of the "top" listing.
17# Add the -ne, -nt options to turn off errors and transport
18# information.
19# Add information about length of time on queue, and -q<list> to
20# control the intervals and turn it off.
21# Add count and percentage of delayed messages to the Received
22# line.
23# Show total number of errors.
24# Add count and percentage of messages with errors to Received
25# line.
26# Add information about relaying and -nr to suppress it.
27# 1997-02-03 Merged in some of the things Nigel Metheringham had done:
28# Re-worded headings
29# Added received histogram as well as delivered
30# Added local senders' league table
31# Added local recipients' league table
32# 1997-03-10 Fixed typo "destinationss"
33# Allow for intermediate address between final and original
34# when testing for relaying
35# Give better message when no input
36# 1997-04-24 Fixed bug in layout of error listing that was depending on
37# text length (output line got repeated).
38# 1997-05-06 Bug in option decoding when only one option.
39# Overflow bug when handling very large volumes.
40# 1997-10-28 Updated to handle revised log format that might show
41# HELO name as well as host name before IP number
42# 1998-01-26 Bugs in the function for calculating the number of seconds
43# since 1970 from a log date
44# 1998-02-02 Delivery to :blackhole: doesn't have a T= entry in the log
45# line; cope with this, thereby avoiding undefined problems
46# Very short log line gave substring error
47# 1998-02-03 A routed delivery to a local transport may not have <> in the
48# log line; terminate the address at white space, not <
49# 1998-09-07 If first line of input was a => line, $thissize was undefined;
50# ensure it is zero.
51# 1998-12-21 Adding of $thissize from => line should have been adding $size.
52# Oops. Should have looked more closely when fixing the previous
53# bug!
54# 1999-11-12 Increased the field widths for printed integers; numbers are
55# bigger than originally envisaged.
56# 2001-03-21 Converted seconds() routine to use Time::Local, fixing a bug
57# whereby seconds($timestamp) - id_seconds($id) gave an
58# incorrect result.
59# Added POD documentation.
60# Moved usage instructions into help() subroutine.
61# Added 'use strict' and declared all global variables.
62# Added '-html' flag and resultant code.
63# Added '-cache' flag and resultant code.
64# Added add_volume() routine and converted all volume variables
65# to use it, fixing the overflow problems for individual hosts
66# on large sites.
67# Converted all volume output to GB/MB/KB as appropriate.
68# Don't store local user stats if -nfl is specified.
69# Modifications done by: Steve Campbell (<steve@computurn.com>)
70# 2001-04-02 Added the -t_remote_users flag. Steve Campbell.
71# 2001-10-15 Added the -domain flag. Steve Campbell.
72# 2001-10-16 Accept files on STDIN or on the command line. Steve Campbell.
73# 2001-10-21 Removed -domain flag and added -bydomain, -byhost, and -byemail.
74# We now generate our main parsing subroutine as an eval statement
75# which improves performance dramatically when not all the results
4c04137d 76# are required. We also cache the last timestamp to time conversion.
059ec3d9
PH
77#
78# NOTE: 'Top 50 destinations by (message count|volume)' lines are
79# now 'Top N (host|email|domain) destinations by (message count|volume)'
80# where N is the topcount. Steve Campbell.
81#
82# 2001-10-30 V1.16 Joachim Wieland.
83# Fixed minor bugs in add_volume() when taking over this version
84# for use in Exim 4: -w gave uninitialized value warnings in
85# two situations: for the first addition to a counter, and if
86# there were never any gigabytes, thereby leaving the $gigs
87# value unset.
88# Initialized $last_timestamp to stop a -w uninitialized warning.
89# Minor layout tweak for grand totals (nitpicking).
90# Put the IP addresses for relaying stats in [] and separated by
91# a space from the domain name.
92# Removed the IPv4-specific address test when picking out addresses
93# for relaying. Anything inside [] is OK.
94#
95# 2002-07-02 Philip Hazel
96# Fixed "uninitialized variable" message that occurred for relay
97# messages that arrived from H=[1.2.3.4] hosts (no name shown).
98# This bug didn't affect the output.
99#
100# 2002-04-15 V1.17 Joachim Wieland.
101# Added -charts, -chartdir. -chartrel options which use
102# GD::Graph modules to create graphical charts of the statistics.
103#
104# 2002-04-15 V1.18 Steve Campbell.
105# Added a check for $domain to to stop a -w uninitialized warning.
106# Added -byemaildomain option.
107# Only print HTML header links to included tables!
108#
109# 2002-08-02 V1.19 Steve Campbell.
110# Changed the debug mode to dump the parser onto STDERR rather
111# than STDOUT. Documented the -d flag into the help().
112# Rejoined the divergent 2002-04-15 and 2002-07-02 releases.
113#
114# 2002-08-21 V1.20 Steve Campbell.
115# Added the '-merge' option to allow merging of previous reports.
116# Fixed a missing semicolon when doing -bydomain.
117# Make volume charts plot the data gigs and bytes rather than just bytes.
118# Only process log lines with $flag =~ /<=|=>|->|==|\*\*|Co/
119# Converted Emaildomain to Edomain - the column header was too wide!
120# This changes the text output slightly. You can revert to the old
121# column widths by changing $COLUMN_WIDTHS to 7;
122#
123# 2002-09-04 V1.21 Andreas J Mueller
124# Local deliveries domain now defaults to 'localdomain'.
125# Don't match F=<From> when looking for the user.
126#
127# 2002-09-05 V1.22 Steve Campbell
128# Fixed a perl 5.005 incompatibility problem ('our' variables).
129#
130# 2002-09-11 V1.23 Steve Campbell
131# Stopped -charts option from throwing errors on null data.
132# Don't print out 'Errors encountered' unless there are any.
133
134# 2002-10-21 V1.23a Philip Hazel - patch from Tony Finch put in until
135# Steve's eximstats catches up.
136# Handle log files that include the timezone after the timestamp.
137# Switch to assuming that log timestamps are in local time, with
138# an option for UTC timestamps, as in Exim itself.
139#
140# 2003-02-05 V1.24 Steve Campbell
141# Added in Sergey Sholokh's code to convert '<' and '>' characters
142# in HTML output. Also added code to convert them back with -merge.
143# Fixed timestamp offsets to convert to seconds rather than minutes.
144# Updated -merge to work with output files using timezones.
4c04137d 145# Added caching to speed up the calculation of timezone offsets.
059ec3d9
PH
146#
147# 2003-02-07 V1.25 Steve Campbell
148# Optimised the usage of mktime() in the seconds subroutine.
149# Removed the now redundant '-cache' option.
150# html2txt() now explicitly matches HTML tags.
151# Implemented a new sorting algorithm - the top_n_sort() routine.
152# Added Danny Carroll's '-nvr' flag and code.
153#
154# 2003-03-13 V1.26 Steve Campbell
155# Implemented HTML compliance changes recommended by Bernard Massot.
156# Bug fix to allow top_n_sort() to handle null keys.
157# Convert all domains and edomains to lowercase.
158# Remove preceding dots from domains.
159#
160# 2003-03-13 V1.27 Steve Campbell
161# Replaced border attributes with 'border=1', as recommended by
162# Bernard Massot.
163#
164# 2003-06-03 V1.28 John Newman
4c04137d 165# Added in the ability to skip over the parsing and evaluation of
059ec3d9
PH
166# specific transports as passed to eximstats via the new "-nt/.../"
167# command line argument. This new switch allows the viewing of
168# not more accurate statistics but more applicable statistics when
169# special transports are in use (ie; SpamAssassin). We need to be
170# able to ignore transports such as this otherwise the resulting
171# local deliveries are significantly skewed (doubled)...
172#
173# 2003-11-06 V1.29 Steve Campbell
174# Added the '-pattern "Description" "/pattern/"' option.
175#
176# 2004-02-17 V1.30 Steve Campbell
177# Added warnings if required GD::Graph modules are not available or
178# insufficient -chart* options are specified.
179#
8e669ac1 180# 2004-02-20 V1.31 Andrea Balzi
059ec3d9
PH
181# Only show the Local Sender/Destination links if the tables exist.
182#
1b4fe9dd
PH
183# 2004-07-05 V1.32 Steve Campbell
184# Fix '-merge -h0' divide by zero error.
185#
186# 2004-07-15 V1.33 Steve Campbell
187# Documentation update - I've converted the subroutine
188# documentation from POD to comments.
8974000d
SC
189#
190# 2004-12-10 V1.34 Steve Campbell
191# Eximstats can now parse syslog lines as well as mainlog lines.
192#
884c2af8 193# 2004-12-20 V1.35 Wouter Verhelst
d5692f86 194# Pie charts by volume were actually generated by count. Fixed.
50adf73a
SC
195#
196# 2005-02-07 V1.36 Gregor Herrmann / Steve Campbell
197# Added average sizes to HTML Top tables.
d5692f86
SC
198#
199# 2005-04-26 V1.37 Frank Heydlauf
200# Added -xls and the ability to specify output files.
201#
202# 2005-04-29 V1.38 Steve Campbell
4c04137d 203# Use FileHandles for outputting results.
d5692f86
SC
204# Allow any combination of xls, txt, and html output.
205# Fixed display of large numbers with -nvr option
206# Fixed merging of reports with empty tables.
207#
208# 2005-05-27 V1.39 Steve Campbell
209# Added the -include_original_destination flag
210# Removed tabs and trailing whitespace.
211#
608bc29d
SC
212# 2005-06-03 V1.40 Steve Campbell
213# Whilst parsing the mainlog(s), store information about
214# the messages in a hash of arrays rather than using
215# individual hashes. This is a bit cleaner and results in
216# dramatic memory savings, albeit at a slight CPU cost.
217#
218# 2005-06-15 V1.41 Steve Campbell
219# Added the -show_rt<list> flag.
220# Added the -show_dt<list> flag.
221#
222# 2005-06-24 V1.42 Steve Campbell
223# Added Histograms for user specified patterns.
d5692f86 224#
81aad8c9
SC
225# 2005-06-30 V1.43 Steve Campbell
226# Bug fix for V1.42 with -h0 specified. Spotted by Chris Lear.
227#
a83c7e95
SC
228# 2005-07-26 V1.44 Steve Campbell
229# Use a glob alias rather than an array ref in the generated
230# parser. This improves both readability and performance.
231#
232# 2005-09-30 V1.45 Marco Gaiarin / Steve Campbell
233# Collect SpamAssassin and rejection statistics.
234# Don't display local sender or destination tables unless
235# there is data to show.
236# Added average volumes into the top table text output.
237#
238# 2006-02-07 V1.46 Steve Campbell
239# Collect data on the number of addresses (recipients)
240# as well as the number of messages.
241#
242# 2006-05-05 V1.47 Steve Campbell
243# Added 'Message too big' to the list of mail rejection
244# reasons (thanks to Marco Gaiarin).
245#
246# 2006-06-05 V1.48 Steve Campbell
247# Mainlog lines which have GMT offsets and are too short to
248# have a flag are now skipped.
249#
250# 2006-11-10 V1.49 Alain Williams
251# Added the -emptyok flag.
252#
253# 2006-11-16 V1.50 Steve Campbell
254# Fixes for obtaining the IP address from reject messages.
255#
e2f7a0d2
SC
256# 2006-11-27 V1.51 Steve Campbell
257# Another update for obtaining the IP address from reject messages.
258#
259# 2006-11-27 V1.52 Steve Campbell
260# Tally any reject message containing SpamAssassin.
261#
f3f065bb
PH
262# 2007-01-31 V1.53 Philip Hazel
263# Allow for [pid] after date in log lines
264#
a797be69
SC
265# 2007-02-14 V1.54 Daniel Tiefnig
266# Improved the '($parent) =' pattern match.
267#
30e9ac3f
SC
268# 2007-03-19 V1.55 Steve Campbell
269# Differentiate between permanent and temporary rejects.
270#
271# 2007-03-29 V1.56 Jez Hancock
272# Fixed some broken HTML links and added missing column headers.
273#
274# 2007-03-30 V1.57 Steve Campbell
275# Fixed Grand Total Summary Domains, Edomains, and Email columns
276# for Rejects, Temp Rejects, Ham, and Spam rows.
277#
5e6e6734
SC
278# 2007-04-11 V1.58 Steve Campbell
279# Fix to get <> and blackhole to show in edomain tables.
280#
37f8a7c9
SC
281# 2007-09-20 V1.59 Steve Campbell
282# Added the -bylocaldomain option
283#
d5c13d66
NM
284# 2007-09-20 V1.60 Heiko Schlittermann
285# Fix for misinterpreted log lines
286#
514ee161
SC
287# 2013-01-14 V1.61 Steve Campbell
288# Watch out for senders sending "HELO [IpAddr]"
a83c7e95 289#
d5692f86
SC
290#
291# For documentation on the logfile format, see
292# http://www.exim.org/exim-html-4.50/doc/html/spec_48.html#IX2793
059ec3d9
PH
293
294=head1 NAME
295
8974000d 296eximstats - generates statistics from Exim mainlog or syslog files.
059ec3d9
PH
297
298=head1 SYNOPSIS
299
d5692f86 300 eximstats [Output] [Options] mainlog1 mainlog2 ...
059ec3d9
PH
301 eximstats -merge [Options] report.1.txt report.2.txt ... > weekly_report.txt
302
d5692f86
SC
303=head2 Output:
304
305=over 4
306
307=item B<-txt>
308
309Output the results in plain text to STDOUT.
310
311=item B<-txt>=I<filename>
312
313Output the results in plain text. Filename '-' for STDOUT is accepted.
314
315=item B<-html>
316
317Output the results in HTML to STDOUT.
318
319=item B<-html>=I<filename>
320
321Output the results in HTML. Filename '-' for STDOUT is accepted.
322
323=item B<-xls>
324
325Output the results in Excel compatible Format to STDOUT.
326Requires the Spreadsheet::WriteExcel CPAN module.
327
328=item B<-xls>=I<filename>
329
330Output the results in Excel compatible format. Filename '-' for STDOUT is accepted.
331
332
333=back
334
335=head2 Options:
059ec3d9
PH
336
337=over 4
338
339=item B<-h>I<number>
340
341histogram divisions per hour. The default is 1, and
3420 suppresses histograms. Valid values are:
343
3440, 1, 2, 3, 5, 10, 15, 20, 30 or 60.
345
346=item B<-ne>
347
348Don't display error information.
349
350=item B<-nr>
351
352Don't display relaying information.
353
354=item B<-nr>I</pattern/>
355
356Don't display relaying information that matches.
357
358=item B<-nt>
359
360Don't display transport information.
361
362=item B<-nt>I</pattern/>
363
364Don't display transport information that matches
365
366=item B<-q>I<list>
367
368List of times for queuing information single 0 item suppresses.
369
370=item B<-t>I<number>
371
372Display top <number> sources/destinations
373default is 50, 0 suppresses top listing.
374
375=item B<-tnl>
376
377Omit local sources/destinations in top listing.
378
379=item B<-t_remote_users>
380
381Include remote users in the top source/destination listings.
382
d5692f86
SC
383=item B<-include_original_destination>
384
385Include the original destination email addresses rather than just
386using the final ones.
387Useful for finding out which of your mailing lists are receiving mail.
388
608bc29d
SC
389=item B<-show_dt>I<list>
390
391Show the delivery times (B<DT>)for all the messages.
392
14eee748 393Exim must have been configured to use the +deliver_time logging option
608bc29d
SC
394for this option to work.
395
396I<list> is an optional list of times. Eg -show_dt1,2,4,8 will show
397the number of messages with delivery times under 1 second, 2 seconds, 4 seconds,
3988 seconds, and over 8 seconds.
399
400=item B<-show_rt>I<list>
401
402Show the receipt times for all the messages. The receipt time is
403defined as the Completed hh:mm:ss - queue_time_overall - the Receipt hh:mm:ss.
404These figures will be skewed by pipelined messages so might not be that useful.
405
406Exim must have been configured to use the +queue_time_overall logging option
407for this option to work.
408
409I<list> is an optional list of times. Eg -show_rt1,2,4,8 will show
410the number of messages with receipt times under 1 second, 2 seconds, 4 seconds,
4118 seconds, and over 8 seconds.
412
059ec3d9
PH
413=item B<-byhost>
414
415Show results by sending host. This may be combined with
416B<-bydomain> and/or B<-byemail> and/or B<-byedomain>. If none of these options
417are specified, then B<-byhost> is assumed as a default.
418
419=item B<-bydomain>
420
421Show results by sending domain.
422May be combined with B<-byhost> and/or B<-byemail> and/or B<-byedomain>.
423
424=item B<-byemail>
425
426Show results by sender's email address.
427May be combined with B<-byhost> and/or B<-bydomain> and/or B<-byedomain>.
428
429=item B<-byemaildomain> or B<-byedomain>
430
431Show results by sender's email domain.
432May be combined with B<-byhost> and/or B<-bydomain> and/or B<-byemail>.
433
434=item B<-pattern> I<Description> I</Pattern/>
435
436Look for the specified pattern and count the number of lines in which it appears.
437This option can be specified multiple times. Eg:
438
439 -pattern 'Refused connections' '/refused connection/'
440
441
442=item B<-merge>
443
444This option allows eximstats to merge old eximstat reports together. Eg:
445
446 eximstats mainlog.sun > report.sun.txt
447 eximstats mainlog.mon > report.mon.txt
448 eximstats mainlog.tue > report.tue.txt
449 eximstats mainlog.wed > report.web.txt
450 eximstats mainlog.thu > report.thu.txt
451 eximstats mainlog.fri > report.fri.txt
452 eximstats mainlog.sat > report.sat.txt
453 eximstats -merge report.*.txt > weekly_report.txt
454 eximstats -merge -html report.*.txt > weekly_report.html
455
456=over 4
457
458=item *
459
460You can merge text or html reports and output the results as text or html.
461
462=item *
463
464You can use all the normal eximstat output options, but only data
465included in the original reports can be shown!
466
467=item *
468
469When merging reports, some loss of accuracy may occur in the top I<n> lists.
470This will be towards the ends of the lists.
471
472=item *
473
474The order of items in the top I<n> lists may vary when the data volumes
475round to the same value.
476
477=back
478
059ec3d9
PH
479=item B<-charts>
480
481Create graphical charts to be displayed in HTML output.
d5692f86 482Only valid in combination with I<-html>.
059ec3d9
PH
483
484This requires the following modules which can be obtained
485from http://www.cpan.org/modules/01modules.index.html
486
487=over 4
488
489=item GD
490
491=item GDTextUtil
492
493=item GDGraph
494
495=back
496
497To install these, download and unpack them, then use the normal perl installation procedure:
498
499 perl Makefile.PL
500 make
501 make test
502 make install
503
504=item B<-chartdir>I <dir>
505
506Create the charts in the directory <dir>
507
508=item B<-chartrel>I <dir>
509
510Specify the relative directory for the "img src=" tags from where to include
511the charts
512
a83c7e95
SC
513=item B<-emptyok>
514
515Specify that it's OK to not find any valid log lines. Without this
516we will output an error message if we don't find any.
517
059ec3d9
PH
518=item B<-d>
519
520Debug flag. This outputs the eval()'d parser onto STDOUT which makes it
521easier to trap errors in the eval section. Remember to add 1 to the line numbers to allow for the
522title!
523
524=back
525
526=head1 DESCRIPTION
527
8974000d 528Eximstats parses exim mainlog and syslog files to output a statistical
059ec3d9 529analysis of the messages processed. By default, a text
d5692f86
SC
530analysis is generated, but you can request other output formats
531using flags. See the help (B<-help>) to learn
059ec3d9
PH
532about how to create charts from the tables.
533
534=head1 AUTHOR
535
20f0f788 536There is a website at https://www.exim.org - this contains details of the
059ec3d9
PH
537mailing list exim-users@exim.org.
538
539=head1 TO DO
540
541This program does not perfectly handle messages whose received
542and delivered log lines are in different files, which can happen
543when you have multiple mail servers and a message cannot be
43236f35 544immediately delivered. Fixing this could be tricky...
059ec3d9 545
d5692f86
SC
546Merging of xls files is not (yet) possible. Be free to implement :)
547
059ec3d9
PH
548=cut
549
8c4f17b3 550use warnings;
059ec3d9 551use integer;
4d3d955f 552BEGIN { pop @INC if $INC[-1] eq '.' };
059ec3d9 553use strict;
d5692f86 554use IO::File;
983da878 555use File::Basename;
059ec3d9
PH
556
557# use Time::Local; # PH/FANF
558use POSIX;
559
44de51a1 560if (@ARGV and $ARGV[0] eq '--version') {
983da878
HSHR
561 print basename($0) . ": $0\n",
562 "build: EXIM_RELEASE_VERSIONEXIM_VARIANT_VERSION\n",
02721dcd 563 "perl(runtime): $]\n";
983da878
HSHR
564 exit 0;
565}
566
d5692f86 567use vars qw($HAVE_GD_Graph_pie $HAVE_GD_Graph_linespoints $HAVE_Spreadsheet_WriteExcel);
059ec3d9
PH
568eval { require GD::Graph::pie; };
569$HAVE_GD_Graph_pie = $@ ? 0 : 1;
570eval { require GD::Graph::linespoints; };
571$HAVE_GD_Graph_linespoints = $@ ? 0 : 1;
d5692f86
SC
572eval { require Spreadsheet::WriteExcel; };
573$HAVE_Spreadsheet_WriteExcel = $@ ? 0 : 1;
059ec3d9
PH
574
575
576##################################################
577# Static data #
578##################################################
579# 'use vars' instead of 'our' as perl5.005 is still in use out there!
580use vars qw(@tab62 @days_per_month $gig);
581use vars qw($VERSION);
582use vars qw($COLUMN_WIDTHS);
608bc29d 583use vars qw($WEEK $DAY $HOUR $MINUTE);
059ec3d9
PH
584
585
586@tab62 =
587 (0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, # 0-9
588 0,10,11,12,13,14,15,16,17,18,19,20, # A-K
589 21,22,23,24,25,26,27,28,29,30,31,32, # L-W
590 33,34,35, 0, 0, 0, 0, 0, # X-Z
591 0,36,37,38,39,40,41,42,43,44,45,46, # a-k
592 47,48,49,50,51,52,53,54,55,56,57,58, # l-w
593 59,60,61); # x-z
594
595@days_per_month = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334);
596$gig = 1024 * 1024 * 1024;
514ee161 597$VERSION = '1.61';
059ec3d9
PH
598
599# How much space do we allow for the Hosts/Domains/Emails/Edomains column headers?
600$COLUMN_WIDTHS = 8;
601
608bc29d
SC
602$MINUTE = 60;
603$HOUR = 60 * $MINUTE;
604$DAY = 24 * $HOUR;
605$WEEK = 7 * $DAY;
606
059ec3d9
PH
607# Declare global variables.
608use vars qw($total_received_data $total_received_data_gigs $total_received_count);
a83c7e95 609use vars qw($total_delivered_data $total_delivered_data_gigs $total_delivered_messages $total_delivered_addresses);
d5692f86 610use vars qw(%timestamp2time); #Hash of timestamp => time.
4c04137d
JS
611use vars qw($last_timestamp $last_time); #The last time conversion done.
612use vars qw($last_date $date_seconds); #The last date conversion done.
613use vars qw($last_offset $offset_seconds); #The last time offset conversion done.
059ec3d9 614use vars qw($localtime_offset);
d5692f86
SC
615use vars qw($i); #General loop counter.
616use vars qw($debug); #Debug mode?
617use vars qw($ntopchart); #How many entries should make it into the chart?
618use vars qw($gddirectory); #Where to put files from GD::Graph
a83c7e95
SC
619
620# SpamAssassin variables
621use vars qw($spam_score $spam_score_gigs);
622use vars qw($ham_score $ham_score_gigs);
623use vars qw(%ham_count_by_ip %spam_count_by_ip);
624use vars qw(%rejected_count_by_ip %rejected_count_by_reason);
30e9ac3f 625use vars qw(%temporarily_rejected_count_by_ip %temporarily_rejected_count_by_reason);
a83c7e95 626
4c04137d 627#For use in Spreadsheet::WriteExcel
a83c7e95
SC
628use vars qw($workbook $ws_global $ws_relayed $ws_errors);
629use vars qw($row $col $row_hist $col_hist);
d5692f86 630use vars qw($run_hist);
a83c7e95 631use vars qw($f_default $f_header1 $f_header2 $f_header2_m $f_headertab $f_percent); #Format Header
d5692f86
SC
632
633# Output FileHandles
634use vars qw($txt_fh $htm_fh $xls_fh);
059ec3d9
PH
635
636$ntopchart = 5;
637
638# The following are parameters whose values are
639# set by command line switches:
640use vars qw($show_errors $show_relay $show_transport $transport_pattern);
37f8a7c9 641use vars qw($topcount $local_league_table $include_remote_users $do_local_domain);
a83c7e95 642use vars qw($hist_opt $hist_interval $hist_number $volume_rounding $emptyOK);
d5692f86 643use vars qw($relay_pattern @queue_times @user_patterns @user_descriptions);
608bc29d 644use vars qw(@rcpt_times @delivery_times);
d5692f86
SC
645use vars qw($include_original_destination);
646use vars qw($txt_fh $htm_fh $xls_fh);
059ec3d9
PH
647
648use vars qw(%do_sender); #Do sender by Host, Domain, Email, and/or Edomain tables.
649use vars qw($charts $chartrel $chartdir $charts_option_specified);
d5692f86 650use vars qw($merge_reports); #Merge old reports ?
059ec3d9
PH
651
652# The following are modified in the parse() routine, and
653# referred to in the print_*() routines.
608bc29d 654use vars qw($delayed_count $relayed_unshown $begin $end);
a83c7e95 655use vars qw(%messages @message);
059ec3d9 656use vars qw(%received_count %received_data %received_data_gigs);
a83c7e95 657use vars qw(%delivered_messages %delivered_data %delivered_data_gigs %delivered_addresses);
059ec3d9 658use vars qw(%received_count_user %received_data_user %received_data_gigs_user);
a83c7e95 659use vars qw(%delivered_messages_user %delivered_addresses_user %delivered_data_user %delivered_data_gigs_user);
37f8a7c9 660use vars qw(%delivered_messages_local_domain %delivered_addresses_local_domain %delivered_data_local_domain %delivered_data_gigs_local_domain);
059ec3d9 661use vars qw(%transported_count %transported_data %transported_data_gigs);
608bc29d
SC
662use vars qw(%relayed %errors_count $message_errors);
663use vars qw(@qt_all_bin @qt_remote_bin);
664use vars qw($qt_all_overflow $qt_remote_overflow);
665use vars qw(@dt_all_bin @dt_remote_bin %rcpt_times_bin);
666use vars qw($dt_all_overflow $dt_remote_overflow %rcpt_times_overflow);
667use vars qw(@received_interval_count @delivered_interval_count);
668use vars qw(@user_pattern_totals @user_pattern_interval_count);
059ec3d9
PH
669
670use vars qw(%report_totals);
671
608bc29d
SC
672# Enumerations
673use vars qw($SIZE $FROM_HOST $FROM_ADDRESS $ARRIVAL_TIME $REMOTE_DELIVERED $PROTOCOL);
674use vars qw($DELAYED $HAD_ERROR);
675$SIZE = 0;
676$FROM_HOST = 1;
677$FROM_ADDRESS = 2;
678$ARRIVAL_TIME = 3;
679$REMOTE_DELIVERED = 4;
680$DELAYED = 5;
681$HAD_ERROR = 6;
682$PROTOCOL = 7;
059ec3d9
PH
683
684
685
686##################################################
687# Subroutines #
688##################################################
689
d5692f86
SC
690#######################################################################
691# get_filehandle($file,\%output_files);
692# Return a filehandle writing to $file.
693#
694# If %output_files is defined, check that $output_files{$file}
695# doesn't exist and die if it does, or set it if it doesn't.
696#######################################################################
697sub get_filehandle {
698 my($file,$output_files_href) = @_;
699
700 $file = '-' if ($file eq '');
701
702 if (defined $output_files_href) {
703 die "You can only output to '$file' once! Use -h for help.\n" if exists $output_files_href->{$file};
704 $output_files_href->{$file} = 1;
705 }
706
707 if ($file eq '-') {
708 return \*STDOUT;
709 }
710
711 if (-e $file) {
712 unlink $file or die "Failed to rm $file: $!";
713 }
714
715 my $fh = new IO::File $file, O_WRONLY|O_CREAT|O_EXCL;
716 die "new IO::File $file failed: $!" unless (defined $fh);
717 return $fh;
718}
719
059ec3d9 720
1b4fe9dd
PH
721#######################################################################
722# volume_rounded();
723#
724# $rounded_volume = volume_rounded($bytes,$gigabytes);
725#
726# Given a data size in bytes, round it to KB, MB, or GB
727# as appropriate.
728#
729# Eg 12000 => 12KB, 15000000 => 14GB, etc.
730#
731# Note: I've experimented with Math::BigInt and it results in a 33%
732# performance degredation as opposed to storing numbers split into
733# bytes and gigabytes.
734#######################################################################
059ec3d9
PH
735sub volume_rounded {
736 my($x,$g) = @_;
737 $x = 0 unless $x;
738 $g = 0 unless $g;
739 my($rounded);
740
741 while ($x > $gig) {
742 $g++;
743 $x -= $gig;
744 }
745
746 if ($volume_rounding) {
747 # Values < 1 GB
748 if ($g <= 0) {
749 if ($x < 10000) {
750 $rounded = sprintf("%6d", $x);
751 }
752 elsif ($x < 10000000) {
753 $rounded = sprintf("%4dKB", ($x + 512)/1024);
754 }
755 else {
756 $rounded = sprintf("%4dMB", ($x + 512*1024)/(1024*1024));
757 }
758 }
759 # Values between 1GB and 10GB are printed in MB
760 elsif ($g < 10) {
761 $rounded = sprintf("%4dMB", ($g * 1024) + ($x + 512*1024)/(1024*1024));
762 }
763 else {
764 # Handle values over 10GB
765 $rounded = sprintf("%4dGB", $g + ($x + $gig/2)/$gig);
766 }
767 }
768 else {
769 # We don't want any rounding to be done.
968178ac 770 # and we don't need broken formatted output which on one hand avoids numbers from
4c04137d 771 # being interpreted as string by Spreadsheet Calculators, on the other hand
d5692f86
SC
772 # breaks if more than 4 digits! -> flexible length instead of fixed length
773 # Format the return value at the output routine! -fh
774 #$rounded = sprintf("%d", ($g * $gig) + $x);
775 no integer;
776 $rounded = sprintf("%.0f", ($g * $gig) + $x);
059ec3d9
PH
777 }
778
779 return $rounded;
780}
781
782
1b4fe9dd
PH
783#######################################################################
784# un_round();
8e669ac1 785#
1b4fe9dd 786# un_round($rounded_volume,\$bytes,\$gigabytes);
8e669ac1 787#
1b4fe9dd
PH
788# Given a volume in KB, MB or GB, as generated by volume_rounded(),
789# do the reverse transformation and convert it back into Bytes and Gigabytes.
790# These are added to the $bytes and $gigabytes parameters.
8e669ac1 791#
1b4fe9dd
PH
792# Given a data size in bytes, round it to KB, MB, or GB
793# as appropriate.
8e669ac1 794#
1b4fe9dd
PH
795# EG: 500 => (500,0), 14GB => (0,14), etc.
796#######################################################################
059ec3d9
PH
797sub un_round {
798 my($rounded,$bytes_sref,$gigabytes_sref) = @_;
799
800 if ($rounded =~ /(\d+)GB/) {
801 $$gigabytes_sref += $1;
802 }
803 elsif ($rounded =~ /(\d+)MB/) {
804 $$gigabytes_sref += $1 / 1024;
805 $$bytes_sref += (($1 % 1024 ) * 1024 * 1024);
806 }
807 elsif ($rounded =~ /(\d+)KB/) {
808 $$gigabytes_sref += $1 / (1024 * 1024);
809 $$bytes_sref += ($1 % (1024 * 1024) * 1024);
810 }
811 elsif ($rounded =~ /(\d+)/) {
d5692f86
SC
812 # We need to turn off integer in case we are merging an -nvr report.
813 no integer;
814 $$gigabytes_sref += int($1 / $gig);
059ec3d9
PH
815 $$bytes_sref += $1 % $gig;
816 }
817
818 #Now reduce the bytes down to less than 1GB.
819 add_volume($bytes_sref,$gigabytes_sref,0) if ($$bytes_sref > $gig);
820}
821
822
1b4fe9dd
PH
823#######################################################################
824# add_volume();
8e669ac1 825#
1b4fe9dd 826# add_volume(\$bytes,\$gigs,$size);
8e669ac1 827#
1b4fe9dd
PH
828# Add $size to $bytes/$gigs where this is a number split into
829# bytes ($bytes) and gigabytes ($gigs). This is significantly
830# faster than using Math::BigInt.
831#######################################################################
059ec3d9 832sub add_volume {
1b4fe9dd
PH
833 my($bytes_ref,$gigs_ref,$size) = @_;
834 $$bytes_ref = 0 if ! defined $$bytes_ref;
835 $$gigs_ref = 0 if ! defined $$gigs_ref;
836 $$bytes_ref += $size;
837 while ($$bytes_ref > $gig) {
838 $$gigs_ref++;
839 $$bytes_ref -= $gig;
059ec3d9
PH
840 }
841}
842
843
1b4fe9dd
PH
844#######################################################################
845# format_time();
8e669ac1 846#
1b4fe9dd 847# $formatted_time = format_time($seconds);
8e669ac1 848#
1b4fe9dd
PH
849# Given a time in seconds, break it down into
850# weeks, days, hours, minutes, and seconds.
8e669ac1 851#
1b4fe9dd
PH
852# Eg 12005 => 3h20m5s
853#######################################################################
059ec3d9
PH
854sub format_time {
855my($t) = pop @_;
856my($s) = $t % 60;
857$t /= 60;
858my($m) = $t % 60;
859$t /= 60;
860my($h) = $t % 24;
861$t /= 24;
862my($d) = $t % 7;
863my($w) = $t/7;
864my($p) = "";
865$p .= "$w"."w" if $w > 0;
866$p .= "$d"."d" if $d > 0;
867$p .= "$h"."h" if $h > 0;
868$p .= "$m"."m" if $m > 0;
869$p .= "$s"."s" if $s > 0 || $p eq "";
870$p;
871}
872
873
1b4fe9dd
PH
874#######################################################################
875# unformat_time();
8e669ac1 876#
1b4fe9dd 877# $seconds = unformat_time($formatted_time);
8e669ac1 878#
1b4fe9dd 879# Given a time in weeks, days, hours, minutes, or seconds, convert it to seconds.
8e669ac1 880#
1b4fe9dd
PH
881# Eg 3h20m5s => 12005
882#######################################################################
059ec3d9 883sub unformat_time {
968178ac 884 my($formatted_time) = pop @_;
059ec3d9
PH
885 my $time = 0;
886
968178ac 887 while ($formatted_time =~ s/^(\d+)([wdhms]?)//) {
059ec3d9
PH
888 $time += $1 if ($2 eq '' || $2 eq 's');
889 $time += $1 * 60 if ($2 eq 'm');
890 $time += $1 * 60 * 60 if ($2 eq 'h');
891 $time += $1 * 60 * 60 * 24 if ($2 eq 'd');
892 $time += $1 * 60 * 60 * 24 * 7 if ($2 eq 'w');
893 }
894 $time;
895}
896
897
1b4fe9dd
PH
898#######################################################################
899# seconds();
8e669ac1 900#
1b4fe9dd 901# $time = seconds($timestamp);
8e669ac1 902#
1b4fe9dd
PH
903# Given a time-of-day timestamp, convert it into a time() value using
904# POSIX::mktime. We expect the timestamp to be of the form
905# "$year-$mon-$day $hour:$min:$sec", with month going from 1 to 12,
906# and the year to be absolute (we do the necessary conversions). The
571b2715 907# seconds value can be followed by decimals, which we ignore. The
1b4fe9dd
PH
908# timestamp may be followed with an offset from UTC like "+$hh$mm"; if the
909# offset is not present, and we have not been told that the log is in UTC
910# (with the -utc option), then we adjust the time by the current local
911# time offset so that it can be compared with the time recorded in message
912# IDs, which is UTC.
8e669ac1 913#
1b4fe9dd
PH
914# To improve performance, we only use mktime on the date ($year-$mon-$day),
915# and only calculate it if the date is different to the previous time we
916# came here. We then add on seconds for the '$hour:$min:$sec'.
8e669ac1 917#
1b4fe9dd
PH
918# We also store the results of the last conversion done, and only
919# recalculate if the date is different.
8e669ac1 920#
1b4fe9dd
PH
921# We used to have the '-cache' flag which would store the results of the
922# mktime() call. However, the current way of just using mktime() on the
923# date obsoletes this.
924#######################################################################
059ec3d9
PH
925sub seconds {
926 my($timestamp) = @_;
927
928 # Is the timestamp the same as the last one?
929 return $last_time if ($last_timestamp eq $timestamp);
930
571b2715 931 return 0 unless ($timestamp =~ /^((\d{4})\-(\d\d)-(\d\d))\s(\d\d):(\d\d):(\d\d)(?:\.\d+)?( ([+-])(\d\d)(\d\d))?/o);
059ec3d9
PH
932
933 unless ($last_date eq $1) {
934 $last_date = $1;
935 my(@timestamp) = (0,0,0,$4,$3,$2);
936 $timestamp[5] -= 1900;
937 $timestamp[4]--;
938 $date_seconds = mktime(@timestamp);
939 }
940 my $time = $date_seconds + ($5 * 3600) + ($6 * 60) + $7;
941
4c04137d 942 # SC. Use caching. Also note we want seconds not minutes.
571b2715 943 #my($this_offset) = ($10 * 60 + $12) * ($9 . "1") if defined $8;
059ec3d9
PH
944 if (defined $8 && ($8 ne $last_offset)) {
945 $last_offset = $8;
946 $offset_seconds = ($10 * 60 + $11) * 60;
947 $offset_seconds = -$offset_seconds if ($9 eq '-');
948 }
949
950
571b2715 951 if (defined $8) {
059ec3d9
PH
952 #$time -= $this_offset;
953 $time -= $offset_seconds;
954 } elsif (defined $localtime_offset) {
955 $time -= $localtime_offset;
956 }
957
958 # Store the last timestamp received.
959 $last_timestamp = $timestamp;
960 $last_time = $time;
961
962 $time;
963}
964
965
1b4fe9dd
PH
966#######################################################################
967# id_seconds();
8e669ac1 968#
1b4fe9dd 969# $time = id_seconds($message_id);
8e669ac1 970#
1b4fe9dd
PH
971# Given a message ID, convert it into a time() value.
972#######################################################################
059ec3d9
PH
973sub id_seconds {
974my($sub_id) = substr((pop @_), 0, 6);
975my($s) = 0;
976my(@c) = split(//, $sub_id);
977while($#c >= 0) { $s = $s * 62 + $tab62[ord(shift @c) - ord('0')] }
978$s;
979}
980
608bc29d
SC
981#######################################################################
982# wdhms_seconds();
983#
984# $seconds = wdhms_seconds($string);
985#
986# Convert a string in a week/day/hour/minute/second format (eg 4h10s)
987# into seconds.
988#######################################################################
989sub wdhms_seconds {
990 if ($_[0] =~ /^(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?/) {
991 return((($1||0) * $WEEK) + (($2||0) * $DAY) + (($3||0) * $HOUR) + (($4||0) * $MINUTE) + ($5||0));
992 }
993 return undef;
994}
995
996#######################################################################
997# queue_time();
998#
999# $queued = queue_time($completed_tod, $arrival_time, $id);
1000#
1001# Given the completed time of day and either the arrival time
1002# (preferred), or the message ID, calculate how long the message has
1003# been on the queue.
1004#
1005#######################################################################
1006sub queue_time {
1007 my($completed_tod, $arrival_time, $id) = @_;
1008
1009 # Note: id_seconds() benchmarks as 42% slower than seconds()
1010 # and computing the time accounts for a significant portion of
1011 # the run time.
1012 if (defined $arrival_time) {
1013 return(seconds($completed_tod) - seconds($arrival_time));
1014 }
1015 else {
1016 return(seconds($completed_tod) - id_seconds($id));
1017 }
1018}
059ec3d9
PH
1019
1020
1b4fe9dd
PH
1021#######################################################################
1022# calculate_localtime_offset();
8e669ac1 1023#
1b4fe9dd 1024# $localtime_offset = calculate_localtime_offset();
8e669ac1 1025#
1b4fe9dd 1026# Calculate the the localtime offset from gmtime in seconds.
8e669ac1 1027#
1b4fe9dd 1028# $localtime = time() + $localtime_offset.
8e669ac1 1029#
1b4fe9dd
PH
1030# These are the same semantics as ISO 8601 and RFC 2822 timezone offsets.
1031# (West is negative, East is positive.)
1032#######################################################################
059ec3d9
PH
1033
1034# $localtime = gmtime() + $localtime_offset. OLD COMMENT
1035# This subroutine commented out as it's not currently in use.
1036
1037#sub calculate_localtime_offset {
1038# # Pick an arbitrary date, convert it to localtime & gmtime, and return the difference.
1039# my (@sample_date) = (0,0,0,5,5,100);
1040# my $localtime = timelocal(@sample_date);
1041# my $gmtime = timegm(@sample_date);
1042# my $offset = $localtime - $gmtime;
1043# return $offset;
1044#}
1045
1046sub calculate_localtime_offset {
1047 # Assume that the offset at the moment is valid across the whole
1048 # period covered by the logs that we're analysing. This may not
1049 # be true around the time the clocks change in spring or autumn.
1050 my $utc = time;
1051 # mktime works on local time and gmtime works in UTC
1052 my $local = mktime(gmtime($utc));
1053 return $local - $utc;
1054}
1055
1056
608bc29d 1057
1b4fe9dd 1058#######################################################################
608bc29d
SC
1059# print_duration_table();
1060#
1061# print_duration_table($title, $message_type, \@times, \@values, $overflow);
1062#
1063# Print a table showing how long a particular step took for
1064# the messages. The parameters are:
1065# $title Eg "Time spent on the queue"
1066# $message_type Eg "Remote"
1067# \@times The maximum time a message took for it to increment
1068# the corresponding @values counter.
1069# \@values An array of message counters.
1070# $overflow The number of messages which exceeded the maximum
1071# time.
1b4fe9dd 1072#######################################################################
608bc29d 1073sub print_duration_table {
059ec3d9 1074no integer;
608bc29d 1075my($title, $message_type, $times_aref, $values_aref, $overflow) = @_;
059ec3d9
PH
1076my(@chartdatanames);
1077my(@chartdatavals);
1078
1079my $printed_one = 0;
1080my $cumulative_percent = 0;
059ec3d9 1081
608bc29d
SC
1082my $queue_total = $overflow;
1083map {$queue_total += $_} @$values_aref;
059ec3d9 1084
608bc29d 1085my $temp = "$title: $message_type";
059ec3d9 1086
d5692f86
SC
1087
1088my $txt_format = "%5s %4s %6d %5.1f%% %5.1f%%\n";
1089my $htm_format = "<tr><td align=\"right\">%s %s</td><td align=\"right\">%d</td><td align=\"right\">%5.1f%%</td><td align=\"right\">%5.1f%%</td>\n";
1090
1091# write header
1092printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
1093if ($htm_fh) {
608bc29d 1094 print $htm_fh "<hr><a name=\"$title $message_type\"></a><h2>$temp</h2>\n";
a83c7e95 1095 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
d5692f86 1096 print $htm_fh "<tr><th>Time</th><th>Messages</th><th>Percentage</th><th>Cumulative Percentage</th>\n";
059ec3d9 1097}
a83c7e95 1098if ($xls_fh) {
608bc29d 1099 $ws_global->write($row++, $col, "$title: ".$message_type, $f_header2);
d5692f86
SC
1100 my @content=("Time", "Messages", "Percentage", "Cumulative Percentage");
1101 &set_worksheet_line($ws_global, $row++, 1, \@content, $f_headertab);
059ec3d9
PH
1102}
1103
d5692f86 1104
608bc29d
SC
1105for ($i = 0; $i <= $#$times_aref; ++$i) {
1106 if ($$values_aref[$i] > 0)
059ec3d9 1107 {
608bc29d 1108 my $percent = ($values_aref->[$i] * 100)/$queue_total;
059ec3d9 1109 $cumulative_percent += $percent;
d5692f86
SC
1110
1111 my @content=($printed_one? " " : "Under",
608bc29d
SC
1112 format_time($times_aref->[$i]),
1113 $values_aref->[$i], $percent, $cumulative_percent);
d5692f86
SC
1114
1115 if ($htm_fh) {
1116 printf $htm_fh ($htm_format, @content);
608bc29d 1117 if (!defined($values_aref->[$i])) {
d5692f86
SC
1118 print $htm_fh "Not defined";
1119 }
1120 }
1121 if ($txt_fh) {
1122 printf $txt_fh ($txt_format, @content);
608bc29d 1123 if (!defined($times_aref->[$i])) {
d5692f86
SC
1124 print $txt_fh "Not defined";
1125 }
1126 }
1127 if ($xls_fh)
1128 {
1129 no integer;
1130 &set_worksheet_line($ws_global, $row, 0, [@content[0,1,2]], $f_default);
1131 &set_worksheet_line($ws_global, $row++, 3, [$content[3]/100,$content[4]/100], $f_percent);
1132
608bc29d 1133 if (!defined($times_aref->[$i])) {
d5692f86
SC
1134 $col=0;
1135 $ws_global->write($row++, $col, "Not defined" );
1136 }
059ec3d9 1137 }
d5692f86 1138
059ec3d9 1139 push(@chartdatanames,
608bc29d
SC
1140 ($printed_one? "" : "Under") . format_time($times_aref->[$i]));
1141 push(@chartdatavals, $$values_aref[$i]);
059ec3d9
PH
1142 $printed_one = 1;
1143 }
1144}
1145
608bc29d
SC
1146if ($overflow && $overflow > 0) {
1147 my $percent = ($overflow * 100)/$queue_total;
059ec3d9 1148 $cumulative_percent += $percent;
d5692f86 1149
608bc29d
SC
1150 my @content = ("Over ", format_time($times_aref->[-1]),
1151 $overflow, $percent, $cumulative_percent);
d5692f86
SC
1152
1153 printf $txt_fh ($txt_format, @content) if $txt_fh;
1154 printf $htm_fh ($htm_format, @content) if $htm_fh;
1155 if ($xls_fh)
1156 {
1157 &set_worksheet_line($ws_global, $row, 0, [@content[0,1,2]], $f_default);
1158 &set_worksheet_line($ws_global, $row++, 3, [$content[3]/100,$content[4]/100], $f_percent);
1159 }
1160
059ec3d9 1161}
d5692f86 1162
608bc29d
SC
1163push(@chartdatanames, "Over " . format_time($times_aref->[-1]));
1164push(@chartdatavals, $overflow);
059ec3d9
PH
1165
1166#printf("Unknown %6d\n", $queue_unknown) if $queue_unknown > 0;
d5692f86 1167if ($htm_fh) {
a83c7e95 1168 print $htm_fh "</table></td><td>";
059ec3d9 1169
608bc29d 1170 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0)) {
059ec3d9
PH
1171 my @data = (
1172 \@chartdatanames,
1173 \@chartdatavals
1174 );
1175 my $graph = GD::Graph::pie->new(200, 200);
608bc29d
SC
1176 my $pngname = "$title-$message_type.png";
1177 $pngname =~ s/[^\w\-\.]/_/;
1178
1179 my $graph_title = "$title ($message_type)";
1180 $graph->set(title => $graph_title) if (length($graph_title) < 21);
1181
059ec3d9
PH
1182 my $gd = $graph->plot(\@data) or warn($graph->error);
1183 if ($gd) {
d5692f86 1184 open(IMG, ">$chartdir/$pngname") or die "Could not write $chartdir/$pngname: $!\n";
059ec3d9
PH
1185 binmode IMG;
1186 print IMG $gd->png;
1187 close IMG;
d5692f86 1188 print $htm_fh "<img src=\"$chartrel/$pngname\">";
059ec3d9
PH
1189 }
1190 }
d5692f86
SC
1191 print $htm_fh "</td></tr></table>\n";
1192}
1193
1194if ($xls_fh)
1195{
1196 $row++;
059ec3d9 1197}
d5692f86
SC
1198print $txt_fh "\n" if $txt_fh;
1199print $htm_fh "\n" if $htm_fh;
1200
059ec3d9
PH
1201}
1202
1203
1b4fe9dd
PH
1204#######################################################################
1205# print_histogram();
8e669ac1 1206#
608bc29d 1207# print_histogram('Deliveries|Messages received|$pattern', $unit, @interval_count);
8e669ac1 1208#
1b4fe9dd
PH
1209# Print a histogram of the messages delivered/received per time slot
1210# (hour by default).
1211#######################################################################
059ec3d9 1212sub print_histogram {
608bc29d 1213my($text, $unit, @interval_count) = @_;
059ec3d9
PH
1214my(@chartdatanames);
1215my(@chartdatavals);
1216my($maxd) = 0;
d5692f86 1217
a83c7e95
SC
1218# save first row of print_histogram for xls output
1219if (!$run_hist) {
d5692f86
SC
1220 $row_hist = $row;
1221}
a83c7e95 1222else {
d5692f86
SC
1223 $row = $row_hist;
1224}
1225
059ec3d9
PH
1226for ($i = 0; $i < $hist_number; $i++)
1227 { $maxd = $interval_count[$i] if $interval_count[$i] > $maxd; }
1228
1229my $scale = int(($maxd + 25)/50);
1230$scale = 1 if $scale == 0;
1231
608bc29d
SC
1232if ($scale != 1) {
1233 if ($unit !~ s/y$/ies/) {
1234 $unit .= 's';
1235 }
d5692f86
SC
1236}
1237
1238# make and output title
1239my $title = sprintf("$text per %s",
1240 ($hist_interval == 60)? "hour" :
1241 ($hist_interval == 1)? "minute" : "$hist_interval minutes");
059ec3d9 1242
608bc29d 1243my $txt_htm_title = $title . " (each dot is $scale $unit)";
059ec3d9 1244
d5692f86
SC
1245printf $txt_fh ("%s\n%s\n\n", $txt_htm_title, "-" x length($txt_htm_title)) if $txt_fh;
1246
1247if ($htm_fh) {
1248 print $htm_fh "<hr><a name=\"$text\"></a><h2>$txt_htm_title</h2>\n";
1249 print $htm_fh "<table border=0 width=\"100%\">\n";
1250 print $htm_fh "<tr><td><pre>\n";
059ec3d9 1251}
d5692f86 1252
a83c7e95 1253if ($xls_fh) {
d5692f86 1254 $title =~ s/Messages/Msg/ ;
a83c7e95 1255 $row += 2;
d5692f86 1256 $ws_global->write($row++, $col_hist+1, $title, $f_headertab);
059ec3d9
PH
1257}
1258
d5692f86 1259
059ec3d9
PH
1260my $hour = 0;
1261my $minutes = 0;
a83c7e95 1262for ($i = 0; $i < $hist_number; $i++) {
059ec3d9
PH
1263 my $c = $interval_count[$i];
1264
1265 # If the interval is an hour (the maximum) print the starting and
1266 # ending hours as a label. Otherwise print the starting hour and
1267 # minutes, which take up the same space.
1268
1269 my $temp;
a83c7e95 1270 if ($hist_opt == 1) {
059ec3d9 1271 $temp = sprintf("%02d-%02d", $hour, $hour + 1);
d5692f86
SC
1272
1273 print $txt_fh $temp if $txt_fh;
1274 print $htm_fh $temp if $htm_fh;
1275
a83c7e95
SC
1276 if ($xls_fh) {
1277 if ($run_hist==0) {
1278 # only on first run
1279 $ws_global->write($row, 0, [$temp], $f_default);
d5692f86
SC
1280 }
1281 }
1282
059ec3d9
PH
1283 push(@chartdatanames, $temp);
1284 $hour++;
d5692f86 1285 }
a83c7e95 1286 else {
059ec3d9
PH
1287 if ($minutes == 0)
1288 { $temp = sprintf("%02d:%02d", $hour, $minutes) }
1289 else
1290 { $temp = sprintf(" :%02d", $minutes) }
d5692f86
SC
1291
1292 print $txt_fh $temp if $txt_fh;
1293 print $htm_fh $temp if $htm_fh;
a83c7e95
SC
1294 if (($xls_fh) and ($run_hist==0)) {
1295 # only on first run
d5692f86 1296 $temp = sprintf("%02d:%02d", $hour, $minutes);
a83c7e95 1297 $ws_global->write($row, 0, [$temp], $f_default);
d5692f86
SC
1298 }
1299
059ec3d9
PH
1300 push(@chartdatanames, $temp);
1301 $minutes += $hist_interval;
a83c7e95 1302 if ($minutes >= 60) {
059ec3d9
PH
1303 $minutes = 0;
1304 $hour++;
a83c7e95 1305 }
059ec3d9 1306 }
d5692f86
SC
1307 push(@chartdatavals, $c);
1308
1309 printf $txt_fh (" %6d %s\n", $c, "." x ($c/$scale)) if $txt_fh;
1310 printf $htm_fh (" %6d %s\n", $c, "." x ($c/$scale)) if $htm_fh;
a83c7e95 1311 $ws_global->write($row++, $col_hist+1, [$c], $f_default) if $xls_fh;
d5692f86
SC
1312
1313} #end for
1314
1315printf $txt_fh "\n" if $txt_fh;
1316printf $htm_fh "\n" if $htm_fh;
1317
1318if ($htm_fh)
1319{
1320 print $htm_fh "</pre>\n";
1321 print $htm_fh "</td><td>\n";
608bc29d 1322 if ($HAVE_GD_Graph_linespoints && $charts && ($#chartdatavals > 0)) {
059ec3d9
PH
1323 # calculate the graph
1324 my @data = (
1325 \@chartdatanames,
1326 \@chartdatavals
1327 );
1328 my $graph = GD::Graph::linespoints->new(300, 300);
1329 $graph->set(
1330 x_label => 'Time',
1331 y_label => 'Amount',
1332 title => $text,
1333 x_labels_vertical => 1
1334 );
608bc29d
SC
1335 my $pngname = "histogram_$text.png";
1336 $pngname =~ s/[^\w\._]/_/g;
1337
059ec3d9
PH
1338 my $gd = $graph->plot(\@data) or warn($graph->error);
1339 if ($gd) {
d5692f86 1340 open(IMG, ">$chartdir/$pngname") or die "Could not write $chartdir/$pngname: $!\n";
059ec3d9
PH
1341 binmode IMG;
1342 print IMG $gd->png;
1343 close IMG;
d5692f86 1344 print $htm_fh "<img src=\"$chartrel/$pngname\">";
059ec3d9
PH
1345 }
1346 }
d5692f86 1347 print $htm_fh "</td></tr></table>\n";
059ec3d9 1348}
d5692f86
SC
1349
1350$col_hist++; # where to continue next times
1351
1352$row+=2; # leave some space after history block
1353$run_hist=1; # we have done this once or more
059ec3d9
PH
1354}
1355
1356
1357
1b4fe9dd
PH
1358#######################################################################
1359# print_league_table();
8e669ac1 1360#
a83c7e95 1361# print_league_table($league_table_type,\%message_count,\%address_count,\%message_data,\%message_data_gigs, $spreadsheet, $row_sref);
8e669ac1 1362#
a83c7e95
SC
1363# Given hashes of message count, address count, and message data,
1364# which are keyed by the table type (eg by the sending host), print a
1365# league table showing the top $topcount (defaults to 50).
1b4fe9dd 1366#######################################################################
059ec3d9 1367sub print_league_table {
a83c7e95
SC
1368 my($text,$m_count,$a_count,$m_data,$m_data_gigs,$spreadsheet, $row_sref) = @_;
1369 my($name) = ($topcount == 1)? "$text" : "$topcount ${text}s";
1370 my($title) = "Top $name by message count";
1371 my(@chartdatanames) = ();
1372 my(@chartdatavals) = ();
1373 my $chartotherval = 0;
1374 $text = ucfirst($text);
059ec3d9
PH
1375
1376 # Align non-local addresses to the right (so all the .com's line up).
1377 # Local addresses are aligned on the left as they are userids.
1378 my $align = ($text !~ /local/i) ? 'right' : 'left';
059ec3d9 1379
d5692f86 1380
a83c7e95
SC
1381 ################################################
1382 # Generate the printf formats and table headers.
1383 ################################################
1384 my(@headers) = ('Messages');
30e9ac3f
SC
1385 #push(@headers,'Addresses') if defined $a_count;
1386 push(@headers,'Addresses') if defined $a_count && %$a_count;
a83c7e95 1387 push(@headers,'Bytes','Average') if defined $m_data;
d5692f86 1388
a83c7e95
SC
1389 my $txt_format = "%10s " x @headers . " %s\n";
1390 my $txt_col_headers = sprintf $txt_format, @headers, $text;
1391 my $htm_format = "<tr>" . '<td align="right">%s</td>'x@headers . "<td align=\"$align\" nowrap>%s</td></tr>\n";
1392 my $htm_col_headers = sprintf $htm_format, @headers, $text;
1393 $htm_col_headers =~ s/(<\/?)td/$1th/g; #Convert <td>'s to <th>'s for the header.
d5692f86 1394
a83c7e95
SC
1395
1396 ################################################
1397 # Write the table headers
1398 ################################################
1399 printf $txt_fh ("%s\n%s\n%s", $title, "-" x length($title),$txt_col_headers) if $txt_fh;
d5692f86
SC
1400
1401 if ($htm_fh) {
a83c7e95
SC
1402 print $htm_fh <<EoText;
1403<hr><a name="$text count"></a><h2>$title</h2>
1404<table border=0 width="100%">
1405<tr><td>
1406<table border=1>
1407EoText
30e9ac3f 1408 print $htm_fh $htm_col_headers
059ec3d9 1409 }
d5692f86 1410
a83c7e95
SC
1411 if ($xls_fh) {
1412 $spreadsheet->write(${$row_sref}++, 0, $title, $f_header2);
1413 $spreadsheet->write(${$row_sref}++, 0, [@headers, $text], $f_headertab);
d5692f86 1414 }
a83c7e95
SC
1415
1416
1417 # write content
1418 foreach my $key (top_n_sort($topcount,$m_count,$m_data_gigs,$m_data)) {
1419
1420 # When displaying the average figures, we calculate the average of
1421 # the rounded data, as the user would calculate it. This reduces
1422 # the accuracy slightly, but we have to do it this way otherwise
1423 # when using -merge to convert results from text to HTML and
1424 # vice-versa discrepencies would occur.
1425 my $messages = $$m_count{$key};
1426 my @content = ($messages);
1427 push(@content, $$a_count{$key}) if defined $a_count;
1428 if (defined $m_data) {
1429 my $rounded_volume = volume_rounded($$m_data{$key},$$m_data_gigs{$key});
1430 my($data,$gigs) = (0,0);
1431 un_round($rounded_volume,\$data,\$gigs);
1432 my $rounded_average = volume_rounded($data/$messages,$gigs/$messages);
1433 push(@content, $rounded_volume, $rounded_average);
1434 }
1435
1436 # write content
1437 printf $txt_fh ($txt_format, @content, $key) if $txt_fh;
1438
1439 if ($htm_fh) {
1440 my $htmlkey = $key;
1441 $htmlkey =~ s/>/\&gt\;/g;
1442 $htmlkey =~ s/</\&lt\;/g;
1443 printf $htm_fh ($htm_format, @content, $htmlkey);
1444 }
1445 $spreadsheet->write(${$row_sref}++, 0, [@content, $key], $f_default) if $xls_fh;
1446
1447 if (scalar @chartdatanames < $ntopchart) {
1448 push(@chartdatanames, $key);
1449 push(@chartdatavals, $$m_count{$key});
1450 }
1451 else {
1452 $chartotherval += $$m_count{$key};
1453 }
059ec3d9 1454 }
d5692f86 1455
a83c7e95
SC
1456 push(@chartdatanames, "Other");
1457 push(@chartdatavals, $chartotherval);
059ec3d9 1458
a83c7e95
SC
1459 print $txt_fh "\n" if $txt_fh;
1460 if ($htm_fh) {
1461 print $htm_fh "</table>\n";
1462 print $htm_fh "</td><td>\n";
1463 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0))
1464 {
1465 # calculate the graph
1466 my @data = (
1467 \@chartdatanames,
1468 \@chartdatavals
1469 );
1470 my $graph = GD::Graph::pie->new(300, 300);
1471 $graph->set(
1472 x_label => 'Name',
1473 y_label => 'Amount',
1474 title => 'By count',
1475 );
1476 my $gd = $graph->plot(\@data) or warn($graph->error);
1477 if ($gd) {
1478 my $temp = $text;
1479 $temp =~ s/ /_/g;
1480 open(IMG, ">$chartdir/${temp}_count.png") or die "Could not write $chartdir/${temp}_count.png: $!\n";
1481 binmode IMG;
1482 print IMG $gd->png;
1483 close IMG;
1484 print $htm_fh "<img src=\"$chartrel/${temp}_count.png\">";
1485 }
059ec3d9 1486 }
a83c7e95
SC
1487 print $htm_fh "</td><td>\n";
1488 print $htm_fh "</td></tr></table>\n\n";
059ec3d9 1489 }
a83c7e95 1490 ++${$row_sref} if $xls_fh;
d5692f86
SC
1491
1492
a83c7e95
SC
1493 if (defined $m_data) {
1494 # write header
059ec3d9 1495
a83c7e95 1496 $title = "Top $name by volume";
d5692f86 1497
a83c7e95 1498 printf $txt_fh ("%s\n%s\n%s", $title, "-" x length($title),$txt_col_headers) if $txt_fh;
059ec3d9 1499
a83c7e95
SC
1500 if ($htm_fh) {
1501 print $htm_fh <<EoText;
1502<hr><a name="$text volume"></a><h2>$title</h2>
1503<table border=0 width="100%">
1504<tr><td>
1505<table border=1>
1506EoText
30e9ac3f 1507 print $htm_fh $htm_col_headers;
a83c7e95
SC
1508 }
1509 if ($xls_fh) {
1510 $spreadsheet->write(${$row_sref}++, 0, $title, $f_header2);
1511 $spreadsheet->write(${$row_sref}++, 0, [@headers, $text], $f_headertab);
1512 }
1513
1514 @chartdatanames = ();
1515 @chartdatavals = ();
1516 $chartotherval = 0;
1517 my $use_gig = 0;
1518 foreach my $key (top_n_sort($topcount,$m_data_gigs,$m_data,$m_count)) {
1519 # The largest volume will be the first (top of the list).
1520 # If it has at least 1 gig, then just use gigabytes to avoid
1521 # risking an integer overflow when generating the pie charts.
1522 if ($$m_data_gigs{$key}) {
1523 $use_gig = 1;
1524 }
d5692f86 1525
a83c7e95
SC
1526 my $messages = $$m_count{$key};
1527 my @content = ($messages);
1528 push(@content, $$a_count{$key}) if defined $a_count;
1529 my $rounded_volume = volume_rounded($$m_data{$key},$$m_data_gigs{$key});
1530 my($data ,$gigs) = (0,0);
1531 un_round($rounded_volume,\$data,\$gigs);
1532 my $rounded_average = volume_rounded($data/$messages,$gigs/$messages);
1533 push(@content, $rounded_volume, $rounded_average );
1534
1535 # write content
1536 printf $txt_fh ($txt_format, @content, $key) if $txt_fh;
1537 if ($htm_fh) {
1538 my $htmlkey = $key;
1539 $htmlkey =~ s/>/\&gt\;/g;
1540 $htmlkey =~ s/</\&lt\;/g;
1541 printf $htm_fh ($htm_format, @content, $htmlkey);
1542 }
1543 $spreadsheet->write(${$row_sref}++, 0, [@content, $key], $f_default) if $xls_fh;
059ec3d9 1544
d5692f86 1545
a83c7e95
SC
1546 if (scalar @chartdatanames < $ntopchart) {
1547 if ($use_gig) {
1548 if ($$m_data_gigs{$key}) {
1549 push(@chartdatanames, $key);
1550 push(@chartdatavals, $$m_data_gigs{$key});
1551 }
1552 }
1553 else {
1554 push(@chartdatanames, $key);
1555 push(@chartdatavals, $$m_data{$key});
1556 }
1557 }
1558 else {
1559 $chartotherval += ($use_gig) ? $$m_data_gigs{$key} : $$m_data{$key};
8974000d 1560 }
059ec3d9 1561 }
a83c7e95
SC
1562 push(@chartdatanames, "Other");
1563 push(@chartdatavals, $chartotherval);
059ec3d9 1564
a83c7e95
SC
1565 print $txt_fh "\n" if $txt_fh;
1566 if ($htm_fh) {
1567 print $htm_fh "</table>\n";
1568 print $htm_fh "</td><td>\n";
1569 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0)) {
1570 # calculate the graph
1571 my @data = (
1572 \@chartdatanames,
1573 \@chartdatavals
1574 );
1575 my $graph = GD::Graph::pie->new(300, 300);
1576 $graph->set(
1577 x_label => 'Name',
1578 y_label => 'Volume' ,
1579 title => 'By Volume',
1580 );
1581 my $gd = $graph->plot(\@data) or warn($graph->error);
1582 if ($gd) {
1583 my $temp = $text;
1584 $temp =~ s/ /_/g;
1585 open(IMG, ">$chartdir/${temp}_volume.png") or die "Could not write $chartdir/${temp}_volume.png: $!\n";
1586 binmode IMG;
1587 print IMG $gd->png;
1588 close IMG;
1589 print $htm_fh "<img src=\"$chartrel/${temp}_volume.png\">";
1590 }
1591 }
1592 print $htm_fh "</td><td>\n";
1593 print $htm_fh "</td></tr></table>\n\n";
059ec3d9 1594 }
a83c7e95
SC
1595
1596 ++${$row_sref} if $xls_fh;
059ec3d9 1597 }
059ec3d9
PH
1598}
1599
1600
1b4fe9dd
PH
1601#######################################################################
1602# top_n_sort();
8e669ac1 1603#
1b4fe9dd 1604# @sorted_keys = top_n_sort($n,$href1,$href2,$href3);
8e669ac1 1605#
1b4fe9dd
PH
1606# Given a hash which has numerical values, return the sorted $n keys which
1607# point to the top values. The second and third hashes are used as
1608# tiebreakers. They all must have the same keys.
8e669ac1 1609#
1b4fe9dd
PH
1610# The idea behind this routine is that when you only want to see the
1611# top n members of a set, rather than sorting the entire set and then
1612# plucking off the top n, sort through the stack as you go, discarding
1613# any member which is lower than your current n'th highest member.
8e669ac1 1614#
1b4fe9dd
PH
1615# This proves to be an order of magnitude faster for large hashes.
1616# On 200,000 lines of mainlog it benchmarked 9 times faster.
1617# On 700,000 lines of mainlog it benchmarked 13.8 times faster.
8e669ac1 1618#
1b4fe9dd
PH
1619# We assume the values are > 0.
1620#######################################################################
059ec3d9
PH
1621sub top_n_sort {
1622 my($n,$href1,$href2,$href3) = @_;
1623
1624 # PH's original sort was:
1625 #
1626 # foreach $key (sort
1627 # {
1628 # $$m_count{$b} <=> $$m_count{$a} ||
1629 # $$m_data_gigs{$b} <=> $$m_data_gigs{$a} ||
1630 # $$m_data{$b} <=> $$m_data{$a} ||
1631 # $a cmp $b
1632 # }
1633 # keys %{$m_count})
1634 #
1635
1636 #We use a key of '_' to represent non-existant values, as null keys are valid.
1637 #'_' is not a valid domain, edomain, host, or email.
1638 my(@top_n_keys) = ('_') x $n;
1639 my($minimum_value1,$minimum_value2,$minimum_value3) = (0,0,0);
1640 my $top_n_key = '';
1641 my $n_minus_1 = $n - 1;
1642 my $n_minus_2 = $n - 2;
1643
a83c7e95
SC
1644 # Create a dummy hash incase the user has not provided us with
1645 # tiebreaker hashes.
1646 my(%dummy_hash);
1647 $href2 = \%dummy_hash unless defined $href2;
1648 $href3 = \%dummy_hash unless defined $href3;
1649
8e669ac1 1650 # Pick out the top $n keys.
059ec3d9
PH
1651 my($key,$value1,$value2,$value3,$i,$comparison,$insert_position);
1652 while (($key,$value1) = each %$href1) {
1653
1654 #print STDERR "key $key ($value1,",$href2->{$key},",",$href3->{$key},") <=> ($minimum_value1,$minimum_value2,$minimum_value3)\n";
8e669ac1 1655
059ec3d9 1656 # Check to see that the new value is bigger than the lowest of the
a83c7e95
SC
1657 # top n keys that we're keeping. We test the main key first, because
1658 # for the majority of cases we can skip creating dummy hash values
1659 # should the user have not provided real tie-breaking hashes.
1660 next unless $value1 >= $minimum_value1;
1661
1662 # Create a dummy hash entry for the key if required.
1663 # Note that setting the dummy_hash value sets it for both href2 &
4c04137d 1664 # href3. Also note that currently we are guaranteed to have a real
a83c7e95
SC
1665 # value for href3 if a real value for href2 exists so don't need to
1666 # test for it as well.
1667 $dummy_hash{$key} = 0 unless exists $href2->{$key};
1668
8e669ac1 1669 $comparison = $value1 <=> $minimum_value1 ||
d5692f86
SC
1670 $href2->{$key} <=> $minimum_value2 ||
1671 $href3->{$key} <=> $minimum_value3 ||
1672 $top_n_key cmp $key;
059ec3d9
PH
1673 next unless ($comparison == 1);
1674
1675 # As we will be using these values a few times, extract them into scalars.
1676 $value2 = $href2->{$key};
1677 $value3 = $href3->{$key};
1678
1679 # This key is bigger than the bottom n key, so the lowest position we
1680 # will insert it into is $n minus 1 (the bottom of the list).
1681 $insert_position = $n_minus_1;
1682
1683 # Now go through the list, stopping when we find a key that we're
1684 # bigger than, or we come to the penultimate position - we've
1685 # already tested bigger than the last.
1686 #
1687 # Note: we go top down as the list starts off empty.
1688 # Note: stepping through the list in this way benchmarks nearly
1689 # three times faster than doing a sort() on the reduced list.
1690 # I assume this is because the list is already in order, and
1691 # we get a performance boost from not having to do hash lookups
1692 # on the new key.
1693 for ($i = 0; $i < $n_minus_1; $i++) {
1694 $top_n_key = $top_n_keys[$i];
1695 if ( ($top_n_key eq '_') ||
d5692f86 1696 ( ($value1 <=> $href1->{$top_n_key} ||
059ec3d9 1697 $value2 <=> $href2->{$top_n_key} ||
d5692f86
SC
1698 $value3 <=> $href3->{$top_n_key} ||
1699 $top_n_key cmp $key) == 1
1700 )
1701 ) {
1702 $insert_position = $i;
1703 last;
059ec3d9
PH
1704 }
1705 }
1706
1707 # Remove the last element, then insert the new one.
1708 $#top_n_keys = $n_minus_2;
1709 splice(@top_n_keys,$insert_position,0,$key);
1710
1711 # Extract our new minimum values.
1712 $top_n_key = $top_n_keys[$n_minus_1];
1713 if ($top_n_key ne '_') {
1714 $minimum_value1 = $href1->{$top_n_key};
1715 $minimum_value2 = $href2->{$top_n_key};
1716 $minimum_value3 = $href3->{$top_n_key};
1717 }
1718 }
1719
1720 # Return the top n list, grepping out non-existant values, just in case
1721 # we didn't have that many values.
1722 return(grep(!/^_$/,@top_n_keys));
1723}
1724
1725
a83c7e95 1726
1b4fe9dd
PH
1727#######################################################################
1728# html_header();
8e669ac1 1729#
1b4fe9dd 1730# $header = html_header($title);
8e669ac1 1731#
1b4fe9dd
PH
1732# Print our HTML header and start the <body> block.
1733#######################################################################
059ec3d9
PH
1734sub html_header {
1735 my($title) = @_;
1736 my $text = << "EoText";
1737<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
1738<html>
1739<head>
1740<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-15">
1741<title>$title</title>
1742</head>
1743<body bgcolor="white">
1744<h1>$title</h1>
1745EoText
1746 return $text;
1747}
1748
1749
1750
1b4fe9dd
PH
1751#######################################################################
1752# help();
8e669ac1 1753#
1b4fe9dd 1754# help();
8e669ac1 1755#
1b4fe9dd
PH
1756# Display usage instructions and exit.
1757#######################################################################
059ec3d9
PH
1758sub help {
1759 print << "EoText";
1760
1761eximstats Version $VERSION
1762
d5692f86
SC
1763Usage:
1764 eximstats [Output] [Options] mainlog1 mainlog2 ...
1765 eximstats -merge -html [Options] report.1.html ... > weekly_rep.html
1766
1767Examples:
1768 eximstats -html=eximstats.html mainlog1 mainlog2 ...
1769 eximstats mainlog1 mainlog2 ... > report.txt
059ec3d9 1770
8974000d 1771Parses exim mainlog or syslog files and generates a statistical analysis
d5692f86
SC
1772of the messages processed.
1773
1774Valid output types are:
1775-txt[=<file>] plain text (default unless no other type is specified)
1776-html[=<file>] HTML
1777-xls[=<file>] Excel
1778With no type and file given, defaults to -txt and STDOUT.
059ec3d9 1779
d5692f86 1780Valid options are:
059ec3d9
PH
1781-h<number> histogram divisions per hour. The default is 1, and
1782 0 suppresses histograms. Other valid values are:
d5692f86 1783 2, 3, 5, 10, 15, 20, 30 or 60.
059ec3d9
PH
1784-ne don't display error information
1785-nr don't display relaying information
1786-nr/pattern/ don't display relaying information that matches
1787-nt don't display transport information
1788-nt/pattern/ don't display transport information that matches
d5692f86 1789-nvr don't do volume rounding. Display in bytes, not KB/MB/GB.
059ec3d9
PH
1790-t<number> display top <number> sources/destinations
1791 default is 50, 0 suppresses top listing
1792-tnl omit local sources/destinations in top listing
1793-t_remote_users show top user sources/destinations from non-local domains
608bc29d
SC
1794-q<list> list of times for queuing information. -q0 suppresses.
1795-show_rt<list> Show the receipt times for all the messages.
1796-show_dt<list> Show the delivery times for all the messages.
1797 <list> is an optional list of times in seconds.
1798 Eg -show_rt1,2,4,8.
1799
d5692f86
SC
1800-include_original_destination show both the final and original
1801 destinations in the results rather than just the final ones.
059ec3d9 1802
d5692f86 1803-byhost show results by sending host (default unless bydomain or
059ec3d9 1804 byemail is specified)
d5692f86
SC
1805-bydomain show results by sending domain.
1806-byemail show results by sender's email address
1807-byedomain show results by sender's email domain
37f8a7c9 1808-bylocaldomain show results by local domain
059ec3d9
PH
1809
1810-pattern "Description" /pattern/
1811 Count lines matching specified patterns and show them in
d5692f86
SC
1812 the results. It can be specified multiple times. Eg:
1813 -pattern 'Refused connections' '/refused connection/'
059ec3d9
PH
1814
1815-merge merge previously generated reports into a new report
1816
d5692f86
SC
1817-charts Create charts (this requires the GD::Graph modules).
1818 Only valid with -html.
059ec3d9
PH
1819-chartdir <dir> Create the charts' png files in the directory <dir>
1820-chartrel <dir> Specify the relative directory for the "img src=" tags
1821 from where to include the charts in the html file
d5692f86 1822 -chartdir and -chartrel default to '.'
059ec3d9 1823
a83c7e95
SC
1824-emptyok It is OK if there is no valid input, don't print an error.
1825
d5692f86 1826-d Debug mode - dump the eval'ed parser onto STDERR.
059ec3d9
PH
1827
1828EoText
1829
1830 exit 1;
1831}
1832
1833
1834
1b4fe9dd
PH
1835#######################################################################
1836# generate_parser();
8e669ac1 1837#
1b4fe9dd 1838# $parser = generate_parser();
8e669ac1 1839#
1b4fe9dd
PH
1840# This subroutine generates the parsing routine which will be
1841# used to parse the mainlog. We take the base operation, and remove bits not in use.
1842# This improves performance depending on what bits you take out or add.
8e669ac1 1843#
1b4fe9dd 1844# I've tested using study(), but this does not improve performance.
8e669ac1 1845#
1b4fe9dd
PH
1846# We store our parsing routing in a variable, and process it looking for #IFDEF (Expression)
1847# or #IFNDEF (Expression) statements and corresponding #ENDIF (Expression) statements. If
1848# the expression evaluates to true, then it is included/excluded accordingly.
1849#######################################################################
059ec3d9
PH
1850sub generate_parser {
1851 my $parser = '
1852 my($ip,$host,$email,$edomain,$domain,$thissize,$size,$old,$new);
a83c7e95 1853 my($tod,$m_hour,$m_min,$id,$flag,$extra,$length);
37f8a7c9 1854 my($seconds,$queued,$rcpt_time,$local_domain);
a83c7e95 1855 my $rej_id = 0;
059ec3d9 1856 while (<$fh>) {
059ec3d9 1857
8974000d
SC
1858 # Convert syslog lines to mainlog format.
1859 if (! /^\\d{4}/) {
1860 next unless s/^.*? exim\\b.*?: //;
1861 }
1862
a83c7e95
SC
1863 $length = length($_);
1864 next if ($length < 38);
571b2715
JH
1865 next unless /^
1866 (\\d{4}\\-\\d\\d-\\d\\d\\s # 1: YYYYMMDD HHMMSS
1867 (\\d\\d) # 2: HH
1868 :
1869 (\\d\\d) # 3: MM
1870 :\\d\\d
1871 )
1872 (\\.\\d+)? # 4: subseconds
1873 (\s[-+]\\d\\d\\d\\d)? # 5: tz-offset
1874 (\s\\[\\d+\\])? # 6: pid
1875 /ox;
1876
1877 $tod = defined($5) ? $1 . $5 : $1;
1878 ($m_hour,$m_min) = ($2,$3);
059ec3d9 1879
a83c7e95 1880 # PH - watch for GMT offsets in the timestamp.
571b2715 1881 if (defined($5)) {
a83c7e95
SC
1882 $extra = 6;
1883 next if ($length < 44);
1884 }
1885 else {
1886 $extra = 0;
1887 }
1888
571b2715
JH
1889 # watch for subsecond precision
1890 if (defined($4)) {
1891 $extra += length($4);
1892 next if ($length < 38 + $extra);
1893 }
1894
f3f065bb 1895 # PH - watch for PID added after the timestamp.
571b2715
JH
1896 if (defined($6)) {
1897 $extra += length($6);
f3f065bb
PH
1898 next if ($length < 38 + $extra);
1899 }
1900
059ec3d9
PH
1901 $id = substr($_, 20 + $extra, 16);
1902 $flag = substr($_, 37 + $extra, 2);
a83c7e95
SC
1903
1904 if ($flag !~ /^([<>=*-]+|SA)$/ && /rejected|refused|dropped/) {
1905 $flag = "Re";
1906 $extra -= 3;
1907 }
1908
1909 # Rejects can have no MSGID...
1910 if ($flag eq "Re" && $id !~ /^[-0-9a-zA-Z]+$/) {
1911 $id = "reject:" . ++$rej_id;
1912 $extra -= 17;
1913 }
059ec3d9
PH
1914';
1915
1916 # Watch for user specified patterns.
1917 my $user_pattern_index = 0;
1918 foreach (@user_patterns) {
1919 $user_pattern_totals[$user_pattern_index] = 0;
f2b67a5b
SC
1920 $parser .= " if ($_) {\n";
1921 $parser .= " \$user_pattern_totals[$user_pattern_index]++;\n";
1922 $parser .= " \$user_pattern_interval_count[$user_pattern_index][(\$m_hour*60 + \$m_min)/$hist_interval]++;\n" if ($hist_opt > 0);
1923 $parser .= " }\n";
059ec3d9
PH
1924 $user_pattern_index++;
1925 }
1926
1927 $parser .= '
a83c7e95 1928 next unless ($flag =~ /<=|=>|->|==|\\*\\*|Co|SA|Re/);
059ec3d9 1929
a83c7e95
SC
1930 #Strip away the timestamp, ID and flag to speed up later pattern matches.
1931 #The flags include Co (Completed), Re (Rejected), and SA (SpamAssassin).
059ec3d9
PH
1932 $_ = substr($_, 40 + $extra); # PH
1933
a83c7e95 1934 # Alias @message to the array of information about the message.
608bc29d
SC
1935 # This minimises the number of calls to hash functions.
1936 $messages{$id} = [] unless exists $messages{$id};
a83c7e95 1937 *message = $messages{$id};
608bc29d
SC
1938
1939
059ec3d9
PH
1940 # JN - Skip over certain transports as specified via the "-nt/.../" command
1941 # line switch (where ... is a perl style regular expression). This is
1942 # required so that transports that skew stats such as SpamAssassin can be
1943 # ignored.
1944 #IFDEF ($transport_pattern)
1945 if (/\\sT=(\\S+)/) {
1946 next if ($1 =~ /$transport_pattern/o) ;
1947 }
1948 #ENDIF ($transport_pattern)
1949
1950
059ec3d9
PH
1951
1952 # Do some pattern matches to get the host and IP address.
1953 # We expect lines to be of the form "H=[IpAddr]" or "H=Host [IpAddr]" or
1954 # "H=Host (UnverifiedHost) [IpAddr]" or "H=(UnverifiedHost) [IpAddr]".
1955 # We do 2 separate matches to keep the matches simple and fast.
a83c7e95 1956 # Host is local unless otherwise specified.
514ee161
SC
1957 # Watch out for "H=([IpAddr])" in case they send "[IpAddr]" as their HELO!
1958 $ip = (/\\bH=(?:|.*? )(\\[[^]]+\\])/) ? $1
d5c13d66
NM
1959 # 2008-03-31 06:25:22 Connection from [213.246.33.217]:39456 refused: too many connections from that IP address // .hs
1960 : (/Connection from (\[\S+\])/) ? $1
1961 # 2008-03-31 06:52:40 SMTP call from mail.cacoshrf.com (ccsd02.ccsd.local) [69.24.118.229]:4511 dropped: too many nonmail commands (last was "RSET") // .hs
1962 : (/SMTP call from .*?(\[\S+\])/) ? $1
1963 : "local";
a83c7e95 1964 $host = (/\\bH=(\\S+)/) ? $1 : "local";
059ec3d9 1965
a83c7e95 1966 $domain = "localdomain"; #Domain is localdomain unless otherwise specified.
059ec3d9 1967
a83c7e95 1968 #IFDEF ($do_sender{Domain})
37f8a7c9
SC
1969 if ($host =~ /^\\[/ || $host =~ /^[\\d\\.]+$/) {
1970 # Host is just an IP address.
1971 $domain = $host;
1972 }
1973 elsif ($host =~ /^(\\(?)[^\\.]+\\.([^\\.]+\\..*)/) {
a83c7e95
SC
1974 # Remove the host portion from the DNS name. We ensure that we end up
1975 # with at least xxx.yyy. $host can be "(x.y.z)" or "x.y.z".
1976 $domain = lc("$1.$2");
1977 $domain =~ s/^\\.//; #Remove preceding dot.
059ec3d9 1978 }
a83c7e95 1979 #ENDIF ($do_sender{Domain})
059ec3d9
PH
1980
1981 #IFDEF ($do_sender{Email})
d5692f86
SC
1982 #IFDEF ($include_original_destination)
1983 # Catch both "a@b.com <c@d.com>" and "e@f.com"
1984 #$email = (/^(\S+) (<(\S*?)>)?/) ? $3 || $1 : "";
1985 $email = (/^(\S+ (<[^@>]+@?[^>]*>)?)/) ? $1 : "";
1986 chomp($email);
1987 #ENDIF ($include_original_destination)
1988
1989 #IFNDEF ($include_original_destination)
1990 $email = (/^(\S+)/) ? $1 : "";
1991 #ENDIF ($include_original_destination)
059ec3d9
PH
1992 #ENDIF ($do_sender{Email})
1993
1994 #IFDEF ($do_sender{Edomain})
5e6e6734
SC
1995 if (/^(<>|blackhole)/) {
1996 $edomain = $1;
1997 }
d5692f86 1998 #IFDEF ($include_original_destination)
5e6e6734
SC
1999 elsif (/^(\S+ (<\S*?\\@(\S+?)>)?)/) {
2000 $edomain = $1;
2001 chomp($edomain);
2002 $edomain =~ s/@(\S+?)>/"@" . lc($1) . ">"/e;
2003 }
d5692f86 2004 #ENDIF ($include_original_destination)
d5692f86 2005 #IFNDEF ($include_original_destination)
5e6e6734
SC
2006 elsif (/^\S*?\\@(\S+)/) {
2007 $edomain = lc($1);
2008 }
d5692f86 2009 #ENDIF ($include_original_destination)
5e6e6734
SC
2010 else {
2011 $edomain = "";
2012 }
2013
059ec3d9
PH
2014 #ENDIF ($do_sender{Edomain})
2015
2016 if ($tod lt $begin) {
2017 $begin = $tod;
2018 }
2019 elsif ($tod gt $end) {
2020 $end = $tod;
2021 }
2022
2023
2024 if ($flag eq "<=") {
2025 $thissize = (/\\sS=(\\d+)( |$)/) ? $1 : 0;
a83c7e95
SC
2026 $message[$SIZE] = $thissize;
2027 $message[$PROTOCOL] = (/ P=(\S+)/) ? $1 : undef;
059ec3d9
PH
2028
2029 #IFDEF ($show_relay)
2030 if ($host ne "local") {
d5692f86
SC
2031 # Save incoming information in case it becomes interesting
2032 # later, when delivery lines are read.
2033 my($from) = /^(\\S+)/;
a83c7e95
SC
2034 $message[$FROM_HOST] = "$host$ip";
2035 $message[$FROM_ADDRESS] = $from;
059ec3d9
PH
2036 }
2037 #ENDIF ($show_relay)
2038
2039 #IFDEF ($local_league_table || $include_remote_users)
d5692f86
SC
2040 if (/\sU=(\\S+)/) {
2041 my $user = $1;
059ec3d9 2042
d5692f86
SC
2043 #IFDEF ($local_league_table && $include_remote_users)
2044 { #Store both local and remote users.
2045 #ENDIF ($local_league_table && $include_remote_users)
059ec3d9 2046
d5692f86
SC
2047 #IFDEF ($local_league_table && ! $include_remote_users)
2048 if ($host eq "local") { #Store local users only.
2049 #ENDIF ($local_league_table && ! $include_remote_users)
059ec3d9 2050
d5692f86
SC
2051 #IFDEF ($include_remote_users && ! $local_league_table)
2052 if ($host ne "local") { #Store remote users only.
2053 #ENDIF ($include_remote_users && ! $local_league_table)
059ec3d9 2054
608bc29d 2055 ++$received_count_user{$user};
d5692f86 2056 add_volume(\\$received_data_user{$user},\\$received_data_gigs_user{$user},$thissize);
059ec3d9 2057 }
d5692f86 2058 }
059ec3d9
PH
2059 #ENDIF ($local_league_table || $include_remote_users)
2060
2061 #IFDEF ($do_sender{Host})
608bc29d 2062 ++$received_count{Host}{$host};
d5692f86 2063 add_volume(\\$received_data{Host}{$host},\\$received_data_gigs{Host}{$host},$thissize);
059ec3d9
PH
2064 #ENDIF ($do_sender{Host})
2065
2066 #IFDEF ($do_sender{Domain})
2067 if ($domain) {
608bc29d 2068 ++$received_count{Domain}{$domain};
d5692f86
SC
2069 add_volume(\\$received_data{Domain}{$domain},\\$received_data_gigs{Domain}{$domain},$thissize);
2070 }
059ec3d9
PH
2071 #ENDIF ($do_sender{Domain})
2072
2073 #IFDEF ($do_sender{Email})
608bc29d 2074 ++$received_count{Email}{$email};
d5692f86 2075 add_volume(\\$received_data{Email}{$email},\\$received_data_gigs{Email}{$email},$thissize);
059ec3d9
PH
2076 #ENDIF ($do_sender{Email})
2077
2078 #IFDEF ($do_sender{Edomain})
608bc29d 2079 ++$received_count{Edomain}{$edomain};
d5692f86 2080 add_volume(\\$received_data{Edomain}{$edomain},\\$received_data_gigs{Edomain}{$edomain},$thissize);
059ec3d9
PH
2081 #ENDIF ($do_sender{Edomain})
2082
608bc29d 2083 ++$total_received_count;
059ec3d9
PH
2084 add_volume(\\$total_received_data,\\$total_received_data_gigs,$thissize);
2085
608bc29d 2086 #IFDEF ($#queue_times >= 0 || $#rcpt_times >= 0)
a83c7e95 2087 $message[$ARRIVAL_TIME] = $tod;
608bc29d 2088 #ENDIF ($#queue_times >= 0 || $#rcpt_times >= 0)
059ec3d9
PH
2089
2090 #IFDEF ($hist_opt > 0)
d5692f86 2091 $received_interval_count[($m_hour*60 + $m_min)/$hist_interval]++;
059ec3d9
PH
2092 #ENDIF ($hist_opt > 0)
2093 }
2094
2095 elsif ($flag eq "=>") {
a83c7e95 2096 $size = $message[$SIZE] || 0;
059ec3d9 2097 if ($host ne "local") {
a83c7e95 2098 $message[$REMOTE_DELIVERED] = 1;
059ec3d9
PH
2099
2100
2101 #IFDEF ($show_relay)
2102 # Determine relaying address if either only one address listed,
2103 # or two the same. If they are different, it implies a forwarding
2104 # or aliasing, which is not relaying. Note that for multi-aliased
2105 # addresses, there may be a further address between the first
2106 # and last.
2107
a83c7e95 2108 if (defined $message[$FROM_HOST]) {
059ec3d9
PH
2109 if (/^(\\S+)(?:\\s+\\([^)]\\))?\\s+<([^>]+)>/) {
2110 ($old,$new) = ($1,$2);
d5692f86 2111 }
059ec3d9 2112 else {
d5692f86
SC
2113 $old = $new = "";
2114 }
059ec3d9
PH
2115
2116 if ("\\L$new" eq "\\L$old") {
2117 ($old) = /^(\\S+)/ if $old eq "";
a83c7e95 2118 my $key = "H=\\L$message[$FROM_HOST]\\E A=\\L$message[$FROM_ADDRESS]\\E => " .
059ec3d9
PH
2119 "H=\\L$host\\E$ip A=\\L$old\\E";
2120 if (!defined $relay_pattern || $key !~ /$relay_pattern/o) {
2121 $relayed{$key} = 0 if !defined $relayed{$key};
608bc29d 2122 ++$relayed{$key};
d5692f86 2123 }
059ec3d9 2124 else {
608bc29d 2125 ++$relayed_unshown;
059ec3d9
PH
2126 }
2127 }
2128 }
2129 #ENDIF ($show_relay)
2130
2131 }
2132
2133 #IFDEF ($local_league_table || $include_remote_users)
d5692f86
SC
2134 #IFDEF ($local_league_table && $include_remote_users)
2135 { #Store both local and remote users.
2136 #ENDIF ($local_league_table && $include_remote_users)
2137
2138 #IFDEF ($local_league_table && ! $include_remote_users)
2139 if ($host eq "local") { #Store local users only.
2140 #ENDIF ($local_league_table && ! $include_remote_users)
2141
2142 #IFDEF ($include_remote_users && ! $local_league_table)
2143 if ($host ne "local") { #Store remote users only.
2144 #ENDIF ($include_remote_users && ! $local_league_table)
2145
2146 if (my($user) = split((/\\s</)? " <" : " ", $_)) {
2147 #IFDEF ($include_original_destination)
2148 {
2149 #ENDIF ($include_original_destination)
2150 #IFNDEF ($include_original_destination)
2151 if ($user =~ /^[\\/|]/) {
2152 #ENDIF ($include_original_destination)
a797be69
SC
2153 #my($parent) = $_ =~ /(<[^@]+@?[^>]*>)/;
2154 my($parent) = $_ =~ / (<.+?>) /; #DT 1.54
37f8a7c9
SC
2155 if (defined $parent) {
2156 $user = "$user $parent";
2157 #IFDEF ($do_local_domain)
2158 if ($parent =~ /\\@(.+)>/) {
2159 $local_domain = lc($1);
2160 ++$delivered_messages_local_domain{$local_domain};
2161 ++$delivered_addresses_local_domain{$local_domain};
2162 add_volume(\\$delivered_data_local_domain{$local_domain},\\$delivered_data_gigs_local_domain{$local_domain},$size);
2163 }
2164 #ENDIF ($do_local_domain)
2165 }
d5692f86 2166 }
a83c7e95
SC
2167 ++$delivered_messages_user{$user};
2168 ++$delivered_addresses_user{$user};
d5692f86
SC
2169 add_volume(\\$delivered_data_user{$user},\\$delivered_data_gigs_user{$user},$size);
2170 }
2171 }
059ec3d9
PH
2172 #ENDIF ($local_league_table || $include_remote_users)
2173
2174 #IFDEF ($do_sender{Host})
a83c7e95
SC
2175 $delivered_messages{Host}{$host}++;
2176 $delivered_addresses{Host}{$host}++;
d5692f86 2177 add_volume(\\$delivered_data{Host}{$host},\\$delivered_data_gigs{Host}{$host},$size);
059ec3d9
PH
2178 #ENDIF ($do_sender{Host})
2179 #IFDEF ($do_sender{Domain})
2180 if ($domain) {
a83c7e95
SC
2181 ++$delivered_messages{Domain}{$domain};
2182 ++$delivered_addresses{Domain}{$domain};
d5692f86
SC
2183 add_volume(\\$delivered_data{Domain}{$domain},\\$delivered_data_gigs{Domain}{$domain},$size);
2184 }
059ec3d9
PH
2185 #ENDIF ($do_sender{Domain})
2186 #IFDEF ($do_sender{Email})
a83c7e95
SC
2187 ++$delivered_messages{Email}{$email};
2188 ++$delivered_addresses{Email}{$email};
d5692f86 2189 add_volume(\\$delivered_data{Email}{$email},\\$delivered_data_gigs{Email}{$email},$size);
059ec3d9
PH
2190 #ENDIF ($do_sender{Email})
2191 #IFDEF ($do_sender{Edomain})
a83c7e95
SC
2192 ++$delivered_messages{Edomain}{$edomain};
2193 ++$delivered_addresses{Edomain}{$edomain};
d5692f86 2194 add_volume(\\$delivered_data{Edomain}{$edomain},\\$delivered_data_gigs{Edomain}{$edomain},$size);
059ec3d9
PH
2195 #ENDIF ($do_sender{Edomain})
2196
a83c7e95
SC
2197 ++$total_delivered_messages;
2198 ++$total_delivered_addresses;
059ec3d9
PH
2199 add_volume(\\$total_delivered_data,\\$total_delivered_data_gigs,$size);
2200
2201 #IFDEF ($show_transport)
2202 my $transport = (/\\sT=(\\S+)/) ? $1 : ":blackhole:";
608bc29d 2203 ++$transported_count{$transport};
059ec3d9
PH
2204 add_volume(\\$transported_data{$transport},\\$transported_data_gigs{$transport},$size);
2205 #ENDIF ($show_transport)
2206
2207 #IFDEF ($hist_opt > 0)
2208 $delivered_interval_count[($m_hour*60 + $m_min)/$hist_interval]++;
2209 #ENDIF ($hist_opt > 0)
2210
608bc29d
SC
2211 #IFDEF ($#delivery_times > 0)
2212 if (/ DT=(\S+)/) {
2213 $seconds = wdhms_seconds($1);
2214 for ($i = 0; $i <= $#delivery_times; $i++) {
2215 if ($seconds < $delivery_times[$i]) {
2216 ++$dt_all_bin[$i];
a83c7e95 2217 ++$dt_remote_bin[$i] if $message[$REMOTE_DELIVERED];
608bc29d
SC
2218 last;
2219 }
2220 }
2221 if ($i > $#delivery_times) {
2222 ++$dt_all_overflow;
a83c7e95 2223 ++$dt_remote_overflow if $message[$REMOTE_DELIVERED];
608bc29d
SC
2224 }
2225 }
2226 #ENDIF ($#delivery_times > 0)
2227
059ec3d9
PH
2228 }
2229
a83c7e95
SC
2230 elsif ($flag eq "->") {
2231
2232 #IFDEF ($local_league_table || $include_remote_users)
2233 #IFDEF ($local_league_table && $include_remote_users)
2234 { #Store both local and remote users.
2235 #ENDIF ($local_league_table && $include_remote_users)
2236
2237 #IFDEF ($local_league_table && ! $include_remote_users)
2238 if ($host eq "local") { #Store local users only.
2239 #ENDIF ($local_league_table && ! $include_remote_users)
2240
2241 #IFDEF ($include_remote_users && ! $local_league_table)
2242 if ($host ne "local") { #Store remote users only.
2243 #ENDIF ($include_remote_users && ! $local_league_table)
2244
2245 if (my($user) = split((/\\s</)? " <" : " ", $_)) {
2246 #IFDEF ($include_original_destination)
2247 {
2248 #ENDIF ($include_original_destination)
2249 #IFNDEF ($include_original_destination)
2250 if ($user =~ /^[\\/|]/) {
2251 #ENDIF ($include_original_destination)
a797be69
SC
2252 #my($parent) = $_ =~ /(<[^@]+@?[^>]*>)/;
2253 my($parent) = $_ =~ / (<.+?>) /; #DT 1.54
a83c7e95
SC
2254 $user = "$user $parent" if defined $parent;
2255 }
2256 ++$delivered_addresses_user{$user};
2257 }
2258 }
2259 #ENDIF ($local_league_table || $include_remote_users)
2260
2261 #IFDEF ($do_sender{Host})
2262 $delivered_addresses{Host}{$host}++;
2263 #ENDIF ($do_sender{Host})
2264 #IFDEF ($do_sender{Domain})
2265 if ($domain) {
2266 ++$delivered_addresses{Domain}{$domain};
2267 }
2268 #ENDIF ($do_sender{Domain})
2269 #IFDEF ($do_sender{Email})
2270 ++$delivered_addresses{Email}{$email};
2271 #ENDIF ($do_sender{Email})
2272 #IFDEF ($do_sender{Edomain})
2273 ++$delivered_addresses{Edomain}{$edomain};
2274 #ENDIF ($do_sender{Edomain})
2275
2276 ++$total_delivered_addresses;
2277 }
2278
2279 elsif ($flag eq "==" && defined($message[$SIZE]) && !defined($message[$DELAYED])) {
608bc29d 2280 ++$delayed_count;
a83c7e95 2281 $message[$DELAYED] = 1;
059ec3d9
PH
2282 }
2283
2284 elsif ($flag eq "**") {
a83c7e95
SC
2285 if (defined ($message[$SIZE])) {
2286 unless (defined $message[$HAD_ERROR]) {
608bc29d 2287 ++$message_errors;
a83c7e95 2288 $message[$HAD_ERROR] = 1;
608bc29d
SC
2289 }
2290 }
059ec3d9
PH
2291
2292 #IFDEF ($show_errors)
608bc29d 2293 ++$errors_count{$_};
059ec3d9
PH
2294 #ENDIF ($show_errors)
2295
2296 }
2297
2298 elsif ($flag eq "Co") {
2299 #Completed?
2300 #IFDEF ($#queue_times >= 0)
a83c7e95 2301 $queued = queue_time($tod, $message[$ARRIVAL_TIME], $id);
059ec3d9
PH
2302
2303 for ($i = 0; $i <= $#queue_times; $i++) {
2304 if ($queued < $queue_times[$i]) {
608bc29d 2305 ++$qt_all_bin[$i];
a83c7e95 2306 ++$qt_remote_bin[$i] if $message[$REMOTE_DELIVERED];
059ec3d9 2307 last;
d5692f86
SC
2308 }
2309 }
608bc29d
SC
2310 if ($i > $#queue_times) {
2311 ++$qt_all_overflow;
a83c7e95 2312 ++$qt_remote_overflow if $message[$REMOTE_DELIVERED];
608bc29d 2313 }
059ec3d9
PH
2314 #ENDIF ($#queue_times >= 0)
2315
608bc29d
SC
2316 #IFDEF ($#rcpt_times >= 0)
2317 if (/ QT=(\S+)/) {
2318 $seconds = wdhms_seconds($1);
2319 #Calculate $queued if not previously calculated above.
2320 #IFNDEF ($#queue_times >= 0)
a83c7e95 2321 $queued = queue_time($tod, $message[$ARRIVAL_TIME], $id);
608bc29d
SC
2322 #ENDIF ($#queue_times >= 0)
2323 $rcpt_time = $seconds - $queued;
2324 my($protocol);
2325
a83c7e95
SC
2326 if (defined $message[$PROTOCOL]) {
2327 $protocol = $message[$PROTOCOL];
608bc29d
SC
2328
2329 # Create the bin if its not already defined.
2330 unless (exists $rcpt_times_bin{$protocol}) {
2331 initialise_rcpt_times($protocol);
2332 }
2333 }
2334
2335
2336 for ($i = 0; $i <= $#rcpt_times; ++$i) {
2337 if ($rcpt_time < $rcpt_times[$i]) {
2338 ++$rcpt_times_bin{all}[$i];
2339 ++$rcpt_times_bin{$protocol}[$i] if defined $protocol;
2340 last;
2341 }
2342 }
059ec3d9 2343
608bc29d
SC
2344 if ($i > $#rcpt_times) {
2345 ++$rcpt_times_overflow{all};
2346 ++$rcpt_times_overflow{$protocol} if defined $protocol;
2347 }
2348 }
2349 #ENDIF ($#rcpt_times >= 0)
2350
2351 delete($messages{$id});
059ec3d9 2352 }
a83c7e95
SC
2353 elsif ($flag eq "SA") {
2354 $ip = (/From.*?(\\[[^]]+\\])/ || /\\((local)\\)/) ? $1 : "";
2355 #SpamAssassin message
2356 if (/Action: ((permanently|temporarily) rejected message|flagged as Spam but accepted): score=(\d+\.\d)/) {
2357 #add_volume(\\$spam_score,\\$spam_score_gigs,$3);
2358 ++$spam_count_by_ip{$ip};
2359 } elsif (/Action: scanned but message isn\'t spam: score=(-?\d+\.\d)/) {
2360 #add_volume(\\$ham_score,\\$ham_score_gigs,$1);
2361 ++$ham_count_by_ip{$ip};
2362 } elsif (/(Not running SA because SAEximRunCond expanded to false|check skipped due to message size)/) {
2363 ++$ham_count_by_ip{$ip};
2364 }
2365 }
2366
2367 # Look for Reject messages or blackholed messages (deliveries
2368 # without a transport)
2369 if ($flag eq "Re" || ($flag eq "=>" && ! /\\sT=\\S+/)) {
2370 # Correct the IP address for rejects:
2371 # rejected EHLO from my.test.net [10.0.0.5]: syntactically invalid argument(s):
e2f7a0d2
SC
2372 # rejected EHLO from [10.0.0.6]: syntactically invalid argument(s):
2373 $ip = $1 if ($ip eq "local" && /^rejected [HE][HE]LO from .*?(\[.+?\]):/);
e2f7a0d2
SC
2374 if (/SpamAssassin/) {
2375 ++$rejected_count_by_reason{"Rejected by SpamAssassin"};
30e9ac3f
SC
2376 ++$rejected_count_by_ip{$ip};
2377 }
2378 elsif (
2379 /(temporarily rejected [A-Z]*) .*?(: .*?)(:|\s*$)/
2380 ) {
2381 ++$temporarily_rejected_count_by_reason{"\u$1$2"};
2382 ++$temporarily_rejected_count_by_ip{$ip};
2383 }
2384 elsif (
2385 /(temporarily refused connection)/
2386 ) {
2387 ++$temporarily_rejected_count_by_reason{"\u$1"};
2388 ++$temporarily_rejected_count_by_ip{$ip};
e2f7a0d2
SC
2389 }
2390 elsif (
a83c7e95
SC
2391 /(listed at [^ ]+)/ ||
2392 /(Forged IP detected in HELO)/ ||
2393 /(Invalid domain or IP given in HELO\/EHLO)/ ||
2394 /(unqualified recipient rejected)/ ||
2395 /(closed connection (after|in response) .*?)\s*$/ ||
2396 /(sender rejected)/ ||
2397 # 2005-09-23 15:07:49 1EInHJ-0007Ex-Au H=(a.b.c) [10.0.0.1] F=<> rejected after DATA: This message contains a virus: (Eicar-Test-Signature) please scan your system.
2398 # 2005-10-06 10:50:07 1ENRS3-0000Nr-Kt => blackhole (DATA ACL discarded recipients): This message contains a virus: (Worm.SomeFool.P) please scan your system.
2399 / rejected after DATA: (.*)/ ||
b322aac8 2400 / (rejected DATA: .*)/ ||
a83c7e95
SC
2401 /.DATA ACL discarded recipients.: (.*)/ ||
2402 /rejected after DATA: (unqualified address not permitted)/ ||
2403 /(VRFY rejected)/ ||
2404# /(sender verify (defer|fail))/i ||
2405 /(too many recipients)/ ||
2406 /(refused relay.*?) to/ ||
2407 /(rejected by non-SMTP ACL: .*)/ ||
2408 /(rejected by local_scan.*)/ ||
2409 # SMTP call from %s dropped: too many syntax or protocol errors (last command was "%s"
2410 # SMTP call from %s dropped: too many nonmail commands
2411 /(dropped: too many ((nonmail|unrecognized) commands|syntax or protocol errors))/ ||
2412
2413 # local_scan() function crashed with signal %d - message temporarily rejected
2414 # local_scan() function timed out - message temporarily rejected
2415 /(local_scan.. function .* - message temporarily rejected)/ ||
a83c7e95
SC
2416 # SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from %s
2417 /(SMTP protocol .*?(error|violation))/ ||
2418 /(message too big)/
2419 ) {
2420 ++$rejected_count_by_reason{"\u$1"};
30e9ac3f 2421 ++$rejected_count_by_ip{$ip};
a83c7e95
SC
2422 }
2423 elsif (/rejected [HE][HE]LO from [^:]*: syntactically invalid argument/) {
2424 ++$rejected_count_by_reason{"Rejected HELO/EHLO: syntactically invalid argument"};
30e9ac3f 2425 ++$rejected_count_by_ip{$ip};
a83c7e95
SC
2426 }
2427 elsif (/response to "RCPT TO.*? was: (.*)/) {
2428 ++$rejected_count_by_reason{"Response to RCPT TO was: $1"};
30e9ac3f 2429 ++$rejected_count_by_ip{$ip};
a83c7e95
SC
2430 }
2431 elsif (
2432 /(lookup of host )\S+ (failed)/ ||
2433
2434 # rejected from <%s>%s%s%s%s: message too big:
2435 /(rejected [A-Z]*) .*?(: .*?)(:|\s*$)/ ||
2436 # refused connection from %s (host_reject_connection)
2437 # refused connection from %s (tcp wrappers)
2438 /(refused connection )from.*? (\(.*)/ ||
2439
2440 # error from remote mailer after RCPT TO:<a@b.c>: host a.b.c [10.0.0.1]: 450 <a@b.c>: Recipient address rejected: Greylisted for 60 seconds
2441 # error from remote mailer after MAIL FROM:<> SIZE=3468: host a.b.c [10.0.0.1]: 421 a.b.c has refused your connection because your server did not have a PTR record.
2442 /(error from remote mailer after .*?:).*(: .*?)(:|\s*$)/ ||
2443
2444 # a.b.c F=<a@b.c> rejected after DATA: "@" or "." expected after "Undisclosed-Recipient": failing address in "To" header is: <Undisclosed-Recipient:;>
2445 /rejected after DATA: ("." or "." expected).*?(: failing address in .*? header)/ ||
2446
2447 # connection from %s refused load average = %.2f
2448 /(Connection )from.*? (refused: load average)/ ||
2449 # connection from %s refused (IP options)
2450 # Connection from %s refused: too many connections
2451 # connection from %s refused
2452 /([Cc]onnection )from.*? (refused.*)/ ||
2453 # [10.0.0.1]: connection refused
2454 /: (Connection refused)()/
2455 ) {
2456 ++$rejected_count_by_reason{"\u$1$2"};
30e9ac3f 2457 ++$rejected_count_by_ip{$ip};
a83c7e95 2458 }
d5c13d66
NM
2459 elsif (
2460 # 2008-03-31 06:25:22 H=mail.densitron.com [216.70.140.224]:45386 temporarily rejected connection in "connect" ACL: too fast reconnects // .hs
2461 # 2008-03-31 06:25:22 H=mail.densitron.com [216.70.140.224]:45386 temporarily rejected connection in "connect" ACL // .hs
2462 /(temporarily rejected connection in .*?ACL:?.*)/
2463 ) {
2464 ++$temporarily_rejected_count_by_ip{$ip};
2465 ++$temporarily_rejected_count_by_reason{"\u$1"};
2466 }
a83c7e95
SC
2467 else {
2468 ++$rejected_count_by_reason{Unknown};
30e9ac3f 2469 ++$rejected_count_by_ip{$ip};
a83c7e95
SC
2470 print STDERR "Unknown rejection: $_" if $debug;
2471 }
2472 }
059ec3d9
PH
2473 }';
2474
2475 # We now do a 'C preprocessor style operation on our parser
2476 # to remove bits not in use.
2477 my(%defines_in_operation,$removing_lines,$processed_parser);
2478 foreach (split (/\n/,$parser)) {
2479 if ((/^\s*#\s*IFDEF\s*\((.*?)\)/i && ! eval $1) ||
d5692f86 2480 (/^\s*#\s*IFNDEF\s*\((.*?)\)/i && eval $1) ) {
059ec3d9
PH
2481 $defines_in_operation{$1} = 1;
2482 $removing_lines = 1;
2483 }
2484
608bc29d
SC
2485 # Convert constants.
2486 while (/(\$[A-Z][A-Z_]*)\b/) {
2487 my $constant = eval $1;
2488 s/(\$[A-Z][A-Z_]*)\b/$constant/;
2489 }
2490
059ec3d9
PH
2491 $processed_parser .= $_."\n" unless $removing_lines;
2492
2493 if (/^\s*#\s*ENDIF\s*\((.*?)\)/i) {
2494 delete $defines_in_operation{$1};
2495 unless (keys %defines_in_operation) {
d5692f86 2496 $removing_lines = 0;
059ec3d9
PH
2497 }
2498 }
2499 }
608bc29d 2500 print STDERR "# START OF PARSER:$processed_parser\n# END OF PARSER\n\n" if $debug;
059ec3d9
PH
2501
2502 return $processed_parser;
2503}
2504
2505
2506
1b4fe9dd
PH
2507#######################################################################
2508# parse();
8e669ac1 2509#
1b4fe9dd 2510# parse($parser,\*FILEHANDLE);
8e669ac1 2511#
1b4fe9dd
PH
2512# This subroutine accepts a parser and a filehandle from main and parses each
2513# line. We store the results into global variables.
2514#######################################################################
059ec3d9
PH
2515sub parse {
2516 my($parser,$fh) = @_;
2517
2518 if ($merge_reports) {
2519 parse_old_eximstat_reports($fh);
2520 }
2521 else {
2522 eval $parser;
2523 die ($@) if $@;
2524 }
2525
2526}
2527
2528
2529
1b4fe9dd
PH
2530#######################################################################
2531# print_header();
8e669ac1 2532#
1b4fe9dd 2533# print_header();
8e669ac1 2534#
1b4fe9dd
PH
2535# Print our headers and contents.
2536#######################################################################
059ec3d9
PH
2537sub print_header {
2538
d5692f86 2539
059ec3d9
PH
2540 my $title = "Exim statistics from $begin to $end";
2541
d5692f86
SC
2542 print $txt_fh "\n$title\n" if $txt_fh;
2543 if ($htm_fh) {
2544 print $htm_fh html_header($title);
2545 print $htm_fh "<ul>\n";
30e9ac3f
SC
2546 print $htm_fh "<li><a href=\"#Grandtotal\">Grand total summary</a>\n";
2547 print $htm_fh "<li><a href=\"#Patterns\">User Specified Patterns</a>\n" if @user_patterns;
2548 print $htm_fh "<li><a href=\"#Transport\">Deliveries by Transport</a>\n" if $show_transport;
059ec3d9 2549 if ($hist_opt) {
d5692f86
SC
2550 print $htm_fh "<li><a href=\"#Messages received\">Messages received per hour</a>\n";
2551 print $htm_fh "<li><a href=\"#Deliveries\">Deliveries per hour</a>\n";
059ec3d9 2552 }
608bc29d 2553
059ec3d9 2554 if ($#queue_times >= 0) {
608bc29d
SC
2555 print $htm_fh "<li><a href=\"#Time spent on the queue all messages\">Time spent on the queue: all messages</a>\n";
2556 print $htm_fh "<li><a href=\"#Time spent on the queue messages with at least one remote delivery\">Time spent on the queue: messages with at least one remote delivery</a>\n";
2557 }
2558
2559 if ($#delivery_times >= 0) {
2560 print $htm_fh "<li><a href=\"#Delivery times all messages\">Delivery times: all messages</a>\n";
2561 print $htm_fh "<li><a href=\"#Delivery times messages with at least one remote delivery\">Delivery times: messages with at least one remote delivery</a>\n";
2562 }
2563
2564 if ($#rcpt_times >= 0) {
2565 print $htm_fh "<li><a href=\"#Receipt times all messages\">Receipt times</a>\n";
059ec3d9 2566 }
608bc29d 2567
d5692f86 2568 print $htm_fh "<li><a href=\"#Relayed messages\">Relayed messages</a>\n" if $show_relay;
059ec3d9 2569 if ($topcount) {
30e9ac3f 2570 print $htm_fh "<li><a href=\"#Mail rejection reason count\">Top $topcount mail rejection reasons by message count</a>\n" if %rejected_count_by_reason;
059ec3d9 2571 foreach ('Host','Domain','Email','Edomain') {
d5692f86 2572 next unless $do_sender{$_};
30e9ac3f
SC
2573 print $htm_fh "<li><a href=\"#Sending \l$_ count\">Top $topcount sending \l${_}s by message count</a>\n";
2574 print $htm_fh "<li><a href=\"#Sending \l$_ volume\">Top $topcount sending \l${_}s by volume</a>\n";
059ec3d9 2575 }
a83c7e95 2576 if (($local_league_table || $include_remote_users) && %received_count_user) {
30e9ac3f
SC
2577 print $htm_fh "<li><a href=\"#Local sender count\">Top $topcount local senders by message count</a>\n";
2578 print $htm_fh "<li><a href=\"#Local sender volume\">Top $topcount local senders by volume</a>\n";
059ec3d9
PH
2579 }
2580 foreach ('Host','Domain','Email','Edomain') {
d5692f86 2581 next unless $do_sender{$_};
30e9ac3f
SC
2582 print $htm_fh "<li><a href=\"#$_ destination count\">Top $topcount \l$_ destinations by message count</a>\n";
2583 print $htm_fh "<li><a href=\"#$_ destination volume\">Top $topcount \l$_ destinations by volume</a>\n";
059ec3d9 2584 }
a83c7e95 2585 if (($local_league_table || $include_remote_users) && %delivered_messages_user) {
30e9ac3f
SC
2586 print $htm_fh "<li><a href=\"#Local destination count\">Top $topcount local destinations by message count</a>\n";
2587 print $htm_fh "<li><a href=\"#Local destination volume\">Top $topcount local destinations by volume</a>\n";
059ec3d9 2588 }
37f8a7c9
SC
2589 if (($local_league_table || $include_remote_users) && %delivered_messages_local_domain) {
2590 print $htm_fh "<li><a href=\"#Local domain destination count\">Top $topcount local domain destinations by message count</a>\n";
2591 print $htm_fh "<li><a href=\"#Local domain destination volume\">Top $topcount local domain destinations by volume</a>\n";
2592 }
a83c7e95 2593
30e9ac3f
SC
2594 print $htm_fh "<li><a href=\"#Rejected ip count\">Top $topcount rejected ips by message count</a>\n" if %rejected_count_by_ip;
2595 print $htm_fh "<li><a href=\"#Temporarily rejected ip count\">Top $topcount temporarily rejected ips by message count</a>\n" if %temporarily_rejected_count_by_ip;
2596 print $htm_fh "<li><a href=\"#Non-rejected spamming ip count\">Top $topcount non-rejected spamming ips by message count</a>\n" if %spam_count_by_ip;
a83c7e95 2597
059ec3d9 2598 }
d5692f86
SC
2599 print $htm_fh "<li><a href=\"#errors\">List of errors</a>\n" if %errors_count;
2600 print $htm_fh "</ul>\n<hr>\n";
059ec3d9 2601 }
d5692f86
SC
2602 if ($xls_fh)
2603 {
2604 $ws_global->write($row++, $col+0, "Exim Statistics", $f_header1);
2605 &set_worksheet_line($ws_global, $row, $col, ["from:", $begin, "to:", $end], $f_default);
2606 $row+=2;
059ec3d9
PH
2607 }
2608}
2609
2610
1b4fe9dd
PH
2611#######################################################################
2612# print_grandtotals();
8e669ac1 2613#
1b4fe9dd 2614# print_grandtotals();
8e669ac1 2615#
1b4fe9dd
PH
2616# Print the grand totals.
2617#######################################################################
059ec3d9
PH
2618sub print_grandtotals {
2619
2620 # Get the sender by headings and results. This is complicated as we can have
2621 # different numbers of columns.
a83c7e95 2622 my($sender_txt_header,$sender_txt_format,$sender_html_format);
059ec3d9 2623 my(@received_totals,@delivered_totals);
d5692f86 2624 my($row_tablehead, $row_max);
a83c7e95 2625 my(@col_headers) = ('TOTAL', 'Volume', 'Messages', 'Addresses');
d5692f86 2626
059ec3d9
PH
2627 foreach ('Host','Domain','Email','Edomain') {
2628 next unless $do_sender{$_};
2629 if ($merge_reports) {
2630 push(@received_totals, get_report_total($report_totals{Received},"${_}s"));
2631 push(@delivered_totals,get_report_total($report_totals{Delivered},"${_}s"));
2632 }
2633 else {
2634 push(@received_totals,scalar(keys %{$received_data{$_}}));
2635 push(@delivered_totals,scalar(keys %{$delivered_data{$_}}));
2636 }
059ec3d9 2637 $sender_txt_header .= " " x ($COLUMN_WIDTHS - length($_)) . $_ . 's';
30e9ac3f
SC
2638 $sender_html_format .= "<td align=\"right\">%s</td>";
2639 $sender_txt_format .= " " x ($COLUMN_WIDTHS - 5) . "%6s";
a83c7e95 2640 push(@col_headers,"${_}s");
059ec3d9
PH
2641 }
2642
a83c7e95 2643 my $txt_format1 = " %-16s %9s %6d %6s $sender_txt_format";
d5692f86 2644 my $txt_format2 = " %6d %4.1f%% %6d %4.1f%%",
a83c7e95 2645 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%s</td><td align=\"right\">%s</td><td align=\"right\">%s</td>$sender_html_format";
d5692f86 2646 my $htm_format2 = "<td align=\"right\">%d</td><td align=\"right\">%4.1f%%</td><td align=\"right\">%d</td><td align=\"right\">%4.1f%%</td>";
059ec3d9 2647
d5692f86 2648 if ($txt_fh) {
059ec3d9 2649 my $sender_spaces = " " x length($sender_txt_header);
d5692f86
SC
2650 print $txt_fh "\n";
2651 print $txt_fh "Grand total summary\n";
2652 print $txt_fh "-------------------\n";
a83c7e95
SC
2653 print $txt_fh " $sender_spaces At least one address\n";
2654 print $txt_fh " TOTAL Volume Messages Addresses $sender_txt_header Delayed Failed\n";
d5692f86
SC
2655 }
2656 if ($htm_fh) {
30e9ac3f 2657 print $htm_fh "<a name=\"Grandtotal\"></a>\n";
d5692f86
SC
2658 print $htm_fh "<h2>Grand total summary</h2>\n";
2659 print $htm_fh "<table border=1>\n";
a83c7e95 2660 print $htm_fh "<tr><th>" . join('</th><th>',@col_headers) . "</th><th colspan=2>At least one addr<br>Delayed</th><th colspan=2>At least one addr<br>Failed</th>\n";
d5692f86 2661 }
a83c7e95
SC
2662 if ($xls_fh) {
2663 $ws_global->write($row++, 0, "Grand total summary", $f_header2);
2664 $ws_global->write($row, 0, \@col_headers, $f_header2);
2665 $ws_global->merge_range($row, scalar(@col_headers), $row, scalar(@col_headers)+1, "At least one addr Delayed", $f_header2_m);
2666 $ws_global->merge_range($row, scalar(@col_headers)+2, $row, scalar(@col_headers)+3, "At least one addr Failed", $f_header2_m);
2667 #$ws_global->write(++$row, scalar(@col_headers), ['Total','Percent','Total','Percent'], $f_header2);
059ec3d9
PH
2668 }
2669
d5692f86 2670
059ec3d9
PH
2671 my($volume,$failed_count);
2672 if ($merge_reports) {
2673 $volume = volume_rounded($report_totals{Received}{Volume}, $report_totals{Received}{'Volume-gigs'});
2674 $total_received_count = get_report_total($report_totals{Received},'Messages');
2675 $failed_count = get_report_total($report_totals{Received},'Failed');
2676 $delayed_count = get_report_total($report_totals{Received},'Delayed');
2677 }
2678 else {
2679 $volume = volume_rounded($total_received_data, $total_received_data_gigs);
608bc29d 2680 $failed_count = $message_errors;
059ec3d9
PH
2681 }
2682
2683 {
2684 no integer;
059ec3d9 2685
d5692f86 2686 my @content=(
a83c7e95 2687 $volume,$total_received_count,'',
d5692f86
SC
2688 @received_totals,
2689 $delayed_count,
2690 ($total_received_count) ? ($delayed_count*100/$total_received_count) : 0,
2691 $failed_count,
2692 ($total_received_count) ? ($failed_count*100/$total_received_count) : 0
2693 );
2694
2695 printf $txt_fh ("$txt_format1$txt_format2\n", 'Received', @content) if $txt_fh;
2696 printf $htm_fh ("$htm_format1$htm_format2\n", 'Received', @content) if $htm_fh;
a83c7e95
SC
2697 if ($xls_fh) {
2698 $ws_global->write(++$row, 0, 'Received', $f_default);
2699 for (my $i=0; $i < scalar(@content); $i++) {
d5692f86 2700 if ($i == 4 || $i == 6) {
a83c7e95 2701 $ws_global->write($row, $i+1, $content[$i]/100, $f_percent);
d5692f86
SC
2702 }
2703 else {
a83c7e95 2704 $ws_global->write($row, $i+1, $content[$i], $f_default);
d5692f86
SC
2705 }
2706 }
2707 }
2708 }
a83c7e95 2709
059ec3d9
PH
2710 if ($merge_reports) {
2711 $volume = volume_rounded($report_totals{Delivered}{Volume}, $report_totals{Delivered}{'Volume-gigs'});
a83c7e95
SC
2712 $total_delivered_messages = get_report_total($report_totals{Delivered},'Messages');
2713 $total_delivered_addresses = get_report_total($report_totals{Delivered},'Addresses');
059ec3d9
PH
2714 }
2715 else {
2716 $volume = volume_rounded($total_delivered_data, $total_delivered_data_gigs);
2717 }
d5692f86 2718
a83c7e95
SC
2719 my @content=($volume, $total_delivered_messages, $total_delivered_addresses, @delivered_totals);
2720 printf $txt_fh ("$txt_format1\n", 'Delivered', @content) if $txt_fh;
2721 printf $htm_fh ("$htm_format1\n", 'Delivered', @content) if $htm_fh;
d5692f86 2722
a83c7e95
SC
2723 if ($xls_fh) {
2724 $ws_global->write(++$row, 0, 'Delivered', $f_default);
2725 for (my $i=0; $i < scalar(@content); $i++) {
2726 $ws_global->write($row, $i+1, $content[$i], $f_default);
2727 }
2728 }
2729
2730 if ($merge_reports) {
30e9ac3f 2731 foreach ('Rejects', 'Temp Rejects', 'Ham', 'Spam') {
a83c7e95
SC
2732 my $messages = get_report_total($report_totals{$_},'Messages');
2733 my $addresses = get_report_total($report_totals{$_},'Addresses');
2734 if ($messages) {
2735 @content = ($_, '', $messages, '');
2736 push(@content,get_report_total($report_totals{$_},'Hosts')) if $do_sender{Host};
30e9ac3f
SC
2737 #These rows do not have entries for the following columns (if specified)
2738 foreach ('Domain','Email','Edomain') {
2739 push(@content,'') if $do_sender{$_};
2740 }
2741
a83c7e95
SC
2742 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2743 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2744 $ws_global->write(++$row, 0, \@content) if $xls_fh;
2745 }
2746 }
2747 }
2748 else {
2749 foreach my $total_aref (['Rejects',\%rejected_count_by_ip],
30e9ac3f 2750 ['Temp Rejects',\%temporarily_rejected_count_by_ip],
a83c7e95
SC
2751 ['Ham',\%ham_count_by_ip],
2752 ['Spam',\%spam_count_by_ip]) {
30e9ac3f 2753 #Count the number of messages of this type.
a83c7e95
SC
2754 my $messages = 0;
2755 map {$messages += $_} values %{$total_aref->[1]};
2756
2757 if ($messages > 0) {
2758 @content = ($total_aref->[0], '', $messages, '');
30e9ac3f 2759
4c04137d 2760 #Count the number of distinct IPs for the Hosts column.
a83c7e95
SC
2761 push(@content,scalar(keys %{$total_aref->[1]})) if $do_sender{Host};
2762
30e9ac3f
SC
2763 #These rows do not have entries for the following columns (if specified)
2764 foreach ('Domain','Email','Edomain') {
2765 push(@content,'') if $do_sender{$_};
2766 }
2767
a83c7e95
SC
2768 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2769 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2770 $ws_global->write(++$row, 0, \@content) if $xls_fh;
d5692f86 2771 }
a83c7e95 2772 }
d5692f86 2773 }
a83c7e95
SC
2774
2775 printf $txt_fh "\n" if $txt_fh;
2776 printf $htm_fh "</table>\n" if $htm_fh;
2777 ++$row;
059ec3d9
PH
2778}
2779
2780
1b4fe9dd
PH
2781#######################################################################
2782# print_user_patterns()
8e669ac1 2783#
1b4fe9dd 2784# print_user_patterns();
8e669ac1 2785#
1b4fe9dd
PH
2786# Print the counts of user specified patterns.
2787#######################################################################
059ec3d9 2788sub print_user_patterns {
d5692f86
SC
2789 my $txt_format1 = " %-18s %6d";
2790 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%d</td>";
059ec3d9 2791
d5692f86
SC
2792 if ($txt_fh) {
2793 print $txt_fh "User Specified Patterns\n";
2794 print $txt_fh "-----------------------";
2795 print $txt_fh "\n Total\n";
059ec3d9 2796 }
d5692f86 2797 if ($htm_fh) {
30e9ac3f 2798 print $htm_fh "<hr><a name=\"Patterns\"></a><h2>User Specified Patterns</h2>\n";
d5692f86
SC
2799 print $htm_fh "<table border=0 width=\"100%\">\n";
2800 print $htm_fh "<tr><td>\n";
2801 print $htm_fh "<table border=1>\n";
2802 print $htm_fh "<tr><th>&nbsp;</th><th>Total</th>\n";
2803 }
2804 if ($xls_fh) {
2805 $ws_global->write($row++, $col, "User Specified Patterns", $f_header2);
2806 &set_worksheet_line($ws_global, $row++, 1, ["Total"], $f_headertab);
059ec3d9
PH
2807 }
2808
d5692f86 2809
059ec3d9
PH
2810 my($key);
2811 if ($merge_reports) {
2812 # We are getting our data from previous reports.
2813 foreach $key (@user_descriptions) {
2814 my $count = get_report_total($report_totals{patterns}{$key},'Total');
d5692f86
SC
2815 printf $txt_fh ("$txt_format1\n",$key,$count) if $txt_fh;
2816 printf $htm_fh ("$htm_format1\n",$key,$count) if $htm_fh;
2817 if ($xls_fh)
2818 {
2819 &set_worksheet_line($ws_global, $row++, 0, [$key,$count], $f_default);
2820 }
059ec3d9
PH
2821 }
2822 }
2823 else {
2824 # We are getting our data from mainlog files.
2825 my $user_pattern_index = 0;
2826 foreach $key (@user_descriptions) {
d5692f86
SC
2827 printf $txt_fh ("$txt_format1\n",$key,$user_pattern_totals[$user_pattern_index]) if $txt_fh;
2828 printf $htm_fh ("$htm_format1\n",$key,$user_pattern_totals[$user_pattern_index]) if $htm_fh;
a83c7e95 2829 $ws_global->write($row++, 0, [$key,$user_pattern_totals[$user_pattern_index]]) if $xls_fh;
059ec3d9
PH
2830 $user_pattern_index++;
2831 }
2832 }
d5692f86
SC
2833 print $txt_fh "\n" if $txt_fh;
2834 print $htm_fh "</table>\n\n" if $htm_fh;
2835 if ($xls_fh)
2836 {
2837 ++$row;
059ec3d9 2838 }
608bc29d
SC
2839
2840 if ($hist_opt > 0) {
2841 my $user_pattern_index = 0;
2842 foreach $key (@user_descriptions) {
2843 print_histogram($key, 'occurence', @{$user_pattern_interval_count[$user_pattern_index]});
2844 $user_pattern_index++;
2845 }
2846 }
059ec3d9
PH
2847}
2848
a83c7e95
SC
2849#######################################################################
2850# print_rejects()
2851#
2852# print_rejects();
2853#
2854# Print statistics about rejected mail.
2855#######################################################################
2856sub print_rejects {
2857 my($format1,$reason);
2858
2859 my $txt_format1 = " %-40s %6d";
2860 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%d</td>";
2861
2862 if ($txt_fh) {
2863 print $txt_fh "Rejected mail by reason\n";
2864 print $txt_fh "-----------------------";
2865 print $txt_fh "\n Total\n";
2866 }
2867 if ($htm_fh) {
2868 print $htm_fh "<hr><a name=\"patterns\"></a><h2>Rejected mail by reason</h2>\n";
2869 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
2870 print $htm_fh "<tr><th>&nbsp;</th><th>Total</th>\n";
2871 }
2872 if ($xls_fh) {
2873 $ws_global->write($row++, $col, "Rejected mail by reason", $f_header2);
2874 &set_worksheet_line($ws_global, $row++, 1, ["Total"], $f_headertab);
2875 }
2876
2877
2878 my $href = ($merge_reports) ? $report_totals{rejected_mail_by_reason} : \%rejected_count_by_reason;
2879 my(@chartdatanames, @chartdatavals_count);
2880
2881 foreach $reason (top_n_sort($topcount, $href, undef, undef)) {
2882 printf $txt_fh ("$txt_format1\n",$reason,$href->{$reason}) if $txt_fh;
2883 printf $htm_fh ("$htm_format1\n",$reason,$href->{$reason}) if $htm_fh;
2884 set_worksheet_line($ws_global, $row++, 0, [$reason,$href->{$reason}], $f_default) if $xls_fh;
2885 push(@chartdatanames, $reason);
2886 push(@chartdatavals_count, $href->{$reason});
2887 }
2888
2889 $row++ if $xls_fh;
2890 print $txt_fh "\n" if $txt_fh;
2891
2892 if ($htm_fh) {
2893 print $htm_fh "</tr></table></td><td>";
2894 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_count > 0)) {
2895 # calculate the graph
2896 my @data = (
2897 \@chartdatanames,
2898 \@chartdatavals_count
2899 );
2900 my $graph = GD::Graph::pie->new(200, 200);
2901 $graph->set(
2902 x_label => 'Rejection Reasons',
2903 y_label => 'Messages',
2904 title => 'By count',
2905 );
2906 my $gd = $graph->plot(\@data) or warn($graph->error);
2907 if ($gd) {
2908 open(IMG, ">$chartdir/rejections_count.png") or die "Could not write $chartdir/rejections_count.png: $!\n";
2909 binmode IMG;
2910 print IMG $gd->png;
2911 close IMG;
2912 print $htm_fh "<img src=\"$chartrel/rejections_count.png\">";
2913 }
2914 }
2915 print $htm_fh "</td></tr></table>\n\n";
2916 }
2917}
2918
2919
2920
2921
059ec3d9 2922
1b4fe9dd
PH
2923#######################################################################
2924# print_transport();
8e669ac1 2925#
1b4fe9dd 2926# print_transport();
8e669ac1 2927#
1b4fe9dd
PH
2928# Print totals by transport.
2929#######################################################################
059ec3d9 2930sub print_transport {
059ec3d9
PH
2931 my(@chartdatanames);
2932 my(@chartdatavals_count);
2933 my(@chartdatavals_vol);
d5692f86
SC
2934 no integer; #Lose this for charting the data.
2935
2936 my $txt_format1 = " %-18s %6s %6d";
2937 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%s</td><td align=\"right\">%d</td>";
059ec3d9 2938
d5692f86
SC
2939 if ($txt_fh) {
2940 print $txt_fh "Deliveries by transport\n";
2941 print $txt_fh "-----------------------";
2942 print $txt_fh "\n Volume Messages\n";
059ec3d9 2943 }
d5692f86 2944 if ($htm_fh) {
30e9ac3f 2945 print $htm_fh "<hr><a name=\"Transport\"></a><h2>Deliveries by Transport</h2>\n";
a83c7e95 2946 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
d5692f86
SC
2947 print $htm_fh "<tr><th>&nbsp;</th><th>Volume</th><th>Messages</th>\n";
2948 }
2949 if ($xls_fh) {
a83c7e95
SC
2950 $ws_global->write(++$row, $col, "Deliveries by transport", $f_header2);
2951 $ws_global->write(++$row, 1, ["Volume", "Messages"], $f_headertab);
059ec3d9
PH
2952 }
2953
2954 my($key);
2955 if ($merge_reports) {
2956 # We are getting our data from previous reports.
2957 foreach $key (sort keys %{$report_totals{transport}}) {
2958 my $count = get_report_total($report_totals{transport}{$key},'Messages');
d5692f86
SC
2959 my @content=($key, volume_rounded($report_totals{transport}{$key}{Volume},
2960 $report_totals{transport}{$key}{'Volume-gigs'}), $count);
059ec3d9
PH
2961 push(@chartdatanames, $key);
2962 push(@chartdatavals_count, $count);
2963 push(@chartdatavals_vol, $report_totals{transport}{$key}{'Volume-gigs'}*$gig + $report_totals{transport}{$key}{Volume} );
d5692f86
SC
2964 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2965 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
a83c7e95 2966 $ws_global->write(++$row, 0, \@content) if $xls_fh;
059ec3d9
PH
2967 }
2968 }
2969 else {
2970 # We are getting our data from mainlog files.
2971 foreach $key (sort keys %transported_data) {
d5692f86
SC
2972 my @content=($key, volume_rounded($transported_data{$key},$transported_data_gigs{$key}),
2973 $transported_count{$key});
059ec3d9
PH
2974 push(@chartdatanames, $key);
2975 push(@chartdatavals_count, $transported_count{$key});
2976 push(@chartdatavals_vol, $transported_data_gigs{$key}*$gig + $transported_data{$key});
d5692f86
SC
2977 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2978 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
a83c7e95 2979 $ws_global->write(++$row, 0, \@content) if $xls_fh;
059ec3d9
PH
2980 }
2981 }
d5692f86
SC
2982 print $txt_fh "\n" if $txt_fh;
2983 if ($htm_fh) {
a83c7e95
SC
2984 print $htm_fh "</tr></table></td><td>";
2985
608bc29d 2986 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_count > 0))
059ec3d9
PH
2987 {
2988 # calculate the graph
2989 my @data = (
2990 \@chartdatanames,
2991 \@chartdatavals_count
2992 );
2993 my $graph = GD::Graph::pie->new(200, 200);
2994 $graph->set(
2995 x_label => 'Transport',
2996 y_label => 'Messages',
2997 title => 'By count',
2998 );
2999 my $gd = $graph->plot(\@data) or warn($graph->error);
3000 if ($gd) {
d5692f86
SC
3001 open(IMG, ">$chartdir/transports_count.png") or die "Could not write $chartdir/transports_count.png: $!\n";
3002 binmode IMG;
3003 print IMG $gd->png;
3004 close IMG;
3005 print $htm_fh "<img src=\"$chartrel/transports_count.png\">";
059ec3d9
PH
3006 }
3007 }
a83c7e95 3008 print $htm_fh "</td><td>";
059ec3d9 3009
608bc29d 3010 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_vol > 0)) {
059ec3d9
PH
3011 my @data = (
3012 \@chartdatanames,
3013 \@chartdatavals_vol
3014 );
3015 my $graph = GD::Graph::pie->new(200, 200);
3016 $graph->set(
3017 title => 'By volume',
3018 );
3019 my $gd = $graph->plot(\@data) or warn($graph->error);
3020 if ($gd) {
a83c7e95 3021 open(IMG, ">$chartdir/transports_vol.png") or die "Could not write $chartdir/transports_vol.png: $!\n";
d5692f86
SC
3022 binmode IMG;
3023 print IMG $gd->png;
3024 close IMG;
3025 print $htm_fh "<img src=\"$chartrel/transports_vol.png\">";
059ec3d9
PH
3026 }
3027 }
a83c7e95 3028
d5692f86
SC
3029 print $htm_fh "</td></tr></table>\n\n";
3030 }
059ec3d9
PH
3031}
3032
3033
3034
1b4fe9dd
PH
3035#######################################################################
3036# print_relay();
8e669ac1 3037#
1b4fe9dd 3038# print_relay();
8e669ac1 3039#
1b4fe9dd
PH
3040# Print our totals by relay.
3041#######################################################################
059ec3d9 3042sub print_relay {
d5692f86 3043 my $row_print_relay=1;
059ec3d9 3044 my $temp = "Relayed messages";
d5692f86 3045 print $htm_fh "<hr><a name=\"$temp\"></a><h2>$temp</h2>\n" if $htm_fh;
059ec3d9
PH
3046 if (scalar(keys %relayed) > 0 || $relayed_unshown > 0) {
3047 my $shown = 0;
3048 my $spacing = "";
d5692f86
SC
3049 my $txt_format = "%7d %s\n => %s\n";
3050 my $htm_format = "<tr><td align=\"right\">%d</td><td>%s</td><td>%s</td>\n";
059ec3d9 3051
d5692f86
SC
3052 printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
3053 if ($htm_fh) {
3054 print $htm_fh "<table border=1>\n";
3055 print $htm_fh "<tr><th>Count</th><th>From</th><th>To</th>\n";
059ec3d9 3056 }
d5692f86
SC
3057 if ($xls_fh) {
3058 $ws_relayed->write($row_print_relay++, $col, $temp, $f_header2);
3059 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, ["Count", "From", "To"], $f_headertab);
059ec3d9
PH
3060 }
3061
d5692f86 3062
059ec3d9
PH
3063 my($key);
3064 foreach $key (sort keys %relayed) {
3065 my $count = $relayed{$key};
3066 $shown += $count;
3067 $key =~ s/[HA]=//g;
3068 my($one,$two) = split(/=> /, $key);
d5692f86
SC
3069 my @content=($count, $one, $two);
3070 printf $txt_fh ($txt_format, @content) if $txt_fh;
3071 printf $htm_fh ($htm_format, @content) if $htm_fh;
3072 if ($xls_fh)
3073 {
3074 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, \@content);
3075 }
059ec3d9
PH
3076 $spacing = "\n";
3077 }
d5692f86
SC
3078
3079 print $htm_fh "</table>\n<p>\n" if $htm_fh;
3080 print $txt_fh "${spacing}Total: $shown (plus $relayed_unshown unshown)\n\n" if $txt_fh;
3081 print $htm_fh "${spacing}Total: $shown (plus $relayed_unshown unshown)\n\n" if $htm_fh;
3082 if ($xls_fh)
3083 {
3084 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, [$shown, "Sum of shown" ]);
3085 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, [$relayed_unshown, "unshown"]);
3086 $row_print_relay++;
3087 }
059ec3d9
PH
3088 }
3089 else {
d5692f86
SC
3090 print $txt_fh "No relayed messages\n-------------------\n\n" if $txt_fh;
3091 print $htm_fh "No relayed messages\n\n" if $htm_fh;
3092 if ($xls_fh)
3093 {
3094 $row_print_relay++;
3095 }
059ec3d9 3096 }
059ec3d9
PH
3097}
3098
3099
3100
1b4fe9dd
PH
3101#######################################################################
3102# print_errors();
8e669ac1 3103#
1b4fe9dd 3104# print_errors();
8e669ac1 3105#
1b4fe9dd
PH
3106# Print our errors. In HTML, we display them as a list rather than a table -
3107# Netscape doesn't like large tables!
3108#######################################################################
059ec3d9
PH
3109sub print_errors {
3110 my $total_errors = 0;
d5692f86 3111 $row=1;
059ec3d9
PH
3112
3113 if (scalar(keys %errors_count) != 0) {
3114 my $temp = "List of errors";
d5692f86
SC
3115 my $htm_format = "<li>%d - %s\n";
3116
3117 printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
3118 if ($htm_fh) {
3119 print $htm_fh "<hr><a name=\"errors\"></a><h2>$temp</h2>\n";
3120 print $htm_fh "<ul><li><b>Count - Error</b>\n";
059ec3d9 3121 }
d5692f86
SC
3122 if ($xls_fh)
3123 {
3124 $ws_errors->write($row++, 0, $temp, $f_header2);
3125 &set_worksheet_line($ws_errors, $row++, 0, ["Count", "Error"], $f_headertab);
059ec3d9
PH
3126 }
3127
d5692f86 3128
059ec3d9
PH
3129 my($key);
3130 foreach $key (sort keys %errors_count) {
3131 my $text = $key;
3132 chomp($text);
d5692f86 3133 $text =~ s/\s\s+/ /g; #Convert multiple spaces to a single space.
059ec3d9 3134 $total_errors += $errors_count{$key};
d5692f86
SC
3135
3136 if ($txt_fh) {
3137 printf $txt_fh ("%5d ", $errors_count{$key});
3138 my $text_remaining = $text;
3139 while (length($text_remaining) > 65) {
3140 my($first,$rest) = $text_remaining =~ /(.{50}\S*)\s+(.+)/;
3141 last if !$first;
3142 printf $txt_fh ("%s\n\t ", $first);
3143 $text_remaining = $rest;
3144 }
3145 printf $txt_fh ("%s\n\n", $text_remaining);
3146 }
3147
3148 if ($htm_fh) {
8e669ac1 3149
059ec3d9
PH
3150 #Translate HTML tag characters. Sergey Sholokh.
3151 $text =~ s/\</\&lt\;/g;
3152 $text =~ s/\>/\&gt\;/g;
3153
d5692f86 3154 printf $htm_fh ($htm_format,$errors_count{$key},$text);
059ec3d9 3155 }
d5692f86
SC
3156 if ($xls_fh)
3157 {
3158 &set_worksheet_line($ws_errors, $row++, 0, [$errors_count{$key},$text]);
059ec3d9
PH
3159 }
3160 }
059ec3d9
PH
3161
3162 $temp = "Errors encountered: $total_errors";
d5692f86
SC
3163
3164 if ($txt_fh) {
3165 print $txt_fh $temp, "\n";
3166 print $txt_fh "-" x length($temp),"\n";
3167 }
3168 if ($htm_fh) {
3169 print $htm_fh "</ul>\n<p>\n";
3170 print $htm_fh $temp, "\n";
3171 }
3172 if ($xls_fh)
3173 {
3174 &set_worksheet_line($ws_errors, $row++, 0, [$total_errors, "Sum of Errors encountered"]);
3175 }
059ec3d9
PH
3176 }
3177
3178}
3179
3180
1b4fe9dd
PH
3181#######################################################################
3182# parse_old_eximstat_reports();
8e669ac1 3183#
1b4fe9dd 3184# parse_old_eximstat_reports($fh);
8e669ac1 3185#
1b4fe9dd 3186# Parse old eximstat output so we can merge daily stats to weekly stats and weekly to monthly etc.
8e669ac1 3187#
1b4fe9dd
PH
3188# To test that the merging still works after changes, do something like the following.
3189# All the diffs should produce no output.
8e669ac1 3190#
1b4fe9dd 3191# options='-bydomain -byemail -byhost -byedomain'
608bc29d 3192# options="$options -show_rt1,2,4 -show_dt 1,2,4"
1b4fe9dd
PH
3193# options="$options -pattern 'Completed Messages' /Completed/"
3194# options="$options -pattern 'Received Messages' /<=/"
8e669ac1 3195#
1b4fe9dd
PH
3196# ./eximstats $options mainlog > mainlog.txt
3197# ./eximstats $options -merge mainlog.txt > mainlog.2.txt
3198# diff mainlog.txt mainlog.2.txt
8e669ac1 3199#
1b4fe9dd
PH
3200# ./eximstats $options -html mainlog > mainlog.html
3201# ./eximstats $options -merge -html mainlog.txt > mainlog.2.html
3202# diff mainlog.html mainlog.2.html
8e669ac1 3203#
1b4fe9dd
PH
3204# ./eximstats $options -merge mainlog.html > mainlog.3.txt
3205# diff mainlog.txt mainlog.3.txt
8e669ac1 3206#
1b4fe9dd
PH
3207# ./eximstats $options -merge -html mainlog.html > mainlog.3.html
3208# diff mainlog.html mainlog.3.html
8e669ac1 3209#
1b4fe9dd
PH
3210# ./eximstats $options -nvr mainlog > mainlog.nvr.txt
3211# ./eximstats $options -merge mainlog.nvr.txt > mainlog.4.txt
3212# diff mainlog.txt mainlog.4.txt
8e669ac1 3213#
1b4fe9dd
PH
3214# # double_mainlog.txt should have twice the values that mainlog.txt has.
3215# ./eximstats $options mainlog mainlog > double_mainlog.txt
3216#######################################################################
059ec3d9
PH
3217sub parse_old_eximstat_reports {
3218 my($fh) = @_;
3219
3220 my(%league_table_value_entered, %league_table_value_was_zero, %table_order);
3221
608bc29d
SC
3222 my(%user_pattern_index);
3223 my $user_pattern_index = 0;
3224 map {$user_pattern_index{$_} = $user_pattern_index++} @user_descriptions;
3225 my $user_pattern_keys = join('|', @user_descriptions);
3226
059ec3d9 3227 while (<$fh>) {
d5692f86 3228 PARSE_OLD_REPORT_LINE:
059ec3d9
PH
3229 if (/Exim statistics from ([\d\-]+ [\d:]+(\s+[\+\-]\d+)?) to ([\d\-]+ [\d:]+(\s+[\+\-]\d+)?)/) {
3230 $begin = $1 if ($1 lt $begin);
3231 $end = $3 if ($3 gt $end);
3232 }
3233 elsif (/Grand total summary/) {
a83c7e95
SC
3234 # Fill in $report_totals{Received|Delivered}{Volume|Messages|Addresses|Hosts|Domains|...|Delayed|DelayedPercent|Failed|FailedPercent}
3235 my(@fields, @delivered_fields);
3236 my $doing_table = 0;
059ec3d9 3237 while (<$fh>) {
d5692f86
SC
3238 $_ = html2txt($_); #Convert general HTML markup to text.
3239 s/At least one addr//g; #Another part of the HTML output we don't want.
059ec3d9 3240
a83c7e95
SC
3241# TOTAL Volume Messages Addresses Hosts Domains Delayed Failed
3242# Received 26MB 237 177 23 8 3.4% 28 11.8%
3243# Delivered 13MB 233 250 99 88
d5692f86 3244 if (/TOTAL\s+(.*?)\s*$/) {
a83c7e95
SC
3245 $doing_table = 1;
3246 @delivered_fields = split(/\s+/,$1);
3247
059ec3d9 3248 #Delayed and Failed have two columns each, so add the extra field names in.
a83c7e95
SC
3249 splice(@delivered_fields,-1,1,'DelayedPercent','Failed','FailedPercent');
3250
3251 # Addresses only figure in the Delivered row, so remove them from the
3252 # normal fields.
3253 @fields = grep !/Addresses/, @delivered_fields;
d5692f86 3254 }
a83c7e95 3255 elsif (/(Received)\s+(.*?)\s*$/) {
d5692f86
SC
3256 print STDERR "Parsing $_" if $debug;
3257 add_to_totals($report_totals{$1},\@fields,$2);
3258 }
a83c7e95
SC
3259 elsif (/(Delivered)\s+(.*?)\s*$/) {
3260 print STDERR "Parsing $_" if $debug;
3261 add_to_totals($report_totals{$1},\@delivered_fields,$2);
3262 my $data = $2;
3263 # If we're merging an old report which doesn't include addresses,
3264 # then use the Messages field instead.
3265 unless (grep(/Addresses/, @delivered_fields)) {
3266 my %tmp;
3267 line_to_hash(\%tmp,\@delivered_fields,$data);
3268 add_to_totals($report_totals{Delivered},['Addresses'],$tmp{Messages});
3269 }
3270 }
30e9ac3f 3271 elsif (/(Temp Rejects|Rejects|Ham|Spam)\s+(.*?)\s*$/) {
a83c7e95
SC
3272 print STDERR "Parsing $_" if $debug;
3273 add_to_totals($report_totals{$1},['Messages','Hosts'],$2);
3274 }
3275 else {
3276 last if $doing_table;
3277 }
059ec3d9
PH
3278 }
3279 }
3280
3281 elsif (/User Specified Patterns/i) {
3282#User Specified Patterns
3283#-----------------------
3284# Total
3285# Description 85
3286
d5692f86 3287 while (<$fh>) { last if (/Total/); } #Wait until we get the table headers.
059ec3d9 3288 while (<$fh>) {
d5692f86
SC
3289 print STDERR "Parsing $_" if $debug;
3290 $_ = html2txt($_); #Convert general HTML markup to text.
3291 if (/^\s*(.*?)\s+(\d+)\s*$/) {
3292 $report_totals{patterns}{$1} = {} unless (defined $report_totals{patterns}{$1});
3293 add_to_totals($report_totals{patterns}{$1},['Total'],$2);
3294 }
3295 last if (/^\s*$/); #Finished if we have a blank line.
059ec3d9
PH
3296 }
3297 }
3298
608bc29d
SC
3299 elsif (/(^|<h2>)($user_pattern_keys) per /o) {
3300 # Parse User defined pattern histograms if they exist.
3301 parse_histogram($fh, $user_pattern_interval_count[$user_pattern_index{$2}] );
3302 }
3303
3304
059ec3d9
PH
3305 elsif (/Deliveries by transport/i) {
3306#Deliveries by transport
3307#-----------------------
3308# Volume Messages
3309# :blackhole: 70KB 51
3310# address_pipe 655KB 1
3311# smtp 11MB 151
3312
d5692f86 3313 while (<$fh>) { last if (/Volume/); } #Wait until we get the table headers.
059ec3d9 3314 while (<$fh>) {
d5692f86
SC
3315 print STDERR "Parsing $_" if $debug;
3316 $_ = html2txt($_); #Convert general HTML markup to text.
3317 if (/(\S+)\s+(\d+\S*\s+\d+)/) {
3318 $report_totals{transport}{$1} = {} unless (defined $report_totals{transport}{$1});
3319 add_to_totals($report_totals{transport}{$1},['Volume','Messages'],$2);
3320 }
3321 last if (/^\s*$/); #Finished if we have a blank line.
059ec3d9
PH
3322 }
3323 }
608bc29d
SC
3324 elsif (/Messages received per/) {
3325 parse_histogram($fh, \@received_interval_count);
3326 }
3327 elsif (/Deliveries per/) {
3328 parse_histogram($fh, \@delivered_interval_count);
059ec3d9
PH
3329 }
3330
608bc29d
SC
3331 #elsif (/Time spent on the queue: (all messages|messages with at least one remote delivery)/) {
3332 elsif (/(Time spent on the queue|Delivery times|Receipt times): ((\S+) messages|messages with at least one remote delivery)((<[^>]*>)*\s*)$/) {
059ec3d9
PH
3333#Time spent on the queue: all messages
3334#-------------------------------------
3335#
3336#Under 1m 217 91.9% 91.9%
3337# 5m 2 0.8% 92.8%
3338# 3h 8 3.4% 96.2%
3339# 6h 7 3.0% 99.2%
3340# 12h 2 0.8% 100.0%
3341
3342 # Set a pointer to the queue bin so we can use the same code
3343 # block for both all messages and remote deliveries.
608bc29d
SC
3344 #my $bin_aref = ($1 eq 'all messages') ? \@qt_all_bin : \@qt_remote_bin;
3345 my($bin_aref, $times_aref, $overflow_sref);
3346 if ($1 eq 'Time spent on the queue') {
3347 $times_aref = \@queue_times;
3348 if ($2 eq 'all messages') {
3349 $bin_aref = \@qt_all_bin;
3350 $overflow_sref = \$qt_all_overflow;
3351 }
3352 else {
3353 $bin_aref = \@qt_remote_bin;
3354 $overflow_sref = \$qt_remote_overflow;
3355 }
3356 }
3357 elsif ($1 eq 'Delivery times') {
3358 $times_aref = \@delivery_times;
3359 if ($2 eq 'all messages') {
3360 $bin_aref = \@dt_all_bin;
3361 $overflow_sref = \$dt_all_overflow;
3362 }
3363 else {
3364 $bin_aref = \@dt_remote_bin;
3365 $overflow_sref = \$dt_remote_overflow;
3366 }
3367 }
3368 else {
3369 unless (exists $rcpt_times_bin{$3}) {
3370 initialise_rcpt_times($3);
3371 }
3372 $bin_aref = $rcpt_times_bin{$3};
3373 $times_aref = \@rcpt_times;
3374 $overflow_sref = \$rcpt_times_overflow{$3};
3375 }
3376
3377
a83c7e95 3378 my ($blank_lines, $reached_table) = (0,0);
059ec3d9 3379 while (<$fh>) {
d5692f86 3380 $_ = html2txt($_); #Convert general HTML markup to text.
a83c7e95
SC
3381 # The table is preceded by one blank line, and has one blank line
3382 # following it. As the table may be empty, the best way to determine
3383 # that we've finished it is to look for the second blank line.
3384 ++$blank_lines if /^\s*$/;
3385 last if ($blank_lines >=2); #Finished the table ?
3386 $reached_table = 1 if (/\d/);
d5692f86
SC
3387 next unless $reached_table;
3388 my $previous_seconds_on_queue = 0;
3389 if (/^\s*(Under|Over|)\s+(\d+[smhdw])\s+(\d+)/) {
3390 print STDERR "Parsing $_" if $debug;
968178ac
AM
3391 my($modifier,$formatted_time,$count) = ($1,$2,$3);
3392 my $seconds = unformat_time($formatted_time);
d5692f86
SC
3393 my $time_on_queue = ($seconds + $previous_seconds_on_queue) / 2;
3394 $previous_seconds_on_queue = $seconds;
3395 $time_on_queue = $seconds * 2 if ($modifier eq 'Over');
3396 my($i);
608bc29d
SC
3397 for ($i = 0; $i <= $#$times_aref; $i++) {
3398 if ($time_on_queue < $times_aref->[$i]) {
d5692f86
SC
3399 $$bin_aref[$i] += $count;
3400 last;
3401 }
3402 }
608bc29d
SC
3403 $$overflow_sref += $count if ($i > $#$times_aref);
3404
d5692f86 3405 }
059ec3d9
PH
3406 }
3407 }
3408
3409 elsif (/Relayed messages/) {
3410#Relayed messages
3411#----------------
3412#
3413# 1 addr.domain.com [1.2.3.4] a.user@domain.com
3414# => addr2.domain2.com [5.6.7.8] a2.user2@do