projects / exim.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Add gnutls_require_{kx,mac,protocols}.
[exim.git] / src / src / eximstats.src
CommitLineData
059ec3d9 1#!PERL_COMMAND -w
a83c7e95 2# $Cambridge: exim/src/src/eximstats.src,v 1.11 2006/11/16 16:16:30 steve Exp $
059ec3d9
PH		 3
4# Copyright (c) 2001 University of Cambridge.
5# See the file NOTICE for conditions of use and distribution.
6
7# Perl script to generate statistics from one or more Exim log files.
8
9# Usage: eximstats [<options>] <log file> <log file> ...
10
11# 1996-05-21: Ignore lines not starting with valid date/time, just in case
12# these get into a log file.
13# 1996-11-19: Add the -h option to control the size of the histogram,
14# and optionally turn it off.
15# Use some Perl 5 things; it should be everywhere by now.
16# Add the Perl -w option and rewrite so no warnings are given.
17# Add the -t option to control the length of the "top" listing.
18# Add the -ne, -nt options to turn off errors and transport
19# information.
20# Add information about length of time on queue, and -q<list> to
21# control the intervals and turn it off.
22# Add count and percentage of delayed messages to the Received
23# line.
24# Show total number of errors.
25# Add count and percentage of messages with errors to Received
26# line.
27# Add information about relaying and -nr to suppress it.
28# 1997-02-03 Merged in some of the things Nigel Metheringham had done:
29# Re-worded headings
30# Added received histogram as well as delivered
31# Added local senders' league table
32# Added local recipients' league table
33# 1997-03-10 Fixed typo "destinationss"
34# Allow for intermediate address between final and original
35# when testing for relaying
36# Give better message when no input
37# 1997-04-24 Fixed bug in layout of error listing that was depending on
38# text length (output line got repeated).
39# 1997-05-06 Bug in option decoding when only one option.
40# Overflow bug when handling very large volumes.
41# 1997-10-28 Updated to handle revised log format that might show
42# HELO name as well as host name before IP number
43# 1998-01-26 Bugs in the function for calculating the number of seconds
44# since 1970 from a log date
45# 1998-02-02 Delivery to :blackhole: doesn't have a T= entry in the log
46# line; cope with this, thereby avoiding undefined problems
47# Very short log line gave substring error
48# 1998-02-03 A routed delivery to a local transport may not have <> in the
49# log line; terminate the address at white space, not <
50# 1998-09-07 If first line of input was a => line, $thissize was undefined;
51# ensure it is zero.
52# 1998-12-21 Adding of $thissize from => line should have been adding $size.
53# Oops. Should have looked more closely when fixing the previous
54# bug!
55# 1999-11-12 Increased the field widths for printed integers; numbers are
56# bigger than originally envisaged.
57# 2001-03-21 Converted seconds() routine to use Time::Local, fixing a bug
58# whereby seconds($timestamp) - id_seconds($id) gave an
59# incorrect result.
60# Added POD documentation.
61# Moved usage instructions into help() subroutine.
62# Added 'use strict' and declared all global variables.
63# Added '-html' flag and resultant code.
64# Added '-cache' flag and resultant code.
65# Added add_volume() routine and converted all volume variables
66# to use it, fixing the overflow problems for individual hosts
67# on large sites.
68# Converted all volume output to GB/MB/KB as appropriate.
69# Don't store local user stats if -nfl is specified.
70# Modifications done by: Steve Campbell (<steve@computurn.com>)
71# 2001-04-02 Added the -t_remote_users flag. Steve Campbell.
72# 2001-10-15 Added the -domain flag. Steve Campbell.
73# 2001-10-16 Accept files on STDIN or on the command line. Steve Campbell.
74# 2001-10-21 Removed -domain flag and added -bydomain, -byhost, and -byemail.
75# We now generate our main parsing subroutine as an eval statement
76# which improves performance dramatically when not all the results
77# are required. We also cache the last timestamp to time convertion.
78#
79# NOTE: 'Top 50 destinations by (message count|volume)' lines are
80# now 'Top N (host|email|domain) destinations by (message count|volume)'
81# where N is the topcount. Steve Campbell.
82#
83# 2001-10-30 V1.16 Joachim Wieland.
84# Fixed minor bugs in add_volume() when taking over this version
85# for use in Exim 4: -w gave uninitialized value warnings in
86# two situations: for the first addition to a counter, and if
87# there were never any gigabytes, thereby leaving the $gigs
88# value unset.
89# Initialized $last_timestamp to stop a -w uninitialized warning.
90# Minor layout tweak for grand totals (nitpicking).
91# Put the IP addresses for relaying stats in [] and separated by
92# a space from the domain name.
93# Removed the IPv4-specific address test when picking out addresses
94# for relaying. Anything inside [] is OK.
95#
96# 2002-07-02 Philip Hazel
97# Fixed "uninitialized variable" message that occurred for relay
98# messages that arrived from H=[1.2.3.4] hosts (no name shown).
99# This bug didn't affect the output.
100#
101# 2002-04-15 V1.17 Joachim Wieland.
102# Added -charts, -chartdir. -chartrel options which use
103# GD::Graph modules to create graphical charts of the statistics.
104#
105# 2002-04-15 V1.18 Steve Campbell.
106# Added a check for $domain to to stop a -w uninitialized warning.
107# Added -byemaildomain option.
108# Only print HTML header links to included tables!
109#
110# 2002-08-02 V1.19 Steve Campbell.
111# Changed the debug mode to dump the parser onto STDERR rather
112# than STDOUT. Documented the -d flag into the help().
113# Rejoined the divergent 2002-04-15 and 2002-07-02 releases.
114#
115# 2002-08-21 V1.20 Steve Campbell.
116# Added the '-merge' option to allow merging of previous reports.
117# Fixed a missing semicolon when doing -bydomain.
118# Make volume charts plot the data gigs and bytes rather than just bytes.
119# Only process log lines with $flag =~ /<=|=>|->|==|\*\*|Co/
120# Converted Emaildomain to Edomain - the column header was too wide!
121# This changes the text output slightly. You can revert to the old
122# column widths by changing $COLUMN_WIDTHS to 7;
123#
124# 2002-09-04 V1.21 Andreas J Mueller
125# Local deliveries domain now defaults to 'localdomain'.
126# Don't match F=<From> when looking for the user.
127#
128# 2002-09-05 V1.22 Steve Campbell
129# Fixed a perl 5.005 incompatibility problem ('our' variables).
130#
131# 2002-09-11 V1.23 Steve Campbell
132# Stopped -charts option from throwing errors on null data.
133# Don't print out 'Errors encountered' unless there are any.
134
135# 2002-10-21 V1.23a Philip Hazel - patch from Tony Finch put in until
136# Steve's eximstats catches up.
137# Handle log files that include the timezone after the timestamp.
138# Switch to assuming that log timestamps are in local time, with
139# an option for UTC timestamps, as in Exim itself.
140#
141# 2003-02-05 V1.24 Steve Campbell
142# Added in Sergey Sholokh's code to convert '<' and '>' characters
143# in HTML output. Also added code to convert them back with -merge.
144# Fixed timestamp offsets to convert to seconds rather than minutes.
145# Updated -merge to work with output files using timezones.
146# Added cacheing to speed up the calculation of timezone offsets.
147#
148# 2003-02-07 V1.25 Steve Campbell
149# Optimised the usage of mktime() in the seconds subroutine.
150# Removed the now redundant '-cache' option.
151# html2txt() now explicitly matches HTML tags.
152# Implemented a new sorting algorithm - the top_n_sort() routine.
153# Added Danny Carroll's '-nvr' flag and code.
154#
155# 2003-03-13 V1.26 Steve Campbell
156# Implemented HTML compliance changes recommended by Bernard Massot.
157# Bug fix to allow top_n_sort() to handle null keys.
158# Convert all domains and edomains to lowercase.
159# Remove preceding dots from domains.
160#
161# 2003-03-13 V1.27 Steve Campbell
162# Replaced border attributes with 'border=1', as recommended by
163# Bernard Massot.
164#
165# 2003-06-03 V1.28 John Newman
166# Added in the ability to skip over the parsing and evaulation of
167# specific transports as passed to eximstats via the new "-nt/.../"
168# command line argument. This new switch allows the viewing of
169# not more accurate statistics but more applicable statistics when
170# special transports are in use (ie; SpamAssassin). We need to be
171# able to ignore transports such as this otherwise the resulting
172# local deliveries are significantly skewed (doubled)...
173#
174# 2003-11-06 V1.29 Steve Campbell
175# Added the '-pattern "Description" "/pattern/"' option.
176#
177# 2004-02-17 V1.30 Steve Campbell
178# Added warnings if required GD::Graph modules are not available or
179# insufficient -chart* options are specified.
180#
8e669ac1 181# 2004-02-20 V1.31 Andrea Balzi
059ec3d9
PH		 182# Only show the Local Sender/Destination links if the tables exist.
183#
1b4fe9dd
PH		 184# 2004-07-05 V1.32 Steve Campbell
185# Fix '-merge -h0' divide by zero error.
186#
187# 2004-07-15 V1.33 Steve Campbell
188# Documentation update - I've converted the subroutine
189# documentation from POD to comments.
8974000d
SC		 190#
191# 2004-12-10 V1.34 Steve Campbell
192# Eximstats can now parse syslog lines as well as mainlog lines.
193#
884c2af8 194# 2004-12-20 V1.35 Wouter Verhelst
d5692f86 195# Pie charts by volume were actually generated by count. Fixed.
50adf73a
SC		 196#
197# 2005-02-07 V1.36 Gregor Herrmann / Steve Campbell
198# Added average sizes to HTML Top tables.
d5692f86
SC		 199#
200# 2005-04-26 V1.37 Frank Heydlauf
201# Added -xls and the ability to specify output files.
202#
203# 2005-04-29 V1.38 Steve Campbell
204# Use FileHandles for outputing results.
205# Allow any combination of xls, txt, and html output.
206# Fixed display of large numbers with -nvr option
207# Fixed merging of reports with empty tables.
208#
209# 2005-05-27 V1.39 Steve Campbell
210# Added the -include_original_destination flag
211# Removed tabs and trailing whitespace.
212#
608bc29d
SC		 213# 2005-06-03 V1.40 Steve Campbell
214# Whilst parsing the mainlog(s), store information about
215# the messages in a hash of arrays rather than using
216# individual hashes. This is a bit cleaner and results in
217# dramatic memory savings, albeit at a slight CPU cost.
218#
219# 2005-06-15 V1.41 Steve Campbell
220# Added the -show_rt<list> flag.
221# Added the -show_dt<list> flag.
222#
223# 2005-06-24 V1.42 Steve Campbell
224# Added Histograms for user specified patterns.
d5692f86 225#
81aad8c9
SC		 226# 2005-06-30 V1.43 Steve Campbell
227# Bug fix for V1.42 with -h0 specified. Spotted by Chris Lear.
228#
a83c7e95
SC		 229# 2005-07-26 V1.44 Steve Campbell
230# Use a glob alias rather than an array ref in the generated
231# parser. This improves both readability and performance.
232#
233# 2005-09-30 V1.45 Marco Gaiarin / Steve Campbell
234# Collect SpamAssassin and rejection statistics.
235# Don't display local sender or destination tables unless
236# there is data to show.
237# Added average volumes into the top table text output.
238#
239# 2006-02-07 V1.46 Steve Campbell
240# Collect data on the number of addresses (recipients)
241# as well as the number of messages.
242#
243# 2006-05-05 V1.47 Steve Campbell
244# Added 'Message too big' to the list of mail rejection
245# reasons (thanks to Marco Gaiarin).
246#
247# 2006-06-05 V1.48 Steve Campbell
248# Mainlog lines which have GMT offsets and are too short to
249# have a flag are now skipped.
250#
251# 2006-11-10 V1.49 Alain Williams
252# Added the -emptyok flag.
253#
254# 2006-11-16 V1.50 Steve Campbell
255# Fixes for obtaining the IP address from reject messages.
256#
257#
d5692f86
SC		 258#
259# For documentation on the logfile format, see
260# http://www.exim.org/exim-html-4.50/doc/html/spec_48.html#IX2793
059ec3d9
PH		 261
262=head1 NAME
263
8974000d 264eximstats - generates statistics from Exim mainlog or syslog files.
059ec3d9
PH		 265
266=head1 SYNOPSIS
267
d5692f86 268 eximstats [Output] [Options] mainlog1 mainlog2 ...
059ec3d9
PH		 269 eximstats -merge [Options] report.1.txt report.2.txt ... > weekly_report.txt
270
d5692f86
SC		 271=head2 Output:
272
273=over 4
274
275=item B<-txt>
276
277Output the results in plain text to STDOUT.
278
279=item B<-txt>=I<filename>
280
281Output the results in plain text. Filename '-' for STDOUT is accepted.
282
283=item B<-html>
284
285Output the results in HTML to STDOUT.
286
287=item B<-html>=I<filename>
288
289Output the results in HTML. Filename '-' for STDOUT is accepted.
290
291=item B<-xls>
292
293Output the results in Excel compatible Format to STDOUT.
294Requires the Spreadsheet::WriteExcel CPAN module.
295
296=item B<-xls>=I<filename>
297
298Output the results in Excel compatible format. Filename '-' for STDOUT is accepted.
299
300
301=back
302
303=head2 Options:
059ec3d9
PH		 304
305=over 4
306
307=item B<-h>I<number>
308
309histogram divisions per hour. The default is 1, and
3100 suppresses histograms. Valid values are:
311
3120, 1, 2, 3, 5, 10, 15, 20, 30 or 60.
313
314=item B<-ne>
315
316Don't display error information.
317
318=item B<-nr>
319
320Don't display relaying information.
321
322=item B<-nr>I</pattern/>
323
324Don't display relaying information that matches.
325
326=item B<-nt>
327
328Don't display transport information.
329
330=item B<-nt>I</pattern/>
331
332Don't display transport information that matches
333
334=item B<-q>I<list>
335
336List of times for queuing information single 0 item suppresses.
337
338=item B<-t>I<number>
339
340Display top <number> sources/destinations
341default is 50, 0 suppresses top listing.
342
343=item B<-tnl>
344
345Omit local sources/destinations in top listing.
346
347=item B<-t_remote_users>
348
349Include remote users in the top source/destination listings.
350
d5692f86
SC		 351=item B<-include_original_destination>
352
353Include the original destination email addresses rather than just
354using the final ones.
355Useful for finding out which of your mailing lists are receiving mail.
356
608bc29d
SC		 357=item B<-show_dt>I<list>
358
359Show the delivery times (B<DT>)for all the messages.
360
361Exim must have been configured to use the +delivery_time logging option
362for this option to work.
363
364I<list> is an optional list of times. Eg -show_dt1,2,4,8 will show
365the number of messages with delivery times under 1 second, 2 seconds, 4 seconds,
3668 seconds, and over 8 seconds.
367
368=item B<-show_rt>I<list>
369
370Show the receipt times for all the messages. The receipt time is
371defined as the Completed hh:mm:ss - queue_time_overall - the Receipt hh:mm:ss.
372These figures will be skewed by pipelined messages so might not be that useful.
373
374Exim must have been configured to use the +queue_time_overall logging option
375for this option to work.
376
377I<list> is an optional list of times. Eg -show_rt1,2,4,8 will show
378the number of messages with receipt times under 1 second, 2 seconds, 4 seconds,
3798 seconds, and over 8 seconds.
380
059ec3d9
PH		 381=item B<-byhost>
382
383Show results by sending host. This may be combined with
384B<-bydomain> and/or B<-byemail> and/or B<-byedomain>. If none of these options
385are specified, then B<-byhost> is assumed as a default.
386
387=item B<-bydomain>
388
389Show results by sending domain.
390May be combined with B<-byhost> and/or B<-byemail> and/or B<-byedomain>.
391
392=item B<-byemail>
393
394Show results by sender's email address.
395May be combined with B<-byhost> and/or B<-bydomain> and/or B<-byedomain>.
396
397=item B<-byemaildomain> or B<-byedomain>
398
399Show results by sender's email domain.
400May be combined with B<-byhost> and/or B<-bydomain> and/or B<-byemail>.
401
402=item B<-pattern> I<Description> I</Pattern/>
403
404Look for the specified pattern and count the number of lines in which it appears.
405This option can be specified multiple times. Eg:
406
407 -pattern 'Refused connections' '/refused connection/'
408
409
410=item B<-merge>
411
412This option allows eximstats to merge old eximstat reports together. Eg:
413
414 eximstats mainlog.sun > report.sun.txt
415 eximstats mainlog.mon > report.mon.txt
416 eximstats mainlog.tue > report.tue.txt
417 eximstats mainlog.wed > report.web.txt
418 eximstats mainlog.thu > report.thu.txt
419 eximstats mainlog.fri > report.fri.txt
420 eximstats mainlog.sat > report.sat.txt
421 eximstats -merge report.*.txt > weekly_report.txt
422 eximstats -merge -html report.*.txt > weekly_report.html
423
424=over 4
425
426=item *
427
428You can merge text or html reports and output the results as text or html.
429
430=item *
431
432You can use all the normal eximstat output options, but only data
433included in the original reports can be shown!
434
435=item *
436
437When merging reports, some loss of accuracy may occur in the top I<n> lists.
438This will be towards the ends of the lists.
439
440=item *
441
442The order of items in the top I<n> lists may vary when the data volumes
443round to the same value.
444
445=back
446
059ec3d9
PH		 447=item B<-charts>
448
449Create graphical charts to be displayed in HTML output.
d5692f86 450Only valid in combination with I<-html>.
059ec3d9
PH		 451
452This requires the following modules which can be obtained
453from http://www.cpan.org/modules/01modules.index.html
454
455=over 4
456
457=item GD
458
459=item GDTextUtil
460
461=item GDGraph
462
463=back
464
465To install these, download and unpack them, then use the normal perl installation procedure:
466
467 perl Makefile.PL
468 make
469 make test
470 make install
471
472=item B<-chartdir>I <dir>
473
474Create the charts in the directory <dir>
475
476=item B<-chartrel>I <dir>
477
478Specify the relative directory for the "img src=" tags from where to include
479the charts
480
a83c7e95
SC		 481=item B<-emptyok>
482
483Specify that it's OK to not find any valid log lines. Without this
484we will output an error message if we don't find any.
485
059ec3d9
PH		 486=item B<-d>
487
488Debug flag. This outputs the eval()'d parser onto STDOUT which makes it
489easier to trap errors in the eval section. Remember to add 1 to the line numbers to allow for the
490title!
491
492=back
493
494=head1 DESCRIPTION
495
8974000d 496Eximstats parses exim mainlog and syslog files to output a statistical
059ec3d9 497analysis of the messages processed. By default, a text
d5692f86
SC		 498analysis is generated, but you can request other output formats
499using flags. See the help (B<-help>) to learn
059ec3d9
PH		 500about how to create charts from the tables.
501
502=head1 AUTHOR
503
504There is a web site at http://www.exim.org - this contains details of the
505mailing list exim-users@exim.org.
506
507=head1 TO DO
508
509This program does not perfectly handle messages whose received
510and delivered log lines are in different files, which can happen
511when you have multiple mail servers and a message cannot be
512immeadiately delivered. Fixing this could be tricky...
513
d5692f86
SC		 514Merging of xls files is not (yet) possible. Be free to implement :)
515
059ec3d9
PH		 516=cut
517
518use integer;
519use strict;
d5692f86 520use IO::File;
059ec3d9
PH		 521
522# use Time::Local; # PH/FANF
523use POSIX;
524
d5692f86 525use vars qw($HAVE_GD_Graph_pie $HAVE_GD_Graph_linespoints $HAVE_Spreadsheet_WriteExcel);
059ec3d9
PH		 526eval { require GD::Graph::pie; };
527$HAVE_GD_Graph_pie = $@ ? 0 : 1;
528eval { require GD::Graph::linespoints; };
529$HAVE_GD_Graph_linespoints = $@ ? 0 : 1;
d5692f86
SC		 530eval { require Spreadsheet::WriteExcel; };
531$HAVE_Spreadsheet_WriteExcel = $@ ? 0 : 1;
059ec3d9
PH		 532
533
534##################################################
535# Static data #
536##################################################
537# 'use vars' instead of 'our' as perl5.005 is still in use out there!
538use vars qw(@tab62 @days_per_month $gig);
539use vars qw($VERSION);
540use vars qw($COLUMN_WIDTHS);
608bc29d 541use vars qw($WEEK $DAY $HOUR $MINUTE);
059ec3d9
PH		 542
543
544@tab62 =
545 (0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, # 0-9
546 0,10,11,12,13,14,15,16,17,18,19,20, # A-K
547 21,22,23,24,25,26,27,28,29,30,31,32, # L-W
548 33,34,35, 0, 0, 0, 0, 0, # X-Z
549 0,36,37,38,39,40,41,42,43,44,45,46, # a-k
550 47,48,49,50,51,52,53,54,55,56,57,58, # l-w
551 59,60,61); # x-z
552
553@days_per_month = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334);
554$gig = 1024 * 1024 * 1024;
a83c7e95 555$VERSION = '1.50';
059ec3d9
PH		 556
557# How much space do we allow for the Hosts/Domains/Emails/Edomains column headers?
558$COLUMN_WIDTHS = 8;
559
608bc29d
SC		 560$MINUTE = 60;
561$HOUR = 60 * $MINUTE;
562$DAY = 24 * $HOUR;
563$WEEK = 7 * $DAY;
564
059ec3d9
PH		 565# Declare global variables.
566use vars qw($total_received_data $total_received_data_gigs $total_received_count);
a83c7e95 567use vars qw($total_delivered_data $total_delivered_data_gigs $total_delivered_messages $total_delivered_addresses);
d5692f86
SC		 568use vars qw(%timestamp2time); #Hash of timestamp => time.
569use vars qw($last_timestamp $last_time); #The last time convertion done.
570use vars qw($last_date $date_seconds); #The last date convertion done.
571use vars qw($last_offset $offset_seconds); #The last time offset convertion done.
059ec3d9 572use vars qw($localtime_offset);
d5692f86
SC		 573use vars qw($i); #General loop counter.
574use vars qw($debug); #Debug mode?
575use vars qw($ntopchart); #How many entries should make it into the chart?
576use vars qw($gddirectory); #Where to put files from GD::Graph
a83c7e95
SC		 577
578# SpamAssassin variables
579use vars qw($spam_score $spam_score_gigs);
580use vars qw($ham_score $ham_score_gigs);
581use vars qw(%ham_count_by_ip %spam_count_by_ip);
582use vars qw(%rejected_count_by_ip %rejected_count_by_reason);
583
584#For use in Speadsheed::WriteExcel
585use vars qw($workbook $ws_global $ws_relayed $ws_errors);
586use vars qw($row $col $row_hist $col_hist);
d5692f86 587use vars qw($run_hist);
a83c7e95 588use vars qw($f_default $f_header1 $f_header2 $f_header2_m $f_headertab $f_percent); #Format Header
d5692f86
SC		 589
590# Output FileHandles
591use vars qw($txt_fh $htm_fh $xls_fh);
059ec3d9
PH		 592
593$ntopchart = 5;
594
595# The following are parameters whose values are
596# set by command line switches:
597use vars qw($show_errors $show_relay $show_transport $transport_pattern);
598use vars qw($topcount $local_league_table $include_remote_users);
a83c7e95 599use vars qw($hist_opt $hist_interval $hist_number $volume_rounding $emptyOK);
d5692f86 600use vars qw($relay_pattern @queue_times @user_patterns @user_descriptions);
608bc29d 601use vars qw(@rcpt_times @delivery_times);
d5692f86
SC		 602use vars qw($include_original_destination);
603use vars qw($txt_fh $htm_fh $xls_fh);
059ec3d9
PH		 604
605use vars qw(%do_sender); #Do sender by Host, Domain, Email, and/or Edomain tables.
606use vars qw($charts $chartrel $chartdir $charts_option_specified);
d5692f86 607use vars qw($merge_reports); #Merge old reports ?
059ec3d9
PH		 608
609# The following are modified in the parse() routine, and
610# referred to in the print_*() routines.
608bc29d 611use vars qw($delayed_count $relayed_unshown $begin $end);
a83c7e95 612use vars qw(%messages @message);
059ec3d9 613use vars qw(%received_count %received_data %received_data_gigs);
a83c7e95 614use vars qw(%delivered_messages %delivered_data %delivered_data_gigs %delivered_addresses);
059ec3d9 615use vars qw(%received_count_user %received_data_user %received_data_gigs_user);
a83c7e95 616use vars qw(%delivered_messages_user %delivered_addresses_user %delivered_data_user %delivered_data_gigs_user);
059ec3d9 617use vars qw(%transported_count %transported_data %transported_data_gigs);
608bc29d
SC		 618use vars qw(%relayed %errors_count $message_errors);
619use vars qw(@qt_all_bin @qt_remote_bin);
620use vars qw($qt_all_overflow $qt_remote_overflow);
621use vars qw(@dt_all_bin @dt_remote_bin %rcpt_times_bin);
622use vars qw($dt_all_overflow $dt_remote_overflow %rcpt_times_overflow);
623use vars qw(@received_interval_count @delivered_interval_count);
624use vars qw(@user_pattern_totals @user_pattern_interval_count);
059ec3d9
PH		 625
626use vars qw(%report_totals);
627
608bc29d
SC		 628# Enumerations
629use vars qw($SIZE $FROM_HOST $FROM_ADDRESS $ARRIVAL_TIME $REMOTE_DELIVERED $PROTOCOL);
630use vars qw($DELAYED $HAD_ERROR);
631$SIZE = 0;
632$FROM_HOST = 1;
633$FROM_ADDRESS = 2;
634$ARRIVAL_TIME = 3;
635$REMOTE_DELIVERED = 4;
636$DELAYED = 5;
637$HAD_ERROR = 6;
638$PROTOCOL = 7;
059ec3d9
PH		 639
640
641
642##################################################
643# Subroutines #
644##################################################
645
d5692f86
SC		 646#######################################################################
647# get_filehandle($file,\%output_files);
648# Return a filehandle writing to $file.
649#
650# If %output_files is defined, check that $output_files{$file}
651# doesn't exist and die if it does, or set it if it doesn't.
652#######################################################################
653sub get_filehandle {
654 my($file,$output_files_href) = @_;
655
656 $file = '-' if ($file eq '');
657
658 if (defined $output_files_href) {
659 die "You can only output to '$file' once! Use -h for help.\n" if exists $output_files_href->{$file};
660 $output_files_href->{$file} = 1;
661 }
662
663 if ($file eq '-') {
664 return \*STDOUT;
665 }
666
667 if (-e $file) {
668 unlink $file or die "Failed to rm $file: $!";
669 }
670
671 my $fh = new IO::File $file, O_WRONLY|O_CREAT|O_EXCL;
672 die "new IO::File $file failed: $!" unless (defined $fh);
673 return $fh;
674}
675
059ec3d9 676
1b4fe9dd
PH		 677#######################################################################
678# volume_rounded();
679#
680# $rounded_volume = volume_rounded($bytes,$gigabytes);
681#
682# Given a data size in bytes, round it to KB, MB, or GB
683# as appropriate.
684#
685# Eg 12000 => 12KB, 15000000 => 14GB, etc.
686#
687# Note: I've experimented with Math::BigInt and it results in a 33%
688# performance degredation as opposed to storing numbers split into
689# bytes and gigabytes.
690#######################################################################
059ec3d9
PH		 691sub volume_rounded {
692 my($x,$g) = @_;
693 $x = 0 unless $x;
694 $g = 0 unless $g;
695 my($rounded);
696
697 while ($x > $gig) {
698 $g++;
699 $x -= $gig;
700 }
701
702 if ($volume_rounding) {
703 # Values < 1 GB
704 if ($g <= 0) {
705 if ($x < 10000) {
706 $rounded = sprintf("%6d", $x);
707 }
708 elsif ($x < 10000000) {
709 $rounded = sprintf("%4dKB", ($x + 512)/1024);
710 }
711 else {
712 $rounded = sprintf("%4dMB", ($x + 512*1024)/(1024*1024));
713 }
714 }
715 # Values between 1GB and 10GB are printed in MB
716 elsif ($g < 10) {
717 $rounded = sprintf("%4dMB", ($g * 1024) + ($x + 512*1024)/(1024*1024));
718 }
719 else {
720 # Handle values over 10GB
721 $rounded = sprintf("%4dGB", $g + ($x + $gig/2)/$gig);
722 }
723 }
724 else {
725 # We don't want any rounding to be done.
d5692f86
SC		 726 # and we don't need broken formated output which on one hand avoids numbers from
727 # being interpreted as string by Spreadsheed Calculators, on the other hand
728 # breaks if more than 4 digits! -> flexible length instead of fixed length
729 # Format the return value at the output routine! -fh
730 #$rounded = sprintf("%d", ($g * $gig) + $x);
731 no integer;
732 $rounded = sprintf("%.0f", ($g * $gig) + $x);
059ec3d9
PH		 733 }
734
735 return $rounded;
736}
737
738
1b4fe9dd
PH		 739#######################################################################
740# un_round();
8e669ac1 741#
1b4fe9dd 742# un_round($rounded_volume,\$bytes,\$gigabytes);
8e669ac1 743#
1b4fe9dd
PH		 744# Given a volume in KB, MB or GB, as generated by volume_rounded(),
745# do the reverse transformation and convert it back into Bytes and Gigabytes.
746# These are added to the $bytes and $gigabytes parameters.
8e669ac1 747#
1b4fe9dd
PH		 748# Given a data size in bytes, round it to KB, MB, or GB
749# as appropriate.
8e669ac1 750#
1b4fe9dd
PH		 751# EG: 500 => (500,0), 14GB => (0,14), etc.
752#######################################################################
059ec3d9
PH		 753sub un_round {
754 my($rounded,$bytes_sref,$gigabytes_sref) = @_;
755
756 if ($rounded =~ /(\d+)GB/) {
757 $$gigabytes_sref += $1;
758 }
759 elsif ($rounded =~ /(\d+)MB/) {
760 $$gigabytes_sref += $1 / 1024;
761 $$bytes_sref += (($1 % 1024 ) * 1024 * 1024);
762 }
763 elsif ($rounded =~ /(\d+)KB/) {
764 $$gigabytes_sref += $1 / (1024 * 1024);
765 $$bytes_sref += ($1 % (1024 * 1024) * 1024);
766 }
767 elsif ($rounded =~ /(\d+)/) {
d5692f86
SC		 768 # We need to turn off integer in case we are merging an -nvr report.
769 no integer;
770 $$gigabytes_sref += int($1 / $gig);
059ec3d9
PH		 771 $$bytes_sref += $1 % $gig;
772 }
773
774 #Now reduce the bytes down to less than 1GB.
775 add_volume($bytes_sref,$gigabytes_sref,0) if ($$bytes_sref > $gig);
776}
777
778
1b4fe9dd
PH		 779#######################################################################
780# add_volume();
8e669ac1 781#
1b4fe9dd 782# add_volume(\$bytes,\$gigs,$size);
8e669ac1 783#
1b4fe9dd
PH		 784# Add $size to $bytes/$gigs where this is a number split into
785# bytes ($bytes) and gigabytes ($gigs). This is significantly
786# faster than using Math::BigInt.
787#######################################################################
059ec3d9 788sub add_volume {
1b4fe9dd
PH		 789 my($bytes_ref,$gigs_ref,$size) = @_;
790 $$bytes_ref = 0 if ! defined $$bytes_ref;
791 $$gigs_ref = 0 if ! defined $$gigs_ref;
792 $$bytes_ref += $size;
793 while ($$bytes_ref > $gig) {
794 $$gigs_ref++;
795 $$bytes_ref -= $gig;
059ec3d9
PH		 796 }
797}
798
799
1b4fe9dd
PH		 800#######################################################################
801# format_time();
8e669ac1 802#
1b4fe9dd 803# $formatted_time = format_time($seconds);
8e669ac1 804#
1b4fe9dd
PH		 805# Given a time in seconds, break it down into
806# weeks, days, hours, minutes, and seconds.
8e669ac1 807#
1b4fe9dd
PH		 808# Eg 12005 => 3h20m5s
809#######################################################################
059ec3d9
PH		 810sub format_time {
811my($t) = pop @_;
812my($s) = $t % 60;
813$t /= 60;
814my($m) = $t % 60;
815$t /= 60;
816my($h) = $t % 24;
817$t /= 24;
818my($d) = $t % 7;
819my($w) = $t/7;
820my($p) = "";
821$p .= "$w"."w" if $w > 0;
822$p .= "$d"."d" if $d > 0;
823$p .= "$h"."h" if $h > 0;
824$p .= "$m"."m" if $m > 0;
825$p .= "$s"."s" if $s > 0 || $p eq "";
826$p;
827}
828
829
1b4fe9dd
PH		 830#######################################################################
831# unformat_time();
8e669ac1 832#
1b4fe9dd 833# $seconds = unformat_time($formatted_time);
8e669ac1 834#
1b4fe9dd 835# Given a time in weeks, days, hours, minutes, or seconds, convert it to seconds.
8e669ac1 836#
1b4fe9dd
PH		 837# Eg 3h20m5s => 12005
838#######################################################################
059ec3d9
PH		 839sub unformat_time {
840 my($formated_time) = pop @_;
841 my $time = 0;
842
843 while ($formated_time =~ s/^(\d+)([wdhms]?)//) {
844 $time += $1 if ($2 eq '' || $2 eq 's');
845 $time += $1 * 60 if ($2 eq 'm');
846 $time += $1 * 60 * 60 if ($2 eq 'h');
847 $time += $1 * 60 * 60 * 24 if ($2 eq 'd');
848 $time += $1 * 60 * 60 * 24 * 7 if ($2 eq 'w');
849 }
850 $time;
851}
852
853
1b4fe9dd
PH		 854#######################################################################
855# seconds();
8e669ac1 856#
1b4fe9dd 857# $time = seconds($timestamp);
8e669ac1 858#
1b4fe9dd
PH		 859# Given a time-of-day timestamp, convert it into a time() value using
860# POSIX::mktime. We expect the timestamp to be of the form
861# "$year-$mon-$day $hour:$min:$sec", with month going from 1 to 12,
862# and the year to be absolute (we do the necessary conversions). The
863# timestamp may be followed with an offset from UTC like "+$hh$mm"; if the
864# offset is not present, and we have not been told that the log is in UTC
865# (with the -utc option), then we adjust the time by the current local
866# time offset so that it can be compared with the time recorded in message
867# IDs, which is UTC.
8e669ac1 868#
1b4fe9dd
PH		 869# To improve performance, we only use mktime on the date ($year-$mon-$day),
870# and only calculate it if the date is different to the previous time we
871# came here. We then add on seconds for the '$hour:$min:$sec'.
8e669ac1 872#
1b4fe9dd
PH		 873# We also store the results of the last conversion done, and only
874# recalculate if the date is different.
8e669ac1 875#
1b4fe9dd
PH		 876# We used to have the '-cache' flag which would store the results of the
877# mktime() call. However, the current way of just using mktime() on the
878# date obsoletes this.
879#######################################################################
059ec3d9
PH		 880sub seconds {
881 my($timestamp) = @_;
882
883 # Is the timestamp the same as the last one?
884 return $last_time if ($last_timestamp eq $timestamp);
885
886 return 0 unless ($timestamp =~ /^((\d{4})\-(\d\d)-(\d\d))\s(\d\d):(\d\d):(\d\d)( ([+-])(\d\d)(\d\d))?/o);
887
888 unless ($last_date eq $1) {
889 $last_date = $1;
890 my(@timestamp) = (0,0,0,$4,$3,$2);
891 $timestamp[5] -= 1900;
892 $timestamp[4]--;
893 $date_seconds = mktime(@timestamp);
894 }
895 my $time = $date_seconds + ($5 * 3600) + ($6 * 60) + $7;
896
897 # SC. Use cacheing. Also note we want seconds not minutes.
898 #my($this_offset) = ($10 * 60 + $11) * ($9 . "1") if defined $8;
899 if (defined $8 && ($8 ne $last_offset)) {
900 $last_offset = $8;
901 $offset_seconds = ($10 * 60 + $11) * 60;
902 $offset_seconds = -$offset_seconds if ($9 eq '-');
903 }
904
905
906 if (defined $7) {
907 #$time -= $this_offset;
908 $time -= $offset_seconds;
909 } elsif (defined $localtime_offset) {
910 $time -= $localtime_offset;
911 }
912
913 # Store the last timestamp received.
914 $last_timestamp = $timestamp;
915 $last_time = $time;
916
917 $time;
918}
919
920
1b4fe9dd
PH		 921#######################################################################
922# id_seconds();
8e669ac1 923#
1b4fe9dd 924# $time = id_seconds($message_id);
8e669ac1 925#
1b4fe9dd
PH		 926# Given a message ID, convert it into a time() value.
927#######################################################################
059ec3d9
PH		 928sub id_seconds {
929my($sub_id) = substr((pop @_), 0, 6);
930my($s) = 0;
931my(@c) = split(//, $sub_id);
932while($#c >= 0) { $s = $s * 62 + $tab62[ord(shift @c) - ord('0')] }
933$s;
934}
935
608bc29d
SC		 936#######################################################################
937# wdhms_seconds();
938#
939# $seconds = wdhms_seconds($string);
940#
941# Convert a string in a week/day/hour/minute/second format (eg 4h10s)
942# into seconds.
943#######################################################################
944sub wdhms_seconds {
945 if ($_[0] =~ /^(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?/) {
946 return((($1||0) * $WEEK) + (($2||0) * $DAY) + (($3||0) * $HOUR) + (($4||0) * $MINUTE) + ($5||0));
947 }
948 return undef;
949}
950
951#######################################################################
952# queue_time();
953#
954# $queued = queue_time($completed_tod, $arrival_time, $id);
955#
956# Given the completed time of day and either the arrival time
957# (preferred), or the message ID, calculate how long the message has
958# been on the queue.
959#
960#######################################################################
961sub queue_time {
962 my($completed_tod, $arrival_time, $id) = @_;
963
964 # Note: id_seconds() benchmarks as 42% slower than seconds()
965 # and computing the time accounts for a significant portion of
966 # the run time.
967 if (defined $arrival_time) {
968 return(seconds($completed_tod) - seconds($arrival_time));
969 }
970 else {
971 return(seconds($completed_tod) - id_seconds($id));
972 }
973}
059ec3d9
PH		 974
975
1b4fe9dd
PH		 976#######################################################################
977# calculate_localtime_offset();
8e669ac1 978#
1b4fe9dd 979# $localtime_offset = calculate_localtime_offset();
8e669ac1 980#
1b4fe9dd 981# Calculate the the localtime offset from gmtime in seconds.
8e669ac1 982#
1b4fe9dd 983# $localtime = time() + $localtime_offset.
8e669ac1 984#
1b4fe9dd
PH		 985# These are the same semantics as ISO 8601 and RFC 2822 timezone offsets.
986# (West is negative, East is positive.)
987#######################################################################
059ec3d9
PH		 988
989# $localtime = gmtime() + $localtime_offset. OLD COMMENT
990# This subroutine commented out as it's not currently in use.
991
992#sub calculate_localtime_offset {
993# # Pick an arbitrary date, convert it to localtime & gmtime, and return the difference.
994# my (@sample_date) = (0,0,0,5,5,100);
995# my $localtime = timelocal(@sample_date);
996# my $gmtime = timegm(@sample_date);
997# my $offset = $localtime - $gmtime;
998# return $offset;
999#}
1000
1001sub calculate_localtime_offset {
1002 # Assume that the offset at the moment is valid across the whole
1003 # period covered by the logs that we're analysing. This may not
1004 # be true around the time the clocks change in spring or autumn.
1005 my $utc = time;
1006 # mktime works on local time and gmtime works in UTC
1007 my $local = mktime(gmtime($utc));
1008 return $local - $utc;
1009}
1010
1011
608bc29d 1012
1b4fe9dd 1013#######################################################################
608bc29d
SC		 1014# print_duration_table();
1015#
1016# print_duration_table($title, $message_type, \@times, \@values, $overflow);
1017#
1018# Print a table showing how long a particular step took for
1019# the messages. The parameters are:
1020# $title Eg "Time spent on the queue"
1021# $message_type Eg "Remote"
1022# \@times The maximum time a message took for it to increment
1023# the corresponding @values counter.
1024# \@values An array of message counters.
1025# $overflow The number of messages which exceeded the maximum
1026# time.
1b4fe9dd 1027#######################################################################
608bc29d 1028sub print_duration_table {
059ec3d9 1029no integer;
608bc29d 1030my($title, $message_type, $times_aref, $values_aref, $overflow) = @_;
059ec3d9
PH		 1031my(@chartdatanames);
1032my(@chartdatavals);
1033
1034my $printed_one = 0;
1035my $cumulative_percent = 0;
059ec3d9 1036
608bc29d
SC		 1037my $queue_total = $overflow;
1038map {$queue_total += $_} @$values_aref;
059ec3d9 1039
608bc29d 1040my $temp = "$title: $message_type";
059ec3d9 1041
d5692f86
SC		 1042
1043my $txt_format = "%5s %4s %6d %5.1f%% %5.1f%%\n";
1044my $htm_format = "<tr><td align=\"right\">%s %s</td><td align=\"right\">%d</td><td align=\"right\">%5.1f%%</td><td align=\"right\">%5.1f%%</td>\n";
1045
1046# write header
1047printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
1048if ($htm_fh) {
608bc29d 1049 print $htm_fh "<hr><a name=\"$title $message_type\"></a><h2>$temp</h2>\n";
a83c7e95 1050 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
d5692f86 1051 print $htm_fh "<tr><th>Time</th><th>Messages</th><th>Percentage</th><th>Cumulative Percentage</th>\n";
059ec3d9 1052}
a83c7e95 1053if ($xls_fh) {
608bc29d 1054 $ws_global->write($row++, $col, "$title: ".$message_type, $f_header2);
d5692f86
SC		 1055 my @content=("Time", "Messages", "Percentage", "Cumulative Percentage");
1056 &set_worksheet_line($ws_global, $row++, 1, \@content, $f_headertab);
059ec3d9
PH		 1057}
1058
d5692f86 1059
608bc29d
SC		 1060for ($i = 0; $i <= $#$times_aref; ++$i) {
1061 if ($$values_aref[$i] > 0)
059ec3d9 1062 {
608bc29d 1063 my $percent = ($values_aref->[$i] * 100)/$queue_total;
059ec3d9 1064 $cumulative_percent += $percent;
d5692f86
SC		 1065
1066 my @content=($printed_one? " " : "Under",
608bc29d
SC		 1067 format_time($times_aref->[$i]),
1068 $values_aref->[$i], $percent, $cumulative_percent);
d5692f86
SC		 1069
1070 if ($htm_fh) {
1071 printf $htm_fh ($htm_format, @content);
608bc29d 1072 if (!defined($values_aref->[$i])) {
d5692f86
SC		 1073 print $htm_fh "Not defined";
1074 }
1075 }
1076 if ($txt_fh) {
1077 printf $txt_fh ($txt_format, @content);
608bc29d 1078 if (!defined($times_aref->[$i])) {
d5692f86
SC		 1079 print $txt_fh "Not defined";
1080 }
1081 }
1082 if ($xls_fh)
1083 {
1084 no integer;
1085 &set_worksheet_line($ws_global, $row, 0, [@content[0,1,2]], $f_default);
1086 &set_worksheet_line($ws_global, $row++, 3, [$content[3]/100,$content[4]/100], $f_percent);
1087
608bc29d 1088 if (!defined($times_aref->[$i])) {
d5692f86
SC		 1089 $col=0;
1090 $ws_global->write($row++, $col, "Not defined" );
1091 }
059ec3d9 1092 }
d5692f86 1093
059ec3d9 1094 push(@chartdatanames,
608bc29d
SC		 1095 ($printed_one? "" : "Under") . format_time($times_aref->[$i]));
1096 push(@chartdatavals, $$values_aref[$i]);
059ec3d9
PH		 1097 $printed_one = 1;
1098 }
1099}
1100
608bc29d
SC		 1101if ($overflow && $overflow > 0) {
1102 my $percent = ($overflow * 100)/$queue_total;
059ec3d9 1103 $cumulative_percent += $percent;
d5692f86 1104
608bc29d
SC		 1105 my @content = ("Over ", format_time($times_aref->[-1]),
1106 $overflow, $percent, $cumulative_percent);
d5692f86
SC		 1107
1108 printf $txt_fh ($txt_format, @content) if $txt_fh;
1109 printf $htm_fh ($htm_format, @content) if $htm_fh;
1110 if ($xls_fh)
1111 {
1112 &set_worksheet_line($ws_global, $row, 0, [@content[0,1,2]], $f_default);
1113 &set_worksheet_line($ws_global, $row++, 3, [$content[3]/100,$content[4]/100], $f_percent);
1114 }
1115
059ec3d9 1116}
d5692f86 1117
608bc29d
SC		 1118push(@chartdatanames, "Over " . format_time($times_aref->[-1]));
1119push(@chartdatavals, $overflow);
059ec3d9
PH		 1120
1121#printf("Unknown %6d\n", $queue_unknown) if $queue_unknown > 0;
d5692f86 1122if ($htm_fh) {
a83c7e95 1123 print $htm_fh "</table></td><td>";
059ec3d9 1124
608bc29d 1125 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0)) {
059ec3d9
PH		 1126 my @data = (
1127 \@chartdatanames,
1128 \@chartdatavals
1129 );
1130 my $graph = GD::Graph::pie->new(200, 200);
608bc29d
SC		 1131 my $pngname = "$title-$message_type.png";
1132 $pngname =~ s/[^\w\-\.]/_/;
1133
1134 my $graph_title = "$title ($message_type)";
1135 $graph->set(title => $graph_title) if (length($graph_title) < 21);
1136
059ec3d9
PH		 1137 my $gd = $graph->plot(\@data) or warn($graph->error);
1138 if ($gd) {
d5692f86 1139 open(IMG, ">$chartdir/$pngname") or die "Could not write $chartdir/$pngname: $!\n";
059ec3d9
PH		 1140 binmode IMG;
1141 print IMG $gd->png;
1142 close IMG;
d5692f86 1143 print $htm_fh "<img src=\"$chartrel/$pngname\">";
059ec3d9
PH		 1144 }
1145 }
d5692f86
SC		 1146 print $htm_fh "</td></tr></table>\n";
1147}
1148
1149if ($xls_fh)
1150{
1151 $row++;
059ec3d9 1152}
d5692f86
SC		 1153print $txt_fh "\n" if $txt_fh;
1154print $htm_fh "\n" if $htm_fh;
1155
059ec3d9
PH		 1156}
1157
1158
1b4fe9dd
PH		 1159#######################################################################
1160# print_histogram();
8e669ac1 1161#
608bc29d 1162# print_histogram('Deliveries|Messages received|$pattern', $unit, @interval_count);
8e669ac1 1163#
1b4fe9dd
PH		 1164# Print a histogram of the messages delivered/received per time slot
1165# (hour by default).
1166#######################################################################
059ec3d9 1167sub print_histogram {
608bc29d 1168my($text, $unit, @interval_count) = @_;
059ec3d9
PH		 1169my(@chartdatanames);
1170my(@chartdatavals);
1171my($maxd) = 0;
d5692f86 1172
a83c7e95
SC		 1173# save first row of print_histogram for xls output
1174if (!$run_hist) {
d5692f86
SC		 1175 $row_hist = $row;
1176}
a83c7e95 1177else {
d5692f86
SC		 1178 $row = $row_hist;
1179}
1180
059ec3d9
PH		 1181for ($i = 0; $i < $hist_number; $i++)
1182 { $maxd = $interval_count[$i] if $interval_count[$i] > $maxd; }
1183
1184my $scale = int(($maxd + 25)/50);
1185$scale = 1 if $scale == 0;
1186
608bc29d
SC		 1187if ($scale != 1) {
1188 if ($unit !~ s/y$/ies/) {
1189 $unit .= 's';
1190 }
d5692f86
SC		 1191}
1192
1193# make and output title
1194my $title = sprintf("$text per %s",
1195 ($hist_interval == 60)? "hour" :
1196 ($hist_interval == 1)? "minute" : "$hist_interval minutes");
059ec3d9 1197
608bc29d 1198my $txt_htm_title = $title . " (each dot is $scale $unit)";
059ec3d9 1199
d5692f86
SC		 1200printf $txt_fh ("%s\n%s\n\n", $txt_htm_title, "-" x length($txt_htm_title)) if $txt_fh;
1201
1202if ($htm_fh) {
1203 print $htm_fh "<hr><a name=\"$text\"></a><h2>$txt_htm_title</h2>\n";
1204 print $htm_fh "<table border=0 width=\"100%\">\n";
1205 print $htm_fh "<tr><td><pre>\n";
059ec3d9 1206}
d5692f86 1207
a83c7e95 1208if ($xls_fh) {
d5692f86 1209 $title =~ s/Messages/Msg/ ;
a83c7e95 1210 $row += 2;
d5692f86 1211 $ws_global->write($row++, $col_hist+1, $title, $f_headertab);
059ec3d9
PH		 1212}
1213
d5692f86 1214
059ec3d9
PH		 1215my $hour = 0;
1216my $minutes = 0;
a83c7e95 1217for ($i = 0; $i < $hist_number; $i++) {
059ec3d9
PH		 1218 my $c = $interval_count[$i];
1219
1220 # If the interval is an hour (the maximum) print the starting and
1221 # ending hours as a label. Otherwise print the starting hour and
1222 # minutes, which take up the same space.
1223
1224 my $temp;
a83c7e95 1225 if ($hist_opt == 1) {
059ec3d9 1226 $temp = sprintf("%02d-%02d", $hour, $hour + 1);
d5692f86
SC		 1227
1228 print $txt_fh $temp if $txt_fh;
1229 print $htm_fh $temp if $htm_fh;
1230
a83c7e95
SC		 1231 if ($xls_fh) {
1232 if ($run_hist==0) {
1233 # only on first run
1234 $ws_global->write($row, 0, [$temp], $f_default);
d5692f86
SC		 1235 }
1236 }
1237
059ec3d9
PH		 1238 push(@chartdatanames, $temp);
1239 $hour++;
d5692f86 1240 }
a83c7e95 1241 else {
059ec3d9
PH		 1242 if ($minutes == 0)
1243 { $temp = sprintf("%02d:%02d", $hour, $minutes) }
1244 else
1245 { $temp = sprintf(" :%02d", $minutes) }
d5692f86
SC		 1246
1247 print $txt_fh $temp if $txt_fh;
1248 print $htm_fh $temp if $htm_fh;
a83c7e95
SC		 1249 if (($xls_fh) and ($run_hist==0)) {
1250 # only on first run
d5692f86 1251 $temp = sprintf("%02d:%02d", $hour, $minutes);
a83c7e95 1252 $ws_global->write($row, 0, [$temp], $f_default);
d5692f86
SC		 1253 }
1254
059ec3d9
PH		 1255 push(@chartdatanames, $temp);
1256 $minutes += $hist_interval;
a83c7e95 1257 if ($minutes >= 60) {
059ec3d9
PH		 1258 $minutes = 0;
1259 $hour++;
a83c7e95 1260 }
059ec3d9 1261 }
d5692f86
SC		 1262 push(@chartdatavals, $c);
1263
1264 printf $txt_fh (" %6d %s\n", $c, "." x ($c/$scale)) if $txt_fh;
1265 printf $htm_fh (" %6d %s\n", $c, "." x ($c/$scale)) if $htm_fh;
a83c7e95 1266 $ws_global->write($row++, $col_hist+1, [$c], $f_default) if $xls_fh;
d5692f86
SC		 1267
1268} #end for
1269
1270printf $txt_fh "\n" if $txt_fh;
1271printf $htm_fh "\n" if $htm_fh;
1272
1273if ($htm_fh)
1274{
1275 print $htm_fh "</pre>\n";
1276 print $htm_fh "</td><td>\n";
608bc29d 1277 if ($HAVE_GD_Graph_linespoints && $charts && ($#chartdatavals > 0)) {
059ec3d9
PH		 1278 # calculate the graph
1279 my @data = (
1280 \@chartdatanames,
1281 \@chartdatavals
1282 );
1283 my $graph = GD::Graph::linespoints->new(300, 300);
1284 $graph->set(
1285 x_label => 'Time',
1286 y_label => 'Amount',
1287 title => $text,
1288 x_labels_vertical => 1
1289 );
608bc29d
SC		 1290 my $pngname = "histogram_$text.png";
1291 $pngname =~ s/[^\w\._]/_/g;
1292
059ec3d9
PH		 1293 my $gd = $graph->plot(\@data) or warn($graph->error);
1294 if ($gd) {
d5692f86 1295 open(IMG, ">$chartdir/$pngname") or die "Could not write $chartdir/$pngname: $!\n";
059ec3d9
PH		 1296 binmode IMG;
1297 print IMG $gd->png;
1298 close IMG;
d5692f86 1299 print $htm_fh "<img src=\"$chartrel/$pngname\">";
059ec3d9
PH		 1300 }
1301 }
d5692f86 1302 print $htm_fh "</td></tr></table>\n";
059ec3d9 1303}
d5692f86
SC		 1304
1305$col_hist++; # where to continue next times
1306
1307$row+=2; # leave some space after history block
1308$run_hist=1; # we have done this once or more
059ec3d9
PH		 1309}
1310
1311
1312
1b4fe9dd
PH		 1313#######################################################################
1314# print_league_table();
8e669ac1 1315#
a83c7e95 1316# print_league_table($league_table_type,\%message_count,\%address_count,\%message_data,\%message_data_gigs, $spreadsheet, $row_sref);
8e669ac1 1317#
a83c7e95
SC		 1318# Given hashes of message count, address count, and message data,
1319# which are keyed by the table type (eg by the sending host), print a
1320# league table showing the top $topcount (defaults to 50).
1b4fe9dd 1321#######################################################################
059ec3d9 1322sub print_league_table {
a83c7e95
SC		 1323 my($text,$m_count,$a_count,$m_data,$m_data_gigs,$spreadsheet, $row_sref) = @_;
1324 my($name) = ($topcount == 1)? "$text" : "$topcount ${text}s";
1325 my($title) = "Top $name by message count";
1326 my(@chartdatanames) = ();
1327 my(@chartdatavals) = ();
1328 my $chartotherval = 0;
1329 $text = ucfirst($text);
059ec3d9
PH		 1330
1331 # Align non-local addresses to the right (so all the .com's line up).
1332 # Local addresses are aligned on the left as they are userids.
1333 my $align = ($text !~ /local/i) ? 'right' : 'left';
059ec3d9 1334
d5692f86 1335
a83c7e95
SC		 1336 ################################################
1337 # Generate the printf formats and table headers.
1338 ################################################
1339 my(@headers) = ('Messages');
1340 push(@headers,'Addresses') if defined $a_count;
1341 push(@headers,'Bytes','Average') if defined $m_data;
d5692f86 1342
a83c7e95
SC		 1343 my $txt_format = "%10s " x @headers . " %s\n";
1344 my $txt_col_headers = sprintf $txt_format, @headers, $text;
1345 my $htm_format = "<tr>" . '<td align="right">%s</td>'x@headers . "<td align=\"$align\" nowrap>%s</td></tr>\n";
1346 my $htm_col_headers = sprintf $htm_format, @headers, $text;
1347 $htm_col_headers =~ s/(<\/?)td/$1th/g; #Convert <td>'s to <th>'s for the header.
d5692f86 1348
a83c7e95
SC		 1349
1350 ################################################
1351 # Write the table headers
1352 ################################################
1353 printf $txt_fh ("%s\n%s\n%s", $title, "-" x length($title),$txt_col_headers) if $txt_fh;
d5692f86
SC		 1354
1355 if ($htm_fh) {
a83c7e95
SC		 1356 print $htm_fh <<EoText;
1357<hr><a name="$text count"></a><h2>$title</h2>
1358<table border=0 width="100%">
1359<tr><td>
1360<table border=1>
1361EoText
1362 print $htm_col_headers;
059ec3d9 1363 }
d5692f86 1364
a83c7e95
SC		 1365 if ($xls_fh) {
1366 $spreadsheet->write(${$row_sref}++, 0, $title, $f_header2);
1367 $spreadsheet->write(${$row_sref}++, 0, [@headers, $text], $f_headertab);
d5692f86 1368 }
a83c7e95
SC		 1369
1370
1371 # write content
1372 foreach my $key (top_n_sort($topcount,$m_count,$m_data_gigs,$m_data)) {
1373
1374 # When displaying the average figures, we calculate the average of
1375 # the rounded data, as the user would calculate it. This reduces
1376 # the accuracy slightly, but we have to do it this way otherwise
1377 # when using -merge to convert results from text to HTML and
1378 # vice-versa discrepencies would occur.
1379 my $messages = $$m_count{$key};
1380 my @content = ($messages);
1381 push(@content, $$a_count{$key}) if defined $a_count;
1382 if (defined $m_data) {
1383 my $rounded_volume = volume_rounded($$m_data{$key},$$m_data_gigs{$key});
1384 my($data,$gigs) = (0,0);
1385 un_round($rounded_volume,\$data,\$gigs);
1386 my $rounded_average = volume_rounded($data/$messages,$gigs/$messages);
1387 push(@content, $rounded_volume, $rounded_average);
1388 }
1389
1390 # write content
1391 printf $txt_fh ($txt_format, @content, $key) if $txt_fh;
1392
1393 if ($htm_fh) {
1394 my $htmlkey = $key;
1395 $htmlkey =~ s/>/\&gt\;/g;
1396 $htmlkey =~ s/</\&lt\;/g;
1397 printf $htm_fh ($htm_format, @content, $htmlkey);
1398 }
1399 $spreadsheet->write(${$row_sref}++, 0, [@content, $key], $f_default) if $xls_fh;
1400
1401 if (scalar @chartdatanames < $ntopchart) {
1402 push(@chartdatanames, $key);
1403 push(@chartdatavals, $$m_count{$key});
1404 }
1405 else {
1406 $chartotherval += $$m_count{$key};
1407 }
059ec3d9 1408 }
d5692f86 1409
a83c7e95
SC		 1410 push(@chartdatanames, "Other");
1411 push(@chartdatavals, $chartotherval);
059ec3d9 1412
a83c7e95
SC		 1413 print $txt_fh "\n" if $txt_fh;
1414 if ($htm_fh) {
1415 print $htm_fh "</table>\n";
1416 print $htm_fh "</td><td>\n";
1417 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0))
1418 {
1419 # calculate the graph
1420 my @data = (
1421 \@chartdatanames,
1422 \@chartdatavals
1423 );
1424 my $graph = GD::Graph::pie->new(300, 300);
1425 $graph->set(
1426 x_label => 'Name',
1427 y_label => 'Amount',
1428 title => 'By count',
1429 );
1430 my $gd = $graph->plot(\@data) or warn($graph->error);
1431 if ($gd) {
1432 my $temp = $text;
1433 $temp =~ s/ /_/g;
1434 open(IMG, ">$chartdir/${temp}_count.png") or die "Could not write $chartdir/${temp}_count.png: $!\n";
1435 binmode IMG;
1436 print IMG $gd->png;
1437 close IMG;
1438 print $htm_fh "<img src=\"$chartrel/${temp}_count.png\">";
1439 }
059ec3d9 1440 }
a83c7e95
SC		 1441 print $htm_fh "</td><td>\n";
1442 print $htm_fh "</td></tr></table>\n\n";
059ec3d9 1443 }
a83c7e95 1444 ++${$row_sref} if $xls_fh;
d5692f86
SC		 1445
1446
a83c7e95
SC		 1447 if (defined $m_data) {
1448 # write header
059ec3d9 1449
a83c7e95 1450 $title = "Top $name by volume";
d5692f86 1451
a83c7e95 1452 printf $txt_fh ("%s\n%s\n%s", $title, "-" x length($title),$txt_col_headers) if $txt_fh;
059ec3d9 1453
a83c7e95
SC		 1454 if ($htm_fh) {
1455 print $htm_fh <<EoText;
1456<hr><a name="$text volume"></a><h2>$title</h2>
1457<table border=0 width="100%">
1458<tr><td>
1459<table border=1>
1460EoText
1461 print $htm_col_headers;
1462 }
1463 if ($xls_fh) {
1464 $spreadsheet->write(${$row_sref}++, 0, $title, $f_header2);
1465 $spreadsheet->write(${$row_sref}++, 0, [@headers, $text], $f_headertab);
1466 }
1467
1468 @chartdatanames = ();
1469 @chartdatavals = ();
1470 $chartotherval = 0;
1471 my $use_gig = 0;
1472 foreach my $key (top_n_sort($topcount,$m_data_gigs,$m_data,$m_count)) {
1473 # The largest volume will be the first (top of the list).
1474 # If it has at least 1 gig, then just use gigabytes to avoid
1475 # risking an integer overflow when generating the pie charts.
1476 if ($$m_data_gigs{$key}) {
1477 $use_gig = 1;
1478 }
d5692f86 1479
a83c7e95
SC		 1480 my $messages = $$m_count{$key};
1481 my @content = ($messages);
1482 push(@content, $$a_count{$key}) if defined $a_count;
1483 my $rounded_volume = volume_rounded($$m_data{$key},$$m_data_gigs{$key});
1484 my($data ,$gigs) = (0,0);
1485 un_round($rounded_volume,\$data,\$gigs);
1486 my $rounded_average = volume_rounded($data/$messages,$gigs/$messages);
1487 push(@content, $rounded_volume, $rounded_average );
1488
1489 # write content
1490 printf $txt_fh ($txt_format, @content, $key) if $txt_fh;
1491 if ($htm_fh) {
1492 my $htmlkey = $key;
1493 $htmlkey =~ s/>/\&gt\;/g;
1494 $htmlkey =~ s/</\&lt\;/g;
1495 printf $htm_fh ($htm_format, @content, $htmlkey);
1496 }
1497 $spreadsheet->write(${$row_sref}++, 0, [@content, $key], $f_default) if $xls_fh;
059ec3d9 1498
d5692f86 1499
a83c7e95
SC		 1500 if (scalar @chartdatanames < $ntopchart) {
1501 if ($use_gig) {
1502 if ($$m_data_gigs{$key}) {
1503 push(@chartdatanames, $key);
1504 push(@chartdatavals, $$m_data_gigs{$key});
1505 }
1506 }
1507 else {
1508 push(@chartdatanames, $key);
1509 push(@chartdatavals, $$m_data{$key});
1510 }
1511 }
1512 else {
1513 $chartotherval += ($use_gig) ? $$m_data_gigs{$key} : $$m_data{$key};
8974000d 1514 }
059ec3d9 1515 }
a83c7e95
SC		 1516 push(@chartdatanames, "Other");
1517 push(@chartdatavals, $chartotherval);
059ec3d9 1518
a83c7e95
SC		 1519 print $txt_fh "\n" if $txt_fh;
1520 if ($htm_fh) {
1521 print $htm_fh "</table>\n";
1522 print $htm_fh "</td><td>\n";
1523 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals > 0)) {
1524 # calculate the graph
1525 my @data = (
1526 \@chartdatanames,
1527 \@chartdatavals
1528 );
1529 my $graph = GD::Graph::pie->new(300, 300);
1530 $graph->set(
1531 x_label => 'Name',
1532 y_label => 'Volume' ,
1533 title => 'By Volume',
1534 );
1535 my $gd = $graph->plot(\@data) or warn($graph->error);
1536 if ($gd) {
1537 my $temp = $text;
1538 $temp =~ s/ /_/g;
1539 open(IMG, ">$chartdir/${temp}_volume.png") or die "Could not write $chartdir/${temp}_volume.png: $!\n";
1540 binmode IMG;
1541 print IMG $gd->png;
1542 close IMG;
1543 print $htm_fh "<img src=\"$chartrel/${temp}_volume.png\">";
1544 }
1545 }
1546 print $htm_fh "</td><td>\n";
1547 print $htm_fh "</td></tr></table>\n\n";
059ec3d9 1548 }
a83c7e95
SC		 1549
1550 ++${$row_sref} if $xls_fh;
059ec3d9 1551 }
059ec3d9
PH		 1552}
1553
1554
1b4fe9dd
PH		 1555#######################################################################
1556# top_n_sort();
8e669ac1 1557#
1b4fe9dd 1558# @sorted_keys = top_n_sort($n,$href1,$href2,$href3);
8e669ac1 1559#
1b4fe9dd
PH		 1560# Given a hash which has numerical values, return the sorted $n keys which
1561# point to the top values. The second and third hashes are used as
1562# tiebreakers. They all must have the same keys.
8e669ac1 1563#
1b4fe9dd
PH		 1564# The idea behind this routine is that when you only want to see the
1565# top n members of a set, rather than sorting the entire set and then
1566# plucking off the top n, sort through the stack as you go, discarding
1567# any member which is lower than your current n'th highest member.
8e669ac1 1568#
1b4fe9dd
PH		 1569# This proves to be an order of magnitude faster for large hashes.
1570# On 200,000 lines of mainlog it benchmarked 9 times faster.
1571# On 700,000 lines of mainlog it benchmarked 13.8 times faster.
8e669ac1 1572#
1b4fe9dd
PH		 1573# We assume the values are > 0.
1574#######################################################################
059ec3d9
PH		 1575sub top_n_sort {
1576 my($n,$href1,$href2,$href3) = @_;
1577
1578 # PH's original sort was:
1579 #
1580 # foreach $key (sort
1581 # {
1582 # $$m_count{$b} <=> $$m_count{$a} ||
1583 # $$m_data_gigs{$b} <=> $$m_data_gigs{$a} ||
1584 # $$m_data{$b} <=> $$m_data{$a} ||
1585 # $a cmp $b
1586 # }
1587 # keys %{$m_count})
1588 #
1589
1590 #We use a key of '_' to represent non-existant values, as null keys are valid.
1591 #'_' is not a valid domain, edomain, host, or email.
1592 my(@top_n_keys) = ('_') x $n;
1593 my($minimum_value1,$minimum_value2,$minimum_value3) = (0,0,0);
1594 my $top_n_key = '';
1595 my $n_minus_1 = $n - 1;
1596 my $n_minus_2 = $n - 2;
1597
a83c7e95
SC		 1598 # Create a dummy hash incase the user has not provided us with
1599 # tiebreaker hashes.
1600 my(%dummy_hash);
1601 $href2 = \%dummy_hash unless defined $href2;
1602 $href3 = \%dummy_hash unless defined $href3;
1603
8e669ac1 1604 # Pick out the top $n keys.
059ec3d9
PH		 1605 my($key,$value1,$value2,$value3,$i,$comparison,$insert_position);
1606 while (($key,$value1) = each %$href1) {
1607
1608 #print STDERR "key $key ($value1,",$href2->{$key},",",$href3->{$key},") <=> ($minimum_value1,$minimum_value2,$minimum_value3)\n";
8e669ac1 1609
059ec3d9 1610 # Check to see that the new value is bigger than the lowest of the
a83c7e95
SC		 1611 # top n keys that we're keeping. We test the main key first, because
1612 # for the majority of cases we can skip creating dummy hash values
1613 # should the user have not provided real tie-breaking hashes.
1614 next unless $value1 >= $minimum_value1;
1615
1616 # Create a dummy hash entry for the key if required.
1617 # Note that setting the dummy_hash value sets it for both href2 &
1618 # href3. Also note that currently we are guarenteed to have a real
1619 # value for href3 if a real value for href2 exists so don't need to
1620 # test for it as well.
1621 $dummy_hash{$key} = 0 unless exists $href2->{$key};
1622
8e669ac1 1623 $comparison = $value1 <=> $minimum_value1 ||
d5692f86
SC		 1624 $href2->{$key} <=> $minimum_value2 ||
1625 $href3->{$key} <=> $minimum_value3 ||
1626 $top_n_key cmp $key;
059ec3d9
PH		 1627 next unless ($comparison == 1);
1628
1629 # As we will be using these values a few times, extract them into scalars.
1630 $value2 = $href2->{$key};
1631 $value3 = $href3->{$key};
1632
1633 # This key is bigger than the bottom n key, so the lowest position we
1634 # will insert it into is $n minus 1 (the bottom of the list).
1635 $insert_position = $n_minus_1;
1636
1637 # Now go through the list, stopping when we find a key that we're
1638 # bigger than, or we come to the penultimate position - we've
1639 # already tested bigger than the last.
1640 #
1641 # Note: we go top down as the list starts off empty.
1642 # Note: stepping through the list in this way benchmarks nearly
1643 # three times faster than doing a sort() on the reduced list.
1644 # I assume this is because the list is already in order, and
1645 # we get a performance boost from not having to do hash lookups
1646 # on the new key.
1647 for ($i = 0; $i < $n_minus_1; $i++) {
1648 $top_n_key = $top_n_keys[$i];
1649 if ( ($top_n_key eq '_') ||
d5692f86 1650 ( ($value1 <=> $href1->{$top_n_key} ||
059ec3d9 1651 $value2 <=> $href2->{$top_n_key} ||
d5692f86
SC		 1652 $value3 <=> $href3->{$top_n_key} ||
1653 $top_n_key cmp $key) == 1
1654 )
1655 ) {
1656 $insert_position = $i;
1657 last;
059ec3d9
PH		 1658 }
1659 }
1660
1661 # Remove the last element, then insert the new one.
1662 $#top_n_keys = $n_minus_2;
1663 splice(@top_n_keys,$insert_position,0,$key);
1664
1665 # Extract our new minimum values.
1666 $top_n_key = $top_n_keys[$n_minus_1];
1667 if ($top_n_key ne '_') {
1668 $minimum_value1 = $href1->{$top_n_key};
1669 $minimum_value2 = $href2->{$top_n_key};
1670 $minimum_value3 = $href3->{$top_n_key};
1671 }
1672 }
1673
1674 # Return the top n list, grepping out non-existant values, just in case
1675 # we didn't have that many values.
1676 return(grep(!/^_$/,@top_n_keys));
1677}
1678
1679
a83c7e95 1680
1b4fe9dd
PH		 1681#######################################################################
1682# html_header();
8e669ac1 1683#
1b4fe9dd 1684# $header = html_header($title);
8e669ac1 1685#
1b4fe9dd
PH		 1686# Print our HTML header and start the <body> block.
1687#######################################################################
059ec3d9
PH		 1688sub html_header {
1689 my($title) = @_;
1690 my $text = << "EoText";
1691<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
1692<html>
1693<head>
1694<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-15">
1695<title>$title</title>
1696</head>
1697<body bgcolor="white">
1698<h1>$title</h1>
1699EoText
1700 return $text;
1701}
1702
1703
1704
1b4fe9dd
PH		 1705#######################################################################
1706# help();
8e669ac1 1707#
1b4fe9dd 1708# help();
8e669ac1 1709#
1b4fe9dd
PH		 1710# Display usage instructions and exit.
1711#######################################################################
059ec3d9
PH		 1712sub help {
1713 print << "EoText";
1714
1715eximstats Version $VERSION
1716
d5692f86
SC		 1717Usage:
1718 eximstats [Output] [Options] mainlog1 mainlog2 ...
1719 eximstats -merge -html [Options] report.1.html ... > weekly_rep.html
1720
1721Examples:
1722 eximstats -html=eximstats.html mainlog1 mainlog2 ...
1723 eximstats mainlog1 mainlog2 ... > report.txt
059ec3d9 1724
8974000d 1725Parses exim mainlog or syslog files and generates a statistical analysis
d5692f86
SC		 1726of the messages processed.
1727
1728Valid output types are:
1729-txt[=<file>] plain text (default unless no other type is specified)
1730-html[=<file>] HTML
1731-xls[=<file>] Excel
1732With no type and file given, defaults to -txt and STDOUT.
059ec3d9 1733
d5692f86 1734Valid options are:
059ec3d9
PH		 1735-h<number> histogram divisions per hour. The default is 1, and
1736 0 suppresses histograms. Other valid values are:
d5692f86 1737 2, 3, 5, 10, 15, 20, 30 or 60.
059ec3d9
PH		 1738-ne don't display error information
1739-nr don't display relaying information
1740-nr/pattern/ don't display relaying information that matches
1741-nt don't display transport information
1742-nt/pattern/ don't display transport information that matches
d5692f86 1743-nvr don't do volume rounding. Display in bytes, not KB/MB/GB.
059ec3d9
PH		 1744-t<number> display top <number> sources/destinations
1745 default is 50, 0 suppresses top listing
1746-tnl omit local sources/destinations in top listing
1747-t_remote_users show top user sources/destinations from non-local domains
608bc29d
SC		 1748-q<list> list of times for queuing information. -q0 suppresses.
1749-show_rt<list> Show the receipt times for all the messages.
1750-show_dt<list> Show the delivery times for all the messages.
1751 <list> is an optional list of times in seconds.
1752 Eg -show_rt1,2,4,8.
1753
d5692f86
SC		 1754-include_original_destination show both the final and original
1755 destinations in the results rather than just the final ones.
059ec3d9 1756
d5692f86 1757-byhost show results by sending host (default unless bydomain or
059ec3d9 1758 byemail is specified)
d5692f86
SC		 1759-bydomain show results by sending domain.
1760-byemail show results by sender's email address
1761-byedomain show results by sender's email domain
059ec3d9
PH		 1762
1763-pattern "Description" /pattern/
1764 Count lines matching specified patterns and show them in
d5692f86
SC		 1765 the results. It can be specified multiple times. Eg:
1766 -pattern 'Refused connections' '/refused connection/'
059ec3d9
PH		 1767
1768-merge merge previously generated reports into a new report
1769
d5692f86
SC		 1770-charts Create charts (this requires the GD::Graph modules).
1771 Only valid with -html.
059ec3d9
PH		 1772-chartdir <dir> Create the charts' png files in the directory <dir>
1773-chartrel <dir> Specify the relative directory for the "img src=" tags
1774 from where to include the charts in the html file
d5692f86 1775 -chartdir and -chartrel default to '.'
059ec3d9 1776
a83c7e95
SC		 1777-emptyok It is OK if there is no valid input, don't print an error.
1778
d5692f86 1779-d Debug mode - dump the eval'ed parser onto STDERR.
059ec3d9
PH		 1780
1781EoText
1782
1783 exit 1;
1784}
1785
1786
1787
1b4fe9dd
PH		 1788#######################################################################
1789# generate_parser();
8e669ac1 1790#
1b4fe9dd 1791# $parser = generate_parser();
8e669ac1 1792#
1b4fe9dd
PH		 1793# This subroutine generates the parsing routine which will be
1794# used to parse the mainlog. We take the base operation, and remove bits not in use.
1795# This improves performance depending on what bits you take out or add.
8e669ac1 1796#
1b4fe9dd 1797# I've tested using study(), but this does not improve performance.
8e669ac1 1798#
1b4fe9dd
PH		 1799# We store our parsing routing in a variable, and process it looking for #IFDEF (Expression)
1800# or #IFNDEF (Expression) statements and corresponding #ENDIF (Expression) statements. If
1801# the expression evaluates to true, then it is included/excluded accordingly.
1802#######################################################################
059ec3d9
PH		 1803sub generate_parser {
1804 my $parser = '
1805 my($ip,$host,$email,$edomain,$domain,$thissize,$size,$old,$new);
a83c7e95 1806 my($tod,$m_hour,$m_min,$id,$flag,$extra,$length);
608bc29d 1807 my($seconds,$queued,$rcpt_time);
a83c7e95 1808 my $rej_id = 0;
059ec3d9 1809 while (<$fh>) {
059ec3d9 1810
8974000d
SC		 1811 # Convert syslog lines to mainlog format.
1812 if (! /^\\d{4}/) {
1813 next unless s/^.*? exim\\b.*?: //;
1814 }
1815
a83c7e95
SC		 1816 $length = length($_);
1817 next if ($length < 38);
059ec3d9
PH		 1818 next unless /^(\\d{4}\\-\\d\\d-\\d\\d\\s(\\d\\d):(\\d\\d):\\d\\d( [-+]\\d\\d\\d\\d)?)/o;
1819
1820 ($tod,$m_hour,$m_min) = ($1,$2,$3);
1821
a83c7e95
SC		 1822 # PH - watch for GMT offsets in the timestamp.
1823 if (defined($4)) {
1824 $extra = 6;
1825 next if ($length < 44);
1826 }
1827 else {
1828 $extra = 0;
1829 }
1830
059ec3d9
PH		 1831 $id = substr($_, 20 + $extra, 16);
1832 $flag = substr($_, 37 + $extra, 2);
a83c7e95
SC		 1833
1834 if ($flag !~ /^([<>=*-]+|SA)$/ && /rejected|refused|dropped/) {
1835 $flag = "Re";
1836 $extra -= 3;
1837 }
1838
1839 # Rejects can have no MSGID...
1840 if ($flag eq "Re" && $id !~ /^[-0-9a-zA-Z]+$/) {
1841 $id = "reject:" . ++$rej_id;
1842 $extra -= 17;
1843 }
059ec3d9
PH		 1844';
1845
1846 # Watch for user specified patterns.
1847 my $user_pattern_index = 0;
1848 foreach (@user_patterns) {
1849 $user_pattern_totals[$user_pattern_index] = 0;
f2b67a5b
SC		 1850 $parser .= " if ($_) {\n";
1851 $parser .= " \$user_pattern_totals[$user_pattern_index]++;\n";
1852 $parser .= " \$user_pattern_interval_count[$user_pattern_index][(\$m_hour*60 + \$m_min)/$hist_interval]++;\n" if ($hist_opt > 0);
1853 $parser .= " }\n";
059ec3d9
PH		 1854 $user_pattern_index++;
1855 }
1856
1857 $parser .= '
a83c7e95 1858 next unless ($flag =~ /<=|=>|->|==|\\*\\*|Co|SA|Re/);
059ec3d9 1859
a83c7e95
SC		 1860 #Strip away the timestamp, ID and flag to speed up later pattern matches.
1861 #The flags include Co (Completed), Re (Rejected), and SA (SpamAssassin).
059ec3d9
PH		 1862 $_ = substr($_, 40 + $extra); # PH
1863
a83c7e95 1864 # Alias @message to the array of information about the message.
608bc29d
SC		 1865 # This minimises the number of calls to hash functions.
1866 $messages{$id} = [] unless exists $messages{$id};
a83c7e95 1867 *message = $messages{$id};
608bc29d
SC		 1868
1869
059ec3d9
PH		 1870 # JN - Skip over certain transports as specified via the "-nt/.../" command
1871 # line switch (where ... is a perl style regular expression). This is
1872 # required so that transports that skew stats such as SpamAssassin can be
1873 # ignored.
1874 #IFDEF ($transport_pattern)
1875 if (/\\sT=(\\S+)/) {
1876 next if ($1 =~ /$transport_pattern/o) ;
1877 }
1878 #ENDIF ($transport_pattern)
1879
1880
059ec3d9
PH		 1881
1882 # Do some pattern matches to get the host and IP address.
1883 # We expect lines to be of the form "H=[IpAddr]" or "H=Host [IpAddr]" or
1884 # "H=Host (UnverifiedHost) [IpAddr]" or "H=(UnverifiedHost) [IpAddr]".
1885 # We do 2 separate matches to keep the matches simple and fast.
a83c7e95
SC		 1886 # Host is local unless otherwise specified.
1887 $ip = (/\\bH=.*?(\\[[^]]+\\])/) ? $1 : "local";
1888 $host = (/\\bH=(\\S+)/) ? $1 : "local";
059ec3d9 1889
a83c7e95 1890 $domain = "localdomain"; #Domain is localdomain unless otherwise specified.
059ec3d9 1891
a83c7e95
SC		 1892 #IFDEF ($do_sender{Domain})
1893 if ($host !~ /^\\[/ && $host =~ /^(\\(?)[^\\.]+\\.([^\\.]+\\..*)/) {
1894 # Remove the host portion from the DNS name. We ensure that we end up
1895 # with at least xxx.yyy. $host can be "(x.y.z)" or "x.y.z".
1896 $domain = lc("$1.$2");
1897 $domain =~ s/^\\.//; #Remove preceding dot.
059ec3d9 1898 }
a83c7e95 1899 #ENDIF ($do_sender{Domain})
059ec3d9
PH		 1900
1901 #IFDEF ($do_sender{Email})
d5692f86
SC		 1902 #IFDEF ($include_original_destination)
1903 # Catch both "a@b.com <c@d.com>" and "e@f.com"
1904 #$email = (/^(\S+) (<(\S*?)>)?/) ? $3 || $1 : "";
1905 $email = (/^(\S+ (<[^@>]+@?[^>]*>)?)/) ? $1 : "";
1906 chomp($email);
1907 #ENDIF ($include_original_destination)
1908
1909 #IFNDEF ($include_original_destination)
1910 $email = (/^(\S+)/) ? $1 : "";
1911 #ENDIF ($include_original_destination)
059ec3d9
PH		 1912 #ENDIF ($do_sender{Email})
1913
1914 #IFDEF ($do_sender{Edomain})
d5692f86
SC		 1915 #IFDEF ($include_original_destination)
1916 #$edomain = (/^(\S+) (<\S*?\\@(\S+)>)?/) ? $3 || $1 : "";
1917 $edomain = (/^(\S+ (<\S*?\\@(\S+?)>)?)/) ? $1 : "";
1918 chomp($edomain);
1919 lc($edomain);
1920 #ENDIF ($include_original_destination)
1921
1922 #IFNDEF ($include_original_destination)
1923 $edomain = (/^\S*?\\@(\S+)/) ? lc($1) : "";
1924 #ENDIF ($include_original_destination)
059ec3d9
PH		 1925 #ENDIF ($do_sender{Edomain})
1926
1927 if ($tod lt $begin) {
1928 $begin = $tod;
1929 }
1930 elsif ($tod gt $end) {
1931 $end = $tod;
1932 }
1933
1934
1935 if ($flag eq "<=") {
1936 $thissize = (/\\sS=(\\d+)( |$)/) ? $1 : 0;
a83c7e95
SC		 1937 $message[$SIZE] = $thissize;
1938 $message[$PROTOCOL] = (/ P=(\S+)/) ? $1 : undef;
059ec3d9
PH		 1939
1940 #IFDEF ($show_relay)
1941 if ($host ne "local") {
d5692f86
SC		 1942 # Save incoming information in case it becomes interesting
1943 # later, when delivery lines are read.
1944 my($from) = /^(\\S+)/;
a83c7e95
SC		 1945 $message[$FROM_HOST] = "$host$ip";
1946 $message[$FROM_ADDRESS] = $from;
059ec3d9
PH		 1947 }
1948 #ENDIF ($show_relay)
1949
1950 #IFDEF ($local_league_table || $include_remote_users)
d5692f86
SC		 1951 if (/\sU=(\\S+)/) {
1952 my $user = $1;
059ec3d9 1953
d5692f86
SC		 1954 #IFDEF ($local_league_table && $include_remote_users)
1955 { #Store both local and remote users.
1956 #ENDIF ($local_league_table && $include_remote_users)
059ec3d9 1957
d5692f86
SC		 1958 #IFDEF ($local_league_table && ! $include_remote_users)
1959 if ($host eq "local") { #Store local users only.
1960 #ENDIF ($local_league_table && ! $include_remote_users)
059ec3d9 1961
d5692f86
SC		 1962 #IFDEF ($include_remote_users && ! $local_league_table)
1963 if ($host ne "local") { #Store remote users only.
1964 #ENDIF ($include_remote_users && ! $local_league_table)
059ec3d9 1965
608bc29d 1966 ++$received_count_user{$user};
d5692f86 1967 add_volume(\\$received_data_user{$user},\\$received_data_gigs_user{$user},$thissize);
059ec3d9 1968 }
d5692f86 1969 }
059ec3d9
PH		 1970 #ENDIF ($local_league_table || $include_remote_users)
1971
1972 #IFDEF ($do_sender{Host})
608bc29d 1973 ++$received_count{Host}{$host};
d5692f86 1974 add_volume(\\$received_data{Host}{$host},\\$received_data_gigs{Host}{$host},$thissize);
059ec3d9
PH		 1975 #ENDIF ($do_sender{Host})
1976
1977 #IFDEF ($do_sender{Domain})
1978 if ($domain) {
608bc29d 1979 ++$received_count{Domain}{$domain};
d5692f86
SC		 1980 add_volume(\\$received_data{Domain}{$domain},\\$received_data_gigs{Domain}{$domain},$thissize);
1981 }
059ec3d9
PH		 1982 #ENDIF ($do_sender{Domain})
1983
1984 #IFDEF ($do_sender{Email})
608bc29d 1985 ++$received_count{Email}{$email};
d5692f86 1986 add_volume(\\$received_data{Email}{$email},\\$received_data_gigs{Email}{$email},$thissize);
059ec3d9
PH		 1987 #ENDIF ($do_sender{Email})
1988
1989 #IFDEF ($do_sender{Edomain})
608bc29d 1990 ++$received_count{Edomain}{$edomain};
d5692f86 1991 add_volume(\\$received_data{Edomain}{$edomain},\\$received_data_gigs{Edomain}{$edomain},$thissize);
059ec3d9
PH		 1992 #ENDIF ($do_sender{Edomain})
1993
608bc29d 1994 ++$total_received_count;
059ec3d9
PH		 1995 add_volume(\\$total_received_data,\\$total_received_data_gigs,$thissize);
1996
608bc29d 1997 #IFDEF ($#queue_times >= 0 || $#rcpt_times >= 0)
a83c7e95 1998 $message[$ARRIVAL_TIME] = $tod;
608bc29d 1999 #ENDIF ($#queue_times >= 0 || $#rcpt_times >= 0)
059ec3d9
PH		 2000
2001 #IFDEF ($hist_opt > 0)
d5692f86 2002 $received_interval_count[($m_hour*60 + $m_min)/$hist_interval]++;
059ec3d9
PH		 2003 #ENDIF ($hist_opt > 0)
2004 }
2005
2006 elsif ($flag eq "=>") {
a83c7e95 2007 $size = $message[$SIZE] || 0;
059ec3d9 2008 if ($host ne "local") {
a83c7e95 2009 $message[$REMOTE_DELIVERED] = 1;
059ec3d9
PH		 2010
2011
2012 #IFDEF ($show_relay)
2013 # Determine relaying address if either only one address listed,
2014 # or two the same. If they are different, it implies a forwarding
2015 # or aliasing, which is not relaying. Note that for multi-aliased
2016 # addresses, there may be a further address between the first
2017 # and last.
2018
a83c7e95 2019 if (defined $message[$FROM_HOST]) {
059ec3d9
PH		 2020 if (/^(\\S+)(?:\\s+\\([^)]\\))?\\s+<([^>]+)>/) {
2021 ($old,$new) = ($1,$2);
d5692f86 2022 }
059ec3d9 2023 else {
d5692f86
SC		 2024 $old = $new = "";
2025 }
059ec3d9
PH		 2026
2027 if ("\\L$new" eq "\\L$old") {
2028 ($old) = /^(\\S+)/ if $old eq "";
a83c7e95 2029 my $key = "H=\\L$message[$FROM_HOST]\\E A=\\L$message[$FROM_ADDRESS]\\E => " .
059ec3d9
PH		 2030 "H=\\L$host\\E$ip A=\\L$old\\E";
2031 if (!defined $relay_pattern || $key !~ /$relay_pattern/o) {
2032 $relayed{$key} = 0 if !defined $relayed{$key};
608bc29d 2033 ++$relayed{$key};
d5692f86 2034 }
059ec3d9 2035 else {
608bc29d 2036 ++$relayed_unshown;
059ec3d9
PH		 2037 }
2038 }
2039 }
2040 #ENDIF ($show_relay)
2041
2042 }
2043
2044 #IFDEF ($local_league_table || $include_remote_users)
d5692f86
SC		 2045 #IFDEF ($local_league_table && $include_remote_users)
2046 { #Store both local and remote users.
2047 #ENDIF ($local_league_table && $include_remote_users)
2048
2049 #IFDEF ($local_league_table && ! $include_remote_users)
2050 if ($host eq "local") { #Store local users only.
2051 #ENDIF ($local_league_table && ! $include_remote_users)
2052
2053 #IFDEF ($include_remote_users && ! $local_league_table)
2054 if ($host ne "local") { #Store remote users only.
2055 #ENDIF ($include_remote_users && ! $local_league_table)
2056
2057 if (my($user) = split((/\\s</)? " <" : " ", $_)) {
2058 #IFDEF ($include_original_destination)
2059 {
2060 #ENDIF ($include_original_destination)
2061 #IFNDEF ($include_original_destination)
2062 if ($user =~ /^[\\/|]/) {
2063 #ENDIF ($include_original_destination)
2064 my($parent) = $_ =~ /(<[^@]+@?[^>]*>)/;
2065 $user = "$user $parent" if defined $parent;
2066 }
a83c7e95
SC		 2067 ++$delivered_messages_user{$user};
2068 ++$delivered_addresses_user{$user};
d5692f86
SC		 2069 add_volume(\\$delivered_data_user{$user},\\$delivered_data_gigs_user{$user},$size);
2070 }
2071 }
059ec3d9
PH		 2072 #ENDIF ($local_league_table || $include_remote_users)
2073
2074 #IFDEF ($do_sender{Host})
a83c7e95
SC		 2075 $delivered_messages{Host}{$host}++;
2076 $delivered_addresses{Host}{$host}++;
d5692f86 2077 add_volume(\\$delivered_data{Host}{$host},\\$delivered_data_gigs{Host}{$host},$size);
059ec3d9
PH		 2078 #ENDIF ($do_sender{Host})
2079 #IFDEF ($do_sender{Domain})
2080 if ($domain) {
a83c7e95
SC		 2081 ++$delivered_messages{Domain}{$domain};
2082 ++$delivered_addresses{Domain}{$domain};
d5692f86
SC		 2083 add_volume(\\$delivered_data{Domain}{$domain},\\$delivered_data_gigs{Domain}{$domain},$size);
2084 }
059ec3d9
PH		 2085 #ENDIF ($do_sender{Domain})
2086 #IFDEF ($do_sender{Email})
a83c7e95
SC		 2087 ++$delivered_messages{Email}{$email};
2088 ++$delivered_addresses{Email}{$email};
d5692f86 2089 add_volume(\\$delivered_data{Email}{$email},\\$delivered_data_gigs{Email}{$email},$size);
059ec3d9
PH		 2090 #ENDIF ($do_sender{Email})
2091 #IFDEF ($do_sender{Edomain})
a83c7e95
SC		 2092 ++$delivered_messages{Edomain}{$edomain};
2093 ++$delivered_addresses{Edomain}{$edomain};
d5692f86 2094 add_volume(\\$delivered_data{Edomain}{$edomain},\\$delivered_data_gigs{Edomain}{$edomain},$size);
059ec3d9
PH		 2095 #ENDIF ($do_sender{Edomain})
2096
a83c7e95
SC		 2097 ++$total_delivered_messages;
2098 ++$total_delivered_addresses;
059ec3d9
PH		 2099 add_volume(\\$total_delivered_data,\\$total_delivered_data_gigs,$size);
2100
2101 #IFDEF ($show_transport)
2102 my $transport = (/\\sT=(\\S+)/) ? $1 : ":blackhole:";
608bc29d 2103 ++$transported_count{$transport};
059ec3d9
PH		 2104 add_volume(\\$transported_data{$transport},\\$transported_data_gigs{$transport},$size);
2105 #ENDIF ($show_transport)
2106
2107 #IFDEF ($hist_opt > 0)
2108 $delivered_interval_count[($m_hour*60 + $m_min)/$hist_interval]++;
2109 #ENDIF ($hist_opt > 0)
2110
608bc29d
SC		 2111 #IFDEF ($#delivery_times > 0)
2112 if (/ DT=(\S+)/) {
2113 $seconds = wdhms_seconds($1);
2114 for ($i = 0; $i <= $#delivery_times; $i++) {
2115 if ($seconds < $delivery_times[$i]) {
2116 ++$dt_all_bin[$i];
a83c7e95 2117 ++$dt_remote_bin[$i] if $message[$REMOTE_DELIVERED];
608bc29d
SC		 2118 last;
2119 }
2120 }
2121 if ($i > $#delivery_times) {
2122 ++$dt_all_overflow;
a83c7e95 2123 ++$dt_remote_overflow if $message[$REMOTE_DELIVERED];
608bc29d
SC		 2124 }
2125 }
2126 #ENDIF ($#delivery_times > 0)
2127
059ec3d9
PH		 2128 }
2129
a83c7e95
SC		 2130 elsif ($flag eq "->") {
2131
2132 #IFDEF ($local_league_table || $include_remote_users)
2133 #IFDEF ($local_league_table && $include_remote_users)
2134 { #Store both local and remote users.
2135 #ENDIF ($local_league_table && $include_remote_users)
2136
2137 #IFDEF ($local_league_table && ! $include_remote_users)
2138 if ($host eq "local") { #Store local users only.
2139 #ENDIF ($local_league_table && ! $include_remote_users)
2140
2141 #IFDEF ($include_remote_users && ! $local_league_table)
2142 if ($host ne "local") { #Store remote users only.
2143 #ENDIF ($include_remote_users && ! $local_league_table)
2144
2145 if (my($user) = split((/\\s</)? " <" : " ", $_)) {
2146 #IFDEF ($include_original_destination)
2147 {
2148 #ENDIF ($include_original_destination)
2149 #IFNDEF ($include_original_destination)
2150 if ($user =~ /^[\\/|]/) {
2151 #ENDIF ($include_original_destination)
2152 my($parent) = $_ =~ /(<[^@]+@?[^>]*>)/;
2153 $user = "$user $parent" if defined $parent;
2154 }
2155 ++$delivered_addresses_user{$user};
2156 }
2157 }
2158 #ENDIF ($local_league_table || $include_remote_users)
2159
2160 #IFDEF ($do_sender{Host})
2161 $delivered_addresses{Host}{$host}++;
2162 #ENDIF ($do_sender{Host})
2163 #IFDEF ($do_sender{Domain})
2164 if ($domain) {
2165 ++$delivered_addresses{Domain}{$domain};
2166 }
2167 #ENDIF ($do_sender{Domain})
2168 #IFDEF ($do_sender{Email})
2169 ++$delivered_addresses{Email}{$email};
2170 #ENDIF ($do_sender{Email})
2171 #IFDEF ($do_sender{Edomain})
2172 ++$delivered_addresses{Edomain}{$edomain};
2173 #ENDIF ($do_sender{Edomain})
2174
2175 ++$total_delivered_addresses;
2176 }
2177
2178 elsif ($flag eq "==" && defined($message[$SIZE]) && !defined($message[$DELAYED])) {
608bc29d 2179 ++$delayed_count;
a83c7e95 2180 $message[$DELAYED] = 1;
059ec3d9
PH		 2181 }
2182
2183 elsif ($flag eq "**") {
a83c7e95
SC		 2184 if (defined ($message[$SIZE])) {
2185 unless (defined $message[$HAD_ERROR]) {
608bc29d 2186 ++$message_errors;
a83c7e95 2187 $message[$HAD_ERROR] = 1;
608bc29d
SC		 2188 }
2189 }
059ec3d9
PH		 2190
2191 #IFDEF ($show_errors)
608bc29d 2192 ++$errors_count{$_};
059ec3d9
PH		 2193 #ENDIF ($show_errors)
2194
2195 }
2196
2197 elsif ($flag eq "Co") {
2198 #Completed?
2199 #IFDEF ($#queue_times >= 0)
a83c7e95 2200 $queued = queue_time($tod, $message[$ARRIVAL_TIME], $id);
059ec3d9
PH		 2201
2202 for ($i = 0; $i <= $#queue_times; $i++) {
2203 if ($queued < $queue_times[$i]) {
608bc29d 2204 ++$qt_all_bin[$i];
a83c7e95 2205 ++$qt_remote_bin[$i] if $message[$REMOTE_DELIVERED];
059ec3d9 2206 last;
d5692f86
SC		 2207 }
2208 }
608bc29d
SC		 2209 if ($i > $#queue_times) {
2210 ++$qt_all_overflow;
a83c7e95 2211 ++$qt_remote_overflow if $message[$REMOTE_DELIVERED];
608bc29d 2212 }
059ec3d9
PH		 2213 #ENDIF ($#queue_times >= 0)
2214
608bc29d
SC		 2215 #IFDEF ($#rcpt_times >= 0)
2216 if (/ QT=(\S+)/) {
2217 $seconds = wdhms_seconds($1);
2218 #Calculate $queued if not previously calculated above.
2219 #IFNDEF ($#queue_times >= 0)
a83c7e95 2220 $queued = queue_time($tod, $message[$ARRIVAL_TIME], $id);
608bc29d
SC		 2221 #ENDIF ($#queue_times >= 0)
2222 $rcpt_time = $seconds - $queued;
2223 my($protocol);
2224
a83c7e95
SC		 2225 if (defined $message[$PROTOCOL]) {
2226 $protocol = $message[$PROTOCOL];
608bc29d
SC		 2227
2228 # Create the bin if its not already defined.
2229 unless (exists $rcpt_times_bin{$protocol}) {
2230 initialise_rcpt_times($protocol);
2231 }
2232 }
2233
2234
2235 for ($i = 0; $i <= $#rcpt_times; ++$i) {
2236 if ($rcpt_time < $rcpt_times[$i]) {
2237 ++$rcpt_times_bin{all}[$i];
2238 ++$rcpt_times_bin{$protocol}[$i] if defined $protocol;
2239 last;
2240 }
2241 }
059ec3d9 2242
608bc29d
SC		 2243 if ($i > $#rcpt_times) {
2244 ++$rcpt_times_overflow{all};
2245 ++$rcpt_times_overflow{$protocol} if defined $protocol;
2246 }
2247 }
2248 #ENDIF ($#rcpt_times >= 0)
2249
2250 delete($messages{$id});
059ec3d9 2251 }
a83c7e95
SC		 2252 elsif ($flag eq "SA") {
2253 $ip = (/From.*?(\\[[^]]+\\])/ || /\\((local)\\)/) ? $1 : "";
2254 #SpamAssassin message
2255 if (/Action: ((permanently|temporarily) rejected message|flagged as Spam but accepted): score=(\d+\.\d)/) {
2256 #add_volume(\\$spam_score,\\$spam_score_gigs,$3);
2257 ++$spam_count_by_ip{$ip};
2258 } elsif (/Action: scanned but message isn\'t spam: score=(-?\d+\.\d)/) {
2259 #add_volume(\\$ham_score,\\$ham_score_gigs,$1);
2260 ++$ham_count_by_ip{$ip};
2261 } elsif (/(Not running SA because SAEximRunCond expanded to false|check skipped due to message size)/) {
2262 ++$ham_count_by_ip{$ip};
2263 }
2264 }
2265
2266 # Look for Reject messages or blackholed messages (deliveries
2267 # without a transport)
2268 if ($flag eq "Re" || ($flag eq "=>" && ! /\\sT=\\S+/)) {
2269 # Correct the IP address for rejects:
2270 # rejected EHLO from my.test.net [10.0.0.5]: syntactically invalid argument(s):
2271 $ip = $1 if ($ip eq "local" && /^rejected [HE][HE]LO from .*? (\[.+?\]):/);
2272 ++$rejected_count_by_ip{$ip};
2273 if (
2274 /(listed at [^ ]+)/ ||
2275 /(Forged IP detected in HELO)/ ||
2276 /(Invalid domain or IP given in HELO\/EHLO)/ ||
2277 /(unqualified recipient rejected)/ ||
2278 /(closed connection (after|in response) .*?)\s*$/ ||
2279 /(sender rejected)/ ||
2280 # 2005-09-23 15:07:49 1EInHJ-0007Ex-Au H=(a.b.c) [10.0.0.1] F=<> rejected after DATA: This message contains a virus: (Eicar-Test-Signature) please scan your system.
2281 # 2005-10-06 10:50:07 1ENRS3-0000Nr-Kt => blackhole (DATA ACL discarded recipients): This message contains a virus: (Worm.SomeFool.P) please scan your system.
2282 / rejected after DATA: (.*)/ ||
2283 /.DATA ACL discarded recipients.: (.*)/ ||
2284 /rejected after DATA: (unqualified address not permitted)/ ||
2285 /(VRFY rejected)/ ||
2286# /(sender verify (defer|fail))/i ||
2287 /(too many recipients)/ ||
2288 /(refused relay.*?) to/ ||
2289 /(rejected by non-SMTP ACL: .*)/ ||
2290 /(rejected by local_scan.*)/ ||
2291 # SMTP call from %s dropped: too many syntax or protocol errors (last command was "%s"
2292 # SMTP call from %s dropped: too many nonmail commands
2293 /(dropped: too many ((nonmail|unrecognized) commands|syntax or protocol errors))/ ||
2294
2295 # local_scan() function crashed with signal %d - message temporarily rejected
2296 # local_scan() function timed out - message temporarily rejected
2297 /(local_scan.. function .* - message temporarily rejected)/ ||
2298 /(temporarily refused connection)/ ||
2299 # SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from %s
2300 /(SMTP protocol .*?(error|violation))/ ||
2301 /(message too big)/
2302 ) {
2303 ++$rejected_count_by_reason{"\u$1"};
2304 }
2305 elsif (/rejected [HE][HE]LO from [^:]*: syntactically invalid argument/) {
2306 ++$rejected_count_by_reason{"Rejected HELO/EHLO: syntactically invalid argument"};
2307 }
2308 elsif (/response to "RCPT TO.*? was: (.*)/) {
2309 ++$rejected_count_by_reason{"Response to RCPT TO was: $1"};
2310 }
2311 elsif (
2312 /(lookup of host )\S+ (failed)/ ||
2313
2314 # rejected from <%s>%s%s%s%s: message too big:
2315 /(rejected [A-Z]*) .*?(: .*?)(:|\s*$)/ ||
2316 # refused connection from %s (host_reject_connection)
2317 # refused connection from %s (tcp wrappers)
2318 /(refused connection )from.*? (\(.*)/ ||
2319
2320 # error from remote mailer after RCPT TO:<a@b.c>: host a.b.c [10.0.0.1]: 450 <a@b.c>: Recipient address rejected: Greylisted for 60 seconds
2321 # error from remote mailer after MAIL FROM:<> SIZE=3468: host a.b.c [10.0.0.1]: 421 a.b.c has refused your connection because your server did not have a PTR record.
2322 /(error from remote mailer after .*?:).*(: .*?)(:|\s*$)/ ||
2323
2324 # a.b.c F=<a@b.c> rejected after DATA: "@" or "." expected after "Undisclosed-Recipient": failing address in "To" header is: <Undisclosed-Recipient:;>
2325 /rejected after DATA: ("." or "." expected).*?(: failing address in .*? header)/ ||
2326
2327 # connection from %s refused load average = %.2f
2328 /(Connection )from.*? (refused: load average)/ ||
2329 # connection from %s refused (IP options)
2330 # Connection from %s refused: too many connections
2331 # connection from %s refused
2332 /([Cc]onnection )from.*? (refused.*)/ ||
2333 # [10.0.0.1]: connection refused
2334 /: (Connection refused)()/
2335 ) {
2336 ++$rejected_count_by_reason{"\u$1$2"};
2337 }
2338 else {
2339 ++$rejected_count_by_reason{Unknown};
2340 print STDERR "Unknown rejection: $_" if $debug;
2341 }
2342 }
059ec3d9
PH		 2343 }';
2344
2345 # We now do a 'C preprocessor style operation on our parser
2346 # to remove bits not in use.
2347 my(%defines_in_operation,$removing_lines,$processed_parser);
2348 foreach (split (/\n/,$parser)) {
2349 if ((/^\s*#\s*IFDEF\s*\((.*?)\)/i && ! eval $1) ||
d5692f86 2350 (/^\s*#\s*IFNDEF\s*\((.*?)\)/i && eval $1) ) {
059ec3d9
PH		 2351 $defines_in_operation{$1} = 1;
2352 $removing_lines = 1;
2353 }
2354
608bc29d
SC		 2355 # Convert constants.
2356 while (/(\$[A-Z][A-Z_]*)\b/) {
2357 my $constant = eval $1;
2358 s/(\$[A-Z][A-Z_]*)\b/$constant/;
2359 }
2360
059ec3d9
PH		 2361 $processed_parser .= $_."\n" unless $removing_lines;
2362
2363 if (/^\s*#\s*ENDIF\s*\((.*?)\)/i) {
2364 delete $defines_in_operation{$1};
2365 unless (keys %defines_in_operation) {
d5692f86 2366 $removing_lines = 0;
059ec3d9
PH		 2367 }
2368 }
2369 }
608bc29d 2370 print STDERR "# START OF PARSER:$processed_parser\n# END OF PARSER\n\n" if $debug;
059ec3d9
PH		 2371
2372 return $processed_parser;
2373}
2374
2375
2376
1b4fe9dd
PH		 2377#######################################################################
2378# parse();
8e669ac1 2379#
1b4fe9dd 2380# parse($parser,\*FILEHANDLE);
8e669ac1 2381#
1b4fe9dd
PH		 2382# This subroutine accepts a parser and a filehandle from main and parses each
2383# line. We store the results into global variables.
2384#######################################################################
059ec3d9
PH		 2385sub parse {
2386 my($parser,$fh) = @_;
2387
2388 if ($merge_reports) {
2389 parse_old_eximstat_reports($fh);
2390 }
2391 else {
2392 eval $parser;
2393 die ($@) if $@;
2394 }
2395
2396}
2397
2398
2399
1b4fe9dd
PH		 2400#######################################################################
2401# print_header();
8e669ac1 2402#
1b4fe9dd 2403# print_header();
8e669ac1 2404#
1b4fe9dd
PH		 2405# Print our headers and contents.
2406#######################################################################
059ec3d9
PH		 2407sub print_header {
2408
d5692f86 2409
059ec3d9
PH		 2410 my $title = "Exim statistics from $begin to $end";
2411
d5692f86
SC		 2412 print $txt_fh "\n$title\n" if $txt_fh;
2413 if ($htm_fh) {
2414 print $htm_fh html_header($title);
2415 print $htm_fh "<ul>\n";
2416 print $htm_fh "<li><a href=\"#grandtotal\">Grand total summary</a>\n";
2417 print $htm_fh "<li><a href=\"#patterns\">User Specified Patterns</a>\n" if @user_patterns;
2418 print $htm_fh "<li><a href=\"#transport\">Deliveries by Transport</a>\n" if $show_transport;
059ec3d9 2419 if ($hist_opt) {
d5692f86
SC		 2420 print $htm_fh "<li><a href=\"#Messages received\">Messages received per hour</a>\n";
2421 print $htm_fh "<li><a href=\"#Deliveries\">Deliveries per hour</a>\n";
059ec3d9 2422 }
608bc29d 2423
059ec3d9 2424 if ($#queue_times >= 0) {
608bc29d
SC		 2425 print $htm_fh "<li><a href=\"#Time spent on the queue all messages\">Time spent on the queue: all messages</a>\n";
2426 print $htm_fh "<li><a href=\"#Time spent on the queue messages with at least one remote delivery\">Time spent on the queue: messages with at least one remote delivery</a>\n";
2427 }
2428
2429 if ($#delivery_times >= 0) {
2430 print $htm_fh "<li><a href=\"#Delivery times all messages\">Delivery times: all messages</a>\n";
2431 print $htm_fh "<li><a href=\"#Delivery times messages with at least one remote delivery\">Delivery times: messages with at least one remote delivery</a>\n";
2432 }
2433
2434 if ($#rcpt_times >= 0) {
2435 print $htm_fh "<li><a href=\"#Receipt times all messages\">Receipt times</a>\n";
059ec3d9 2436 }
608bc29d 2437
d5692f86 2438 print $htm_fh "<li><a href=\"#Relayed messages\">Relayed messages</a>\n" if $show_relay;
059ec3d9 2439 if ($topcount) {
a83c7e95 2440 print $htm_fh "<li><a href=\"#mail rejection reason count\">Top $topcount mail rejection reasons by message count</a>\n" if %rejected_count_by_reason;
059ec3d9 2441 foreach ('Host','Domain','Email','Edomain') {
d5692f86
SC		 2442 next unless $do_sender{$_};
2443 print $htm_fh "<li><a href=\"#sending \l$_ count\">Top $topcount sending \l${_}s by message count</a>\n";
2444 print $htm_fh "<li><a href=\"#sending \l$_ volume\">Top $topcount sending \l${_}s by volume</a>\n";
059ec3d9 2445 }
a83c7e95 2446 if (($local_league_table || $include_remote_users) && %received_count_user) {
d5692f86
SC		 2447 print $htm_fh "<li><a href=\"#local sender count\">Top $topcount local senders by message count</a>\n";
2448 print $htm_fh "<li><a href=\"#local sender volume\">Top $topcount local senders by volume</a>\n";
059ec3d9
PH		 2449 }
2450 foreach ('Host','Domain','Email','Edomain') {
d5692f86
SC		 2451 next unless $do_sender{$_};
2452 print $htm_fh "<li><a href=\"#\l$_ destination count\">Top $topcount \l$_ destinations by message count</a>\n";
2453 print $htm_fh "<li><a href=\"#\l$_ destination volume\">Top $topcount \l$_ destinations by volume</a>\n";
059ec3d9 2454 }
a83c7e95 2455 if (($local_league_table || $include_remote_users) && %delivered_messages_user) {
d5692f86
SC		 2456 print $htm_fh "<li><a href=\"#local destination count\">Top $topcount local destinations by message count</a>\n";
2457 print $htm_fh "<li><a href=\"#local destination volume\">Top $topcount local destinations by volume</a>\n";
059ec3d9 2458 }
a83c7e95
SC		 2459
2460 print $htm_fh "<li><a href=\"#rejected ip count\">Top $topcount rejected ips by message count</a>\n" if %rejected_count_by_ip;
2461 print $htm_fh "<li><a href=\"#non-rejected spamming ip count\">Top $topcount non-rejected spamming ips by message count</a>\n" if %spam_count_by_ip;
2462
059ec3d9 2463 }
d5692f86
SC		 2464 print $htm_fh "<li><a href=\"#errors\">List of errors</a>\n" if %errors_count;
2465 print $htm_fh "</ul>\n<hr>\n";
059ec3d9 2466 }
d5692f86
SC		 2467 if ($xls_fh)
2468 {
2469 $ws_global->write($row++, $col+0, "Exim Statistics", $f_header1);
2470 &set_worksheet_line($ws_global, $row, $col, ["from:", $begin, "to:", $end], $f_default);
2471 $row+=2;
059ec3d9
PH		 2472 }
2473}
2474
2475
1b4fe9dd
PH		 2476#######################################################################
2477# print_grandtotals();
8e669ac1 2478#
1b4fe9dd 2479# print_grandtotals();
8e669ac1 2480#
1b4fe9dd
PH		 2481# Print the grand totals.
2482#######################################################################
059ec3d9
PH		 2483sub print_grandtotals {
2484
2485 # Get the sender by headings and results. This is complicated as we can have
2486 # different numbers of columns.
a83c7e95 2487 my($sender_txt_header,$sender_txt_format,$sender_html_format);
059ec3d9 2488 my(@received_totals,@delivered_totals);
d5692f86 2489 my($row_tablehead, $row_max);
a83c7e95 2490 my(@col_headers) = ('TOTAL', 'Volume', 'Messages', 'Addresses');
d5692f86 2491
059ec3d9
PH		 2492 foreach ('Host','Domain','Email','Edomain') {
2493 next unless $do_sender{$_};
2494 if ($merge_reports) {
2495 push(@received_totals, get_report_total($report_totals{Received},"${_}s"));
2496 push(@delivered_totals,get_report_total($report_totals{Delivered},"${_}s"));
2497 }
2498 else {
2499 push(@received_totals,scalar(keys %{$received_data{$_}}));
2500 push(@delivered_totals,scalar(keys %{$delivered_data{$_}}));
2501 }
059ec3d9
PH		 2502 $sender_txt_header .= " " x ($COLUMN_WIDTHS - length($_)) . $_ . 's';
2503 $sender_html_format .= "<td align=\"right\">%d</td>";
2504 $sender_txt_format .= " " x ($COLUMN_WIDTHS - 5) . "%6d";
a83c7e95 2505 push(@col_headers,"${_}s");
059ec3d9
PH		 2506 }
2507
a83c7e95 2508 my $txt_format1 = " %-16s %9s %6d %6s $sender_txt_format";
d5692f86 2509 my $txt_format2 = " %6d %4.1f%% %6d %4.1f%%",
a83c7e95 2510 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%s</td><td align=\"right\">%s</td><td align=\"right\">%s</td>$sender_html_format";
d5692f86 2511 my $htm_format2 = "<td align=\"right\">%d</td><td align=\"right\">%4.1f%%</td><td align=\"right\">%d</td><td align=\"right\">%4.1f%%</td>";
059ec3d9 2512
d5692f86 2513 if ($txt_fh) {
059ec3d9 2514 my $sender_spaces = " " x length($sender_txt_header);
d5692f86
SC		 2515 print $txt_fh "\n";
2516 print $txt_fh "Grand total summary\n";
2517 print $txt_fh "-------------------\n";
a83c7e95
SC		 2518 print $txt_fh " $sender_spaces At least one address\n";
2519 print $txt_fh " TOTAL Volume Messages Addresses $sender_txt_header Delayed Failed\n";
d5692f86
SC		 2520 }
2521 if ($htm_fh) {
2522 print $htm_fh "<a name=\"grandtotal\"></a>\n";
2523 print $htm_fh "<h2>Grand total summary</h2>\n";
2524 print $htm_fh "<table border=1>\n";
a83c7e95 2525 print $htm_fh "<tr><th>" . join('</th><th>',@col_headers) . "</th><th colspan=2>At least one addr<br>Delayed</th><th colspan=2>At least one addr<br>Failed</th>\n";
d5692f86 2526 }
a83c7e95
SC		 2527 if ($xls_fh) {
2528 $ws_global->write($row++, 0, "Grand total summary", $f_header2);
2529 $ws_global->write($row, 0, \@col_headers, $f_header2);
2530 $ws_global->merge_range($row, scalar(@col_headers), $row, scalar(@col_headers)+1, "At least one addr Delayed", $f_header2_m);
2531 $ws_global->merge_range($row, scalar(@col_headers)+2, $row, scalar(@col_headers)+3, "At least one addr Failed", $f_header2_m);
2532 #$ws_global->write(++$row, scalar(@col_headers), ['Total','Percent','Total','Percent'], $f_header2);
059ec3d9
PH		 2533 }
2534
d5692f86 2535
059ec3d9
PH		 2536 my($volume,$failed_count);
2537 if ($merge_reports) {
2538 $volume = volume_rounded($report_totals{Received}{Volume}, $report_totals{Received}{'Volume-gigs'});
2539 $total_received_count = get_report_total($report_totals{Received},'Messages');
2540 $failed_count = get_report_total($report_totals{Received},'Failed');
2541 $delayed_count = get_report_total($report_totals{Received},'Delayed');
2542 }
2543 else {
2544 $volume = volume_rounded($total_received_data, $total_received_data_gigs);
608bc29d 2545 $failed_count = $message_errors;
059ec3d9
PH		 2546 }
2547
2548 {
2549 no integer;
059ec3d9 2550
d5692f86 2551 my @content=(
a83c7e95 2552 $volume,$total_received_count,'',
d5692f86
SC		 2553 @received_totals,
2554 $delayed_count,
2555 ($total_received_count) ? ($delayed_count*100/$total_received_count) : 0,
2556 $failed_count,
2557 ($total_received_count) ? ($failed_count*100/$total_received_count) : 0
2558 );
2559
2560 printf $txt_fh ("$txt_format1$txt_format2\n", 'Received', @content) if $txt_fh;
2561 printf $htm_fh ("$htm_format1$htm_format2\n", 'Received', @content) if $htm_fh;
a83c7e95
SC		 2562 if ($xls_fh) {
2563 $ws_global->write(++$row, 0, 'Received', $f_default);
2564 for (my $i=0; $i < scalar(@content); $i++) {
d5692f86 2565 if ($i == 4 || $i == 6) {
a83c7e95 2566 $ws_global->write($row, $i+1, $content[$i]/100, $f_percent);
d5692f86
SC		 2567 }
2568 else {
a83c7e95 2569 $ws_global->write($row, $i+1, $content[$i], $f_default);
d5692f86
SC		 2570 }
2571 }
2572 }
2573 }
a83c7e95 2574
059ec3d9
PH		 2575 if ($merge_reports) {
2576 $volume = volume_rounded($report_totals{Delivered}{Volume}, $report_totals{Delivered}{'Volume-gigs'});
a83c7e95
SC		 2577 $total_delivered_messages = get_report_total($report_totals{Delivered},'Messages');
2578 $total_delivered_addresses = get_report_total($report_totals{Delivered},'Addresses');
059ec3d9
PH		 2579 }
2580 else {
2581 $volume = volume_rounded($total_delivered_data, $total_delivered_data_gigs);
2582 }
d5692f86 2583
a83c7e95
SC		 2584 my @content=($volume, $total_delivered_messages, $total_delivered_addresses, @delivered_totals);
2585 printf $txt_fh ("$txt_format1\n", 'Delivered', @content) if $txt_fh;
2586 printf $htm_fh ("$htm_format1\n", 'Delivered', @content) if $htm_fh;
d5692f86 2587
a83c7e95
SC		 2588 if ($xls_fh) {
2589 $ws_global->write(++$row, 0, 'Delivered', $f_default);
2590 for (my $i=0; $i < scalar(@content); $i++) {
2591 $ws_global->write($row, $i+1, $content[$i], $f_default);
2592 }
2593 }
2594
2595 if ($merge_reports) {
2596 foreach ('Rejects', 'Ham', 'Spam') {
2597 my $messages = get_report_total($report_totals{$_},'Messages');
2598 my $addresses = get_report_total($report_totals{$_},'Addresses');
2599 if ($messages) {
2600 @content = ($_, '', $messages, '');
2601 push(@content,get_report_total($report_totals{$_},'Hosts')) if $do_sender{Host};
2602 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2603 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2604 $ws_global->write(++$row, 0, \@content) if $xls_fh;
2605 }
2606 }
2607 }
2608 else {
2609 foreach my $total_aref (['Rejects',\%rejected_count_by_ip],
2610 ['Ham',\%ham_count_by_ip],
2611 ['Spam',\%spam_count_by_ip]) {
2612 my $messages = 0;
2613 map {$messages += $_} values %{$total_aref->[1]};
2614
2615 if ($messages > 0) {
2616 @content = ($total_aref->[0], '', $messages, '');
2617 push(@content,scalar(keys %{$total_aref->[1]})) if $do_sender{Host};
2618
2619 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2620 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
2621 $ws_global->write(++$row, 0, \@content) if $xls_fh;
d5692f86 2622 }
a83c7e95 2623 }
d5692f86 2624 }
a83c7e95
SC		 2625
2626 printf $txt_fh "\n" if $txt_fh;
2627 printf $htm_fh "</table>\n" if $htm_fh;
2628 ++$row;
059ec3d9
PH		 2629}
2630
2631
1b4fe9dd
PH		 2632#######################################################################
2633# print_user_patterns()
8e669ac1 2634#
1b4fe9dd 2635# print_user_patterns();
8e669ac1 2636#
1b4fe9dd
PH		 2637# Print the counts of user specified patterns.
2638#######################################################################
059ec3d9 2639sub print_user_patterns {
d5692f86
SC		 2640 my $txt_format1 = " %-18s %6d";
2641 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%d</td>";
059ec3d9 2642
d5692f86
SC		 2643 if ($txt_fh) {
2644 print $txt_fh "User Specified Patterns\n";
2645 print $txt_fh "-----------------------";
2646 print $txt_fh "\n Total\n";
059ec3d9 2647 }
d5692f86
SC		 2648 if ($htm_fh) {
2649 print $htm_fh "<hr><a name=\"patterns\"></a><h2>User Specified Patterns</h2>\n";
2650 print $htm_fh "<table border=0 width=\"100%\">\n";
2651 print $htm_fh "<tr><td>\n";
2652 print $htm_fh "<table border=1>\n";
2653 print $htm_fh "<tr><th>&nbsp;</th><th>Total</th>\n";
2654 }
2655 if ($xls_fh) {
2656 $ws_global->write($row++, $col, "User Specified Patterns", $f_header2);
2657 &set_worksheet_line($ws_global, $row++, 1, ["Total"], $f_headertab);
059ec3d9
PH		 2658 }
2659
d5692f86 2660
059ec3d9
PH		 2661 my($key);
2662 if ($merge_reports) {
2663 # We are getting our data from previous reports.
2664 foreach $key (@user_descriptions) {
2665 my $count = get_report_total($report_totals{patterns}{$key},'Total');
d5692f86
SC		 2666 printf $txt_fh ("$txt_format1\n",$key,$count) if $txt_fh;
2667 printf $htm_fh ("$htm_format1\n",$key,$count) if $htm_fh;
2668 if ($xls_fh)
2669 {
2670 &set_worksheet_line($ws_global, $row++, 0, [$key,$count], $f_default);
2671 }
059ec3d9
PH		 2672 }
2673 }
2674 else {
2675 # We are getting our data from mainlog files.
2676 my $user_pattern_index = 0;
2677 foreach $key (@user_descriptions) {
d5692f86
SC		 2678 printf $txt_fh ("$txt_format1\n",$key,$user_pattern_totals[$user_pattern_index]) if $txt_fh;
2679 printf $htm_fh ("$htm_format1\n",$key,$user_pattern_totals[$user_pattern_index]) if $htm_fh;
a83c7e95 2680 $ws_global->write($row++, 0, [$key,$user_pattern_totals[$user_pattern_index]]) if $xls_fh;
059ec3d9
PH		 2681 $user_pattern_index++;
2682 }
2683 }
d5692f86
SC		 2684 print $txt_fh "\n" if $txt_fh;
2685 print $htm_fh "</table>\n\n" if $htm_fh;
2686 if ($xls_fh)
2687 {
2688 ++$row;
059ec3d9 2689 }
608bc29d
SC		 2690
2691 if ($hist_opt > 0) {
2692 my $user_pattern_index = 0;
2693 foreach $key (@user_descriptions) {
2694 print_histogram($key, 'occurence', @{$user_pattern_interval_count[$user_pattern_index]});
2695 $user_pattern_index++;
2696 }
2697 }
059ec3d9
PH		 2698}
2699
a83c7e95
SC		 2700#######################################################################
2701# print_rejects()
2702#
2703# print_rejects();
2704#
2705# Print statistics about rejected mail.
2706#######################################################################
2707sub print_rejects {
2708 my($format1,$reason);
2709
2710 my $txt_format1 = " %-40s %6d";
2711 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%d</td>";
2712
2713 if ($txt_fh) {
2714 print $txt_fh "Rejected mail by reason\n";
2715 print $txt_fh "-----------------------";
2716 print $txt_fh "\n Total\n";
2717 }
2718 if ($htm_fh) {
2719 print $htm_fh "<hr><a name=\"patterns\"></a><h2>Rejected mail by reason</h2>\n";
2720 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
2721 print $htm_fh "<tr><th>&nbsp;</th><th>Total</th>\n";
2722 }
2723 if ($xls_fh) {
2724 $ws_global->write($row++, $col, "Rejected mail by reason", $f_header2);
2725 &set_worksheet_line($ws_global, $row++, 1, ["Total"], $f_headertab);
2726 }
2727
2728
2729 my $href = ($merge_reports) ? $report_totals{rejected_mail_by_reason} : \%rejected_count_by_reason;
2730 my(@chartdatanames, @chartdatavals_count);
2731
2732 foreach $reason (top_n_sort($topcount, $href, undef, undef)) {
2733 printf $txt_fh ("$txt_format1\n",$reason,$href->{$reason}) if $txt_fh;
2734 printf $htm_fh ("$htm_format1\n",$reason,$href->{$reason}) if $htm_fh;
2735 set_worksheet_line($ws_global, $row++, 0, [$reason,$href->{$reason}], $f_default) if $xls_fh;
2736 push(@chartdatanames, $reason);
2737 push(@chartdatavals_count, $href->{$reason});
2738 }
2739
2740 $row++ if $xls_fh;
2741 print $txt_fh "\n" if $txt_fh;
2742
2743 if ($htm_fh) {
2744 print $htm_fh "</tr></table></td><td>";
2745 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_count > 0)) {
2746 # calculate the graph
2747 my @data = (
2748 \@chartdatanames,
2749 \@chartdatavals_count
2750 );
2751 my $graph = GD::Graph::pie->new(200, 200);
2752 $graph->set(
2753 x_label => 'Rejection Reasons',
2754 y_label => 'Messages',
2755 title => 'By count',
2756 );
2757 my $gd = $graph->plot(\@data) or warn($graph->error);
2758 if ($gd) {
2759 open(IMG, ">$chartdir/rejections_count.png") or die "Could not write $chartdir/rejections_count.png: $!\n";
2760 binmode IMG;
2761 print IMG $gd->png;
2762 close IMG;
2763 print $htm_fh "<img src=\"$chartrel/rejections_count.png\">";
2764 }
2765 }
2766 print $htm_fh "</td></tr></table>\n\n";
2767 }
2768}
2769
2770
2771
2772
059ec3d9 2773
1b4fe9dd
PH		 2774#######################################################################
2775# print_transport();
8e669ac1 2776#
1b4fe9dd 2777# print_transport();
8e669ac1 2778#
1b4fe9dd
PH		 2779# Print totals by transport.
2780#######################################################################
059ec3d9 2781sub print_transport {
059ec3d9
PH		 2782 my(@chartdatanames);
2783 my(@chartdatavals_count);
2784 my(@chartdatavals_vol);
d5692f86
SC		 2785 no integer; #Lose this for charting the data.
2786
2787 my $txt_format1 = " %-18s %6s %6d";
2788 my $htm_format1 = "<tr><td>%s</td><td align=\"right\">%s</td><td align=\"right\">%d</td>";
059ec3d9 2789
d5692f86
SC		 2790 if ($txt_fh) {
2791 print $txt_fh "Deliveries by transport\n";
2792 print $txt_fh "-----------------------";
2793 print $txt_fh "\n Volume Messages\n";
059ec3d9 2794 }
d5692f86
SC		 2795 if ($htm_fh) {
2796 print $htm_fh "<hr><a name=\"transport\"></a><h2>Deliveries by Transport</h2>\n";
a83c7e95 2797 print $htm_fh "<table border=0 width=\"100%\"><tr><td><table border=1>\n";
d5692f86
SC		 2798 print $htm_fh "<tr><th>&nbsp;</th><th>Volume</th><th>Messages</th>\n";
2799 }
2800 if ($xls_fh) {
a83c7e95
SC		 2801 $ws_global->write(++$row, $col, "Deliveries by transport", $f_header2);
2802 $ws_global->write(++$row, 1, ["Volume", "Messages"], $f_headertab);
059ec3d9
PH		 2803 }
2804
2805 my($key);
2806 if ($merge_reports) {
2807 # We are getting our data from previous reports.
2808 foreach $key (sort keys %{$report_totals{transport}}) {
2809 my $count = get_report_total($report_totals{transport}{$key},'Messages');
d5692f86
SC		 2810 my @content=($key, volume_rounded($report_totals{transport}{$key}{Volume},
2811 $report_totals{transport}{$key}{'Volume-gigs'}), $count);
059ec3d9
PH		 2812 push(@chartdatanames, $key);
2813 push(@chartdatavals_count, $count);
2814 push(@chartdatavals_vol, $report_totals{transport}{$key}{'Volume-gigs'}*$gig + $report_totals{transport}{$key}{Volume} );
d5692f86
SC		 2815 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2816 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
a83c7e95 2817 $ws_global->write(++$row, 0, \@content) if $xls_fh;
059ec3d9
PH		 2818 }
2819 }
2820 else {
2821 # We are getting our data from mainlog files.
2822 foreach $key (sort keys %transported_data) {
d5692f86
SC		 2823 my @content=($key, volume_rounded($transported_data{$key},$transported_data_gigs{$key}),
2824 $transported_count{$key});
059ec3d9
PH		 2825 push(@chartdatanames, $key);
2826 push(@chartdatavals_count, $transported_count{$key});
2827 push(@chartdatavals_vol, $transported_data_gigs{$key}*$gig + $transported_data{$key});
d5692f86
SC		 2828 printf $txt_fh ("$txt_format1\n", @content) if $txt_fh;
2829 printf $htm_fh ("$htm_format1\n", @content) if $htm_fh;
a83c7e95 2830 $ws_global->write(++$row, 0, \@content) if $xls_fh;
059ec3d9
PH		 2831 }
2832 }
d5692f86
SC		 2833 print $txt_fh "\n" if $txt_fh;
2834 if ($htm_fh) {
a83c7e95
SC		 2835 print $htm_fh "</tr></table></td><td>";
2836
608bc29d 2837 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_count > 0))
059ec3d9
PH		 2838 {
2839 # calculate the graph
2840 my @data = (
2841 \@chartdatanames,
2842 \@chartdatavals_count
2843 );
2844 my $graph = GD::Graph::pie->new(200, 200);
2845 $graph->set(
2846 x_label => 'Transport',
2847 y_label => 'Messages',
2848 title => 'By count',
2849 );
2850 my $gd = $graph->plot(\@data) or warn($graph->error);
2851 if ($gd) {
d5692f86
SC		 2852 open(IMG, ">$chartdir/transports_count.png") or die "Could not write $chartdir/transports_count.png: $!\n";
2853 binmode IMG;
2854 print IMG $gd->png;
2855 close IMG;
2856 print $htm_fh "<img src=\"$chartrel/transports_count.png\">";
059ec3d9
PH		 2857 }
2858 }
a83c7e95 2859 print $htm_fh "</td><td>";
059ec3d9 2860
608bc29d 2861 if ($HAVE_GD_Graph_pie && $charts && ($#chartdatavals_vol > 0)) {
059ec3d9
PH		 2862 my @data = (
2863 \@chartdatanames,
2864 \@chartdatavals_vol
2865 );
2866 my $graph = GD::Graph::pie->new(200, 200);
2867 $graph->set(
2868 title => 'By volume',
2869 );
2870 my $gd = $graph->plot(\@data) or warn($graph->error);
2871 if ($gd) {
a83c7e95 2872 open(IMG, ">$chartdir/transports_vol.png") or die "Could not write $chartdir/transports_vol.png: $!\n";
d5692f86
SC		 2873 binmode IMG;
2874 print IMG $gd->png;
2875 close IMG;
2876 print $htm_fh "<img src=\"$chartrel/transports_vol.png\">";
059ec3d9
PH		 2877 }
2878 }
a83c7e95 2879
d5692f86
SC		 2880 print $htm_fh "</td></tr></table>\n\n";
2881 }
059ec3d9
PH		 2882}
2883
2884
2885
1b4fe9dd
PH		 2886#######################################################################
2887# print_relay();
8e669ac1 2888#
1b4fe9dd 2889# print_relay();
8e669ac1 2890#
1b4fe9dd
PH		 2891# Print our totals by relay.
2892#######################################################################
059ec3d9 2893sub print_relay {
d5692f86 2894 my $row_print_relay=1;
059ec3d9 2895 my $temp = "Relayed messages";
d5692f86 2896 print $htm_fh "<hr><a name=\"$temp\"></a><h2>$temp</h2>\n" if $htm_fh;
059ec3d9
PH		 2897 if (scalar(keys %relayed) > 0 || $relayed_unshown > 0) {
2898 my $shown = 0;
2899 my $spacing = "";
d5692f86
SC		 2900 my $txt_format = "%7d %s\n => %s\n";
2901 my $htm_format = "<tr><td align=\"right\">%d</td><td>%s</td><td>%s</td>\n";
059ec3d9 2902
d5692f86
SC		 2903 printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
2904 if ($htm_fh) {
2905 print $htm_fh "<table border=1>\n";
2906 print $htm_fh "<tr><th>Count</th><th>From</th><th>To</th>\n";
059ec3d9 2907 }
d5692f86
SC		 2908 if ($xls_fh) {
2909 $ws_relayed->write($row_print_relay++, $col, $temp, $f_header2);
2910 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, ["Count", "From", "To"], $f_headertab);
059ec3d9
PH		 2911 }
2912
d5692f86 2913
059ec3d9
PH		 2914 my($key);
2915 foreach $key (sort keys %relayed) {
2916 my $count = $relayed{$key};
2917 $shown += $count;
2918 $key =~ s/[HA]=//g;
2919 my($one,$two) = split(/=> /, $key);
d5692f86
SC		 2920 my @content=($count, $one, $two);
2921 printf $txt_fh ($txt_format, @content) if $txt_fh;
2922 printf $htm_fh ($htm_format, @content) if $htm_fh;
2923 if ($xls_fh)
2924 {
2925 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, \@content);
2926 }
059ec3d9
PH		 2927 $spacing = "\n";
2928 }
d5692f86
SC		 2929
2930 print $htm_fh "</table>\n<p>\n" if $htm_fh;
2931 print $txt_fh "${spacing}Total: $shown (plus $relayed_unshown unshown)\n\n" if $txt_fh;
2932 print $htm_fh "${spacing}Total: $shown (plus $relayed_unshown unshown)\n\n" if $htm_fh;
2933 if ($xls_fh)
2934 {
2935 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, [$shown, "Sum of shown" ]);
2936 &set_worksheet_line($ws_relayed, $row_print_relay++, 0, [$relayed_unshown, "unshown"]);
2937 $row_print_relay++;
2938 }
059ec3d9
PH		 2939 }
2940 else {
d5692f86
SC		 2941 print $txt_fh "No relayed messages\n-------------------\n\n" if $txt_fh;
2942 print $htm_fh "No relayed messages\n\n" if $htm_fh;
2943 if ($xls_fh)
2944 {
2945 $row_print_relay++;
2946 }
059ec3d9 2947 }
059ec3d9
PH		 2948}
2949
2950
2951
1b4fe9dd
PH		 2952#######################################################################
2953# print_errors();
8e669ac1 2954#
1b4fe9dd 2955# print_errors();
8e669ac1 2956#
1b4fe9dd
PH		 2957# Print our errors. In HTML, we display them as a list rather than a table -
2958# Netscape doesn't like large tables!
2959#######################################################################
059ec3d9
PH		 2960sub print_errors {
2961 my $total_errors = 0;
d5692f86 2962 $row=1;
059ec3d9
PH		 2963
2964 if (scalar(keys %errors_count) != 0) {
2965 my $temp = "List of errors";
d5692f86
SC		 2966 my $htm_format = "<li>%d - %s\n";
2967
2968 printf $txt_fh ("%s\n%s\n\n", $temp, "-" x length($temp)) if $txt_fh;
2969 if ($htm_fh) {
2970 print $htm_fh "<hr><a name=\"errors\"></a><h2>$temp</h2>\n";
2971 print $htm_fh "<ul><li><b>Count - Error</b>\n";
059ec3d9 2972 }
d5692f86
SC		 2973 if ($xls_fh)
2974 {
2975 $ws_errors->write($row++, 0, $temp, $f_header2);
2976 &set_worksheet_line($ws_errors, $row++, 0, ["Count", "Error"], $f_headertab);
059ec3d9
PH		 2977 }
2978
d5692f86 2979
059ec3d9
PH		 2980 my($key);
2981 foreach $key (sort keys %errors_count) {
2982 my $text = $key;
2983 chomp($text);
d5692f86 2984 $text =~ s/\s\s+/ /g; #Convert multiple spaces to a single space.
059ec3d9 2985 $total_errors += $errors_count{$key};
d5692f86
SC		 2986
2987 if ($txt_fh) {
2988 printf $txt_fh ("%5d ", $errors_count{$key});
2989 my $text_remaining = $text;
2990 while (length($text_remaining) > 65) {
2991 my($first,$rest) = $text_remaining =~ /(.{50}\S*)\s+(.+)/;
2992 last if !$first;
2993 printf $txt_fh ("%s\n\t ", $first);
2994 $text_remaining = $rest;
2995 }
2996 printf $txt_fh ("%s\n\n", $text_remaining);
2997 }
2998
2999 if ($htm_fh) {
8e669ac1 3000
059ec3d9
PH		 3001 #Translate HTML tag characters. Sergey Sholokh.
3002 $text =~ s/\</\&lt\;/g;
3003 $text =~ s/\>/\&gt\;/g;
3004
d5692f86 3005 printf $htm_fh ($htm_format,$errors_count{$key},$text);
059ec3d9 3006 }
d5692f86
SC		 3007 if ($xls_fh)
3008 {
3009 &set_worksheet_line($ws_errors, $row++, 0, [$errors_count{$key},$text]);
059ec3d9
PH		 3010 }
3011 }
059ec3d9
PH		 3012
3013 $temp = "Errors encountered: $total_errors";
d5692f86
SC		 3014
3015 if ($txt_fh) {
3016 print $txt_fh $temp, "\n";
3017 print $txt_fh "-" x length($temp),"\n";
3018 }
3019 if ($htm_fh) {
3020 print $htm_fh "</ul>\n<p>\n";
3021 print $htm_fh $temp, "\n";
3022 }
3023 if ($xls_fh)
3024 {
3025 &set_worksheet_line($ws_errors, $row++, 0, [$total_errors, "Sum of Errors encountered"]);
3026 }
059ec3d9
PH		 3027 }
3028
3029}
3030
3031
1b4fe9dd
PH		 3032#######################################################################
3033# parse_old_eximstat_reports();
8e669ac1 3034#
1b4fe9dd 3035# parse_old_eximstat_reports($fh);
8e669ac1 3036#
1b4fe9dd 3037# Parse old eximstat output so we can merge daily stats to weekly stats and weekly to monthly etc.
8e669ac1 3038#
1b4fe9dd
PH		 3039# To test that the merging still works after changes, do something like the following.
3040# All the diffs should produce no output.
8e669ac1 3041#
1b4fe9dd 3042# options='-bydomain -byemail -byhost -byedomain'
608bc29d 3043# options="$options -show_rt1,2,4 -show_dt 1,2,4"
1b4fe9dd
PH		 3044# options="$options -pattern 'Completed Messages' /Completed/"
3045# options="$options -pattern 'Received Messages' /<=/"
8e669ac1 3046#
1b4fe9dd
PH		 3047# ./eximstats $options mainlog > mainlog.txt
3048# ./eximstats $options -merge mainlog.txt > mainlog.2.txt
3049# diff mainlog.txt mainlog.2.txt
8e669ac1 3050#
1b4fe9dd
PH		 3051# ./eximstats $options -html mainlog > mainlog.html
3052# ./eximstats $options -merge -html mainlog.txt > mainlog.2.html
3053# diff mainlog.html mainlog.2.html
8e669ac1 3054#
1b4fe9dd
PH		 3055# ./eximstats $options -merge mainlog.html > mainlog.3.txt
3056# diff mainlog.txt mainlog.3.txt
8e669ac1 3057#
1b4fe9dd
PH		 3058# ./eximstats $options -merge -html mainlog.html > mainlog.3.html
3059# diff mainlog.html mainlog.3.html
8e669ac1 3060#
1b4fe9dd
PH		 3061# ./eximstats $options -nvr mainlog > mainlog.nvr.txt
3062# ./eximstats $options -merge mainlog.nvr.txt > mainlog.4.txt
3063# diff mainlog.txt mainlog.4.txt
8e669ac1 3064#
1b4fe9dd
PH		 3065# # double_mainlog.txt should have twice the values that mainlog.txt has.
3066# ./eximstats $options mainlog mainlog > double_mainlog.txt
3067#######################################################################
059ec3d9
PH		 3068sub parse_old_eximstat_reports {
3069 my($fh) = @_;
3070
3071 my(%league_table_value_entered, %league_table_value_was_zero, %table_order);
3072
608bc29d
SC		 3073 my(%user_pattern_index);
3074 my $user_pattern_index = 0;
3075 map {$user_pattern_index{$_} = $user_pattern_index++} @user_descriptions;
3076 my $user_pattern_keys = join('|', @user_descriptions);
3077
059ec3d9 3078 while (<$fh>) {
d5692f86 3079 PARSE_OLD_REPORT_LINE:
059ec3d9
PH		 3080 if (/Exim statistics from ([\d\-]+ [\d:]+(\s+[\+\-]\d+)?) to ([\d\-]+ [\d:]+(\s+[\+\-]\d+)?)/) {
3081 $begin = $1 if ($1 lt $begin);
3082 $end = $3 if ($3 gt $end);
3083 }
3084 elsif (/Grand total summary/) {
a83c7e95
SC		 3085 # Fill in $report_totals{Received|Delivered}{Volume|Messages|Addresses|Hosts|Domains|...|Delayed|DelayedPercent|Failed|FailedPercent}
3086 my(@fields, @delivered_fields);
3087 my $doing_table = 0;
059ec3d9 3088 while (<$fh>) {
d5692f86
SC		 3089 $_ = html2txt($_); #Convert general HTML markup to text.
3090 s/At least one addr//g; #Another part of the HTML output we don't want.
059ec3d9 3091
a83c7e95
SC		 3092# TOTAL Volume Messages Addresses Hosts Domains Delayed Failed
3093# Received 26MB 237 177 23 8 3.4% 28 11.8%
3094# Delivered 13MB 233 250 99 88
d5692f86 3095 if (/TOTAL\s+(.*?)\s*$/) {
a83c7e95
SC		 3096 $doing_table = 1;
3097 @delivered_fields = split(/\s+/,$1);
3098
059ec3d9 3099 #Delayed and Failed have two columns each, so add the extra field names in.
a83c7e95
SC		 3100 splice(@delivered_fields,-1,1,'DelayedPercent','Failed','FailedPercent');
3101
3102 # Addresses only figure in the Delivered row, so remove them from the
3103 # normal fields.
3104 @fields = grep !/Addresses/, @delivered_fields;
d5692f86 3105 }
a83c7e95 3106 elsif (/(Received)\s+(.*?)\s*$/) {
d5692f86
SC		 3107 print STDERR "Parsing $_" if $debug;
3108 add_to_totals($report_totals{$1},\@fields,$2);
3109 }
a83c7e95
SC		 3110 elsif (/(Delivered)\s+(.*?)\s*$/) {
3111 print STDERR "Parsing $_" if $debug;
3112 add_to_totals($report_totals{$1},\@delivered_fields,$2);
3113 my $data = $2;
3114 # If we're merging an old report which doesn't include addresses,
3115 # then use the Messages field instead.
3116 unless (grep(/Addresses/, @delivered_fields)) {
3117 my %tmp;
3118 line_to_hash(\%tmp,\@delivered_fields,$data);
3119 add_to_totals($report_totals{Delivered},['Addresses'],$tmp{Messages});
3120 }
3121 }
3122 elsif (/(Rejects|Ham|Spam)\s+(.*?)\s*$/) {
3123 print STDERR "Parsing $_" if $debug;
3124 add_to_totals($report_totals{$1},['Messages','Hosts'],$2);
3125 }
3126 else {
3127 last if $doing_table;
3128 }
059ec3d9
PH		 3129 }
3130 }
3131
3132 elsif (/User Specified Patterns/i) {
3133#User Specified Patterns
3134#-----------------------
3135# Total
3136# Description 85
3137
d5692f86 3138 while (<$fh>) { last if (/Total/); } #Wait until we get the table headers.
059ec3d9 3139 while (<$fh>) {
d5692f86
SC		 3140 print STDERR "Parsing $_" if $debug;
3141 $_ = html2txt($_); #Convert general HTML markup to text.
3142 if (/^\s*(.*?)\s+(\d+)\s*$/) {
3143 $report_totals{patterns}{$1} = {} unless (defined $report_totals{patterns}{$1});
3144 add_to_totals($report_totals{patterns}{$1},['Total'],$2);
3145 }
3146 last if (/^\s*$/); #Finished if we have a blank line.
059ec3d9
PH		 3147 }
3148 }
3149
608bc29d
SC		 3150 elsif (/(^|<h2>)($user_pattern_keys) per /o) {
3151 # Parse User defined pattern histograms if they exist.
3152 parse_histogram($fh, $user_pattern_interval_count[$user_pattern_index{$2}] );
3153 }
3154
3155
059ec3d9
PH		 3156 elsif (/Deliveries by transport/i) {
3157#Deliveries by transport
3158#-----------------------
3159# Volume Messages
3160# :blackhole: 70KB 51
3161# address_pipe 655KB 1
3162# smtp 11MB 151
3163
d5692f86 3164 while (<$fh>) { last if (/Volume/); } #Wait until we get the table headers.
059ec3d9 3165 while (<$fh>) {
d5692f86
SC		 3166 print STDERR "Parsing $_" if $debug;
3167 $_ = html2txt($_); #Convert general HTML markup to text.
3168 if (/(\S+)\s+(\d+\S*\s+\d+)/) {
3169 $report_totals{transport}{$1} = {} unless (defined $report_totals{transport}{$1});
3170 add_to_totals($report_totals{transport}{$1},['Volume','Messages'],$2);
3171 }
3172 last if (/^\s*$/); #Finished if we have a blank line.
059ec3d9
PH		 3173 }
3174 }
608bc29d
SC		 3175 elsif (/Messages received per/) {
3176 parse_histogram($fh, \@received_interval_count);
3177 }
3178 elsif (/Deliveries per/) {
3179 parse_histogram($fh, \@delivered_interval_count);
059ec3d9
PH		 3180 }
3181
608bc29d
SC		 3182 #elsif (/Time spent on the queue: (all messages|messages with at least one remote delivery)/) {
3183 elsif (/(Time spent on the queue|Delivery times|Receipt times): ((\S+) messages|messages with at least one remote delivery)((<[^>]*>)*\s*)$/) {
059ec3d9
PH		 3184#Time spent on the queue: all messages
3185#-------------------------------------
3186#
3187#Under 1m 217 91.9% 91.9%
3188# 5m 2 0.8% 92.8%
3189# 3h 8 3.4% 96.2%
3190# 6h 7 3.0% 99.2%
3191# 12h 2 0.8% 100.0%
3192
3193 # Set a pointer to the queue bin so we can use the same code
3194 # block for both all messages and remote deliveries.
608bc29d
SC		 3195 #my $bin_aref = ($1 eq 'all messages') ? \@qt_all_bin : \@qt_remote_bin;
3196 my($bin_aref, $times_aref, $overflow_sref);
3197 if ($1 eq 'Time spent on the queue') {
3198 $times_aref = \@queue_times;
3199 if ($2 eq 'all messages') {
3200 $bin_aref = \@qt_all_bin;
3201 $overflow_sref = \$qt_all_overflow;
3202 }
3203 else {
3204 $bin_aref = \@qt_remote_bin;
3205 $overflow_sref = \$qt_remote_overflow;
3206 }
3207 }
3208 elsif ($1 eq 'Delivery times') {
3209 $times_aref = \@delivery_times;
3210 if ($2 eq 'all messages') {
3211 $bin_aref = \@dt_all_bin;
3212 $overflow_sref = \$dt_all_overflow;
3213 }
3214 else {
3215 $bin_aref = \@dt_remote_bin;
3216 $overflow_sref = \$dt_remote_overflow;
3217 }
3218 }
3219 else {
3220 unless (exists $rcpt_times_bin{$3}) {
3221 initialise_rcpt_times($3);
3222 }
3223 $bin_aref = $rcpt_times_bin{$3};
3224 $times_aref = \@rcpt_times;
3225 $overflow_sref = \$rcpt_times_overflow{$3};
3226 }
3227
3228
a83c7e95 3229 my ($blank_lines, $reached_table) = (0,0);
059ec3d9 3230 while (<$fh>) {
d5692f86 3231 $_ = html2txt($_); #Convert general HTML markup to text.
a83c7e95
SC		 3232 # The table is preceded by one blank line, and has one blank line
3233 # following it. As the table may be empty, the best way to determine
3234 # that we've finished it is to look for the second blank line.
3235 ++$blank_lines if /^\s*$/;
3236 last if ($blank_lines >=2); #Finished the table ?
3237 $reached_table = 1 if (/\d/);
d5692f86
SC		 3238 next unless $reached_table;
3239 my $previous_seconds_on_queue = 0;
3240 if (/^\s*(Under|Over|)\s+(\d+[smhdw])\s+(\d+)/) {
3241 print STDERR "Parsing $_" if $debug;
3242 my($modifier,$formated_time,$count) = ($1,$2,$3);
3243 my $seconds = unformat_time($formated_time);
3244 my $time_on_queue = ($seconds + $previous_seconds_on_queue) / 2;
3245 $previous_seconds_on_queue = $seconds;
3246 $time_on_queue = $seconds * 2 if ($modifier eq 'Over');
3247 my($i);
608bc29d
SC		 3248 for ($i = 0; $i <= $#$times_aref; $i++) {
3249 if ($time_on_queue < $times_aref->[$i]) {
d5692f86
SC		 3250 $$bin_aref[$i] += $count;
3251 last;
3252 }
3253 }
608bc29d
SC		 3254 $$overflow_sref += $count if ($i > $#$times_aref);
3255
d5692f86 3256 }
059ec3d9
PH		 3257 }
3258 }
3259
3260 elsif (/Relayed messages/) {
3261#Relayed messages
3262#----------------
3263#
3264# 1 addr.domain.com [1.2.3.4] a.user@domain.com
3265# => addr2.domain2.com [5.6.7.8] a2.user2@domain2.com
3266#
3267#<tr><td align="right">1</td><td>addr.domain.com [1.2.3.4] a.user@domain.com </td><td>addr2.domain2.com [5.6.7.8] a2.user2@domain2.com</td>
3268
3269 my $reached_table = 0;
3270 my($count,$sender);
3271 while (<$fh>) {
d5692f86
SC		 3272 unless ($reached_table) {
3273 last if (/No relayed messages/);
3274 $reached_table = 1 if (/^\s*\d/ || />\d+</);
3275 next unless $reached_table;
3276 }
3277 if (/>(\d+)<.td><td>(.*?) ?<.td><td>(.*?)</) {
3278 update_relayed($1,$2,$3);
3279 }
3280 elsif (/^\s*(\d+)\s+(.*?)\s*$/) {
3281 ($count,$sender) = ($1,$2);
3282 }
3283 elsif (/=>\s+(.*?)\s*$/) {
3284 update_relayed($count,$sender,$1);
3285 }
3286 else {
3287 last; #Finished the table ?
3288 }
059ec3d9
PH		 3289 }
3290 }
3291
3292 elsif (/Top (.*?) by (message count|volume)/) {
3293#Top 50 sending hosts by message count
3294#-------------------------------------
3295#
3296# 48 1468KB local
50adf73a
SC		 3297# Could also have average values for HTML output.
3298# 48 1468KB 30KB local
3299
059ec3d9
PH		 3300 my($category,$by_count_or_volume) = ($1,$2);
3301
3302 #As we show 2 views of each table (by count and by volume),
3303 #most (but not all) entries will appear in both tables.
3304 #Set up a hash to record which entries we have already seen
3305 #and one to record which ones we are seeing for the first time.
3306 if ($by_count_or_volume =~ /count/) {
d5692f86
SC		 3307 undef %league_table_value_entered;
3308 undef %league_table_value_was_zero;
3309 undef %table_order;
059ec3d9
PH		 3310 }
3311
3312 #As this section processes multiple different table categories,
3313 #set up pointers to the hashes to be updated.
a83c7e95 3314 my($messages_href,$addresses_href,$data_href,$data_gigs_href);
059ec3d9 3315 if ($category =~ /local sender/) {
a83c7e95
SC		 3316 $messages_href = \%received_count_user;
3317 $addresses_href = undef;
d5692f86
SC		 3318 $data_href = \%received_data_user;
3319 $data_gigs_href = \%received_data_gigs_user;
059ec3d9
PH		 3320 }
3321 elsif ($category =~ /sending (\S+?)s?\b/) {
3322 #Top 50 sending (host|domain|email|edomain)s
3323 #Top sending (host|domain|email|edomain)
a83c7e95 3324 $messages_href = \%{$received_count{"\u$1"}};
d5692f86
SC		 3325 $data_href = \%{$received_data{"\u$1"}};
3326 $data_gigs_href = \%{$received_data_gigs{"\u$1"}};
059ec3d9
PH		 3327 }
3328 elsif ($category =~ /local destination/) {
a83c7e95
SC		 3329 $messages_href = \%delivered_messages_user;
3330 $addresses_href = \%delivered_addresses_user;
d5692f86
SC		 3331 $data_href = \%delivered_data_user;
3332 $data_gigs_href = \%delivered_data_gigs_user;
059ec3d9
PH		 3333 }
3334 elsif ($category =~ /(\S+) destination/) {
3335 #Top 50 (host|domain|email|edomain) destinations
3336 #Top (host|domain|email|edomain) destination
a83c7e95
SC		 3337 $messages_href = \%{$delivered_messages{"\u$1"}};
3338 $addresses_href = \%{$delivered_addresses{"\u$1"}};
d5692f86
SC		 3339 $data_href = \%{$delivered_data{"\u$1"}};
3340 $data_gigs_href = \%{$delivered_data_gigs{"\u$1"}};
059ec3d9 3341 }
a83c7e95
SC		 3342 elsif ($category =~ /rejected ips/) {
3343 $messages_href = \%rejected_count_by_ip;
3344 }
3345 elsif ($category =~ /non-rejected spamming ips/) {
3346 $messages_href = \%spam_count_by_ip;
3347 }
3348 elsif ($category =~ /mail rejection reasons/) {
3349 $messages_href = \%rejected_count_by_reason;
3350 }
059ec3d9
PH		 3351
3352 my $reached_table = 0;
a83c7e95 3353 my $row_re;
059ec3d9 3354 while (<$fh>) {
d5692f86 3355 # Watch out for empty tables.
a83c7e95 3356 goto PARSE_OLD_REPORT_LINE if (/<h2>/ or (/^\s*[a-zA-Z]/ && !/^\s*Messages/));
d5692f86
SC		 3357
3358 $_ = html2txt($_); #Convert general HTML markup to text.
3359
a83c7e95
SC		 3360 # Messages Addresses Bytes Average
3361 if (/^\s*Messages/) {
3362 my $pattern = '^\s*(\d+)';
3363 $pattern .= (/Addresses/) ? '\s+(\d+)' : '()';
3364 $pattern .= (/Bytes/) ? '\s+([\dKMGB]+)' : '()';
3365 $pattern .= (/Average/) ? '\s+[\dKMGB]+' : '';
3366 $pattern .= '\s+(.*?)\s*$';
3367 $row_re = qr/$pattern/;
3368 $reached_table = 1;
3369 next;
3370 }
d5692f86 3371 next unless $reached_table;
50adf73a 3372
a83c7e95 3373 my($messages, $addresses, $rounded_volume, $entry);
059ec3d9 3374
a83c7e95
SC		 3375 if (/$row_re/) {
3376 ($messages, $addresses, $rounded_volume, $entry) = ($1, $2, $3, $4);
d5692f86 3377 }
a83c7e95
SC		 3378 else {
3379 #Else we have finished the table and we may need to do some
3380 #kludging to retain the order of the entries.
3381
d5692f86
SC		 3382 if ($by_count_or_volume =~ /volume/) {
3383 #Add a few bytes to appropriate entries to preserve the order.
d5692f86
SC		 3384 foreach $rounded_volume (keys %table_order) {
3385 #For each rounded volume, we want to create a list which has things
3386 #ordered from the volume table at the front, and additional things
3387 #from the count table ordered at the back.
3388 @{$table_order{$rounded_volume}{volume}} = () unless defined $table_order{$rounded_volume}{volume};
3389 @{$table_order{$rounded_volume}{'message count'}} = () unless defined $table_order{$rounded_volume}{'message count'};
3390 my(@order,%mark);
3391 map {$mark{$_} = 1} @{$table_order{$rounded_volume}{volume}};
3392 @order = @{$table_order{$rounded_volume}{volume}};
3393 map {push(@order,$_)} grep(!$mark{$_},@{$table_order{$rounded_volume}{'message count'}});
3394
3395 my $bonus_bytes = $#order;
3396 $bonus_bytes = 511 if ($bonus_bytes > 511); #Don't go over the half-K boundary!
3397 while (@order and ($bonus_bytes > 0)) {
3398 my $entry = shift(@order);
3399 if ($league_table_value_was_zero{$entry}) {
3400 $$data_href{$entry} += $bonus_bytes;
3401 print STDERR "$category by $by_count_or_volume: added $bonus_bytes bonus bytes to $entry\n" if $debug;
3402 }
3403 $bonus_bytes--;
3404 }
3405 }
3406 }
d5692f86
SC		 3407 last;
3408 }
a83c7e95
SC		 3409
3410 # Store a new table entry.
3411
3412 # Add the entry into the %table_order hash if it has a rounded
3413 # volume (KB/MB/GB).
3414 push(@{$table_order{$rounded_volume}{$by_count_or_volume}},$entry) if ($rounded_volume =~ /\D/);
3415
3416 unless ($league_table_value_entered{$entry}) {
3417 $league_table_value_entered{$entry} = 1;
3418 unless ($$messages_href{$entry}) {
3419 $$messages_href{$entry} = 0;
3420 $$addresses_href{$entry} = 0;
3421 $$data_href{$entry} = 0;
3422 $$data_gigs_href{$entry} = 0;
3423 $league_table_value_was_zero{$entry} = 1;
3424 }
3425
3426 $$messages_href{$entry} += $messages;
3427
3428 # When adding the addresses, be aware that we could be merging
3429 # an old report which does not include addresses. In this case,
3430 # we add the messages instead.
3431 $$addresses_href{$entry} += ($addresses) ? $addresses : $messages;
3432
3433 #Add the rounded value to the data and data_gigs hashes.
3434 un_round($rounded_volume,\$$data_href{$entry},\$$data_gigs_href{$entry}) if $rounded_volume;
3435 print STDERR "$category by $by_count_or_volume: added $messages,$rounded_volume to $entry\n" if $debug;
3436 }
3437
059ec3d9
PH		 3438 }
3439 }
3440 elsif (/List of errors/) {
3441#List of errors
3442#--------------
3443#
3444# 1 07904931641@one2one.net R=external T=smtp: SMTP error
3445# from remote mailer after RCPT TO:<07904931641@one2one.net>:
3446# host mail.one2one.net [193.133.192.24]: 550 User unknown
3447#
3448#<li>1 - ally.dufc@dunbar.org.uk R=external T=smtp: SMTP error from remote mailer after RCPT TO:<ally.dufc@dunbar.org.uk>: host mail.dunbar.org.uk [216.167.89.88]: 550 Unknown local part ally.dufc in <ally.dufc@dunbar.org.uk>
3449
3450
3451 my $reached_table = 0;
3452 my($count,$error,$blanks);
3453 while (<$fh>) {
d5692f86
SC		 3454 $reached_table = 1 if (/^( *|<li>)(\d+)/);
3455 next unless $reached_table;
059ec3d9 3456
d5692f86
SC		 3457 s/^<li>(\d+) -/$1/; #Convert an HTML line to a text line.
3458 $_ = html2txt($_); #Convert general HTML markup to text.
059ec3d9 3459
d5692f86
SC		 3460 if (/\t\s*(.*)/) {
3461 $error .= ' ' . $1; #Join a multiline error.
3462 }
3463 elsif (/^\s*(\d+)\s+(.*)/) {
3464 if ($error) {
059ec3d9 3465 #Finished with a previous multiline error so save it.
d5692f86
SC		 3466 $errors_count{$error} = 0 unless $errors_count{$error};
3467 $errors_count{$error} += $count;
3468 }
3469 ($count,$error) = ($1,$2);
3470 }
3471 elsif (/Errors encountered/) {
3472 if ($error) {
059ec3d9 3473 #Finished the section, so save our stored last error.
d5692f86
SC		 3474 $errors_count{$error} = 0 unless $errors_count{$error};
3475 $errors_count{$error} += $count;
3476 }
3477 last;
3478 }
059ec3d9
PH		 3479 }
3480 }
3481
3482 }
3483}
3484
608bc29d
SC		 3485#######################################################################
3486# parse_histogram($fh, \@delivered_interval_count);
3487# Parse a histogram into the provided array of counters.
3488#######################################################################
3489sub parse_histogram {
3490 my($fh, $counters_aref) = @_;
3491
3492 # Messages received per hour (each dot is 2 messages)
3493 #---------------------------------------------------
3494 #
3495 #00-01 106 .....................................................
3496 #01-02 103 ...................................................
3497
3498 my $reached_table = 0;
3499 while (<$fh>) {
3500 $reached_table = 1 if (/^00/);
3501 next unless $reached_table;
3502 print STDERR "Parsing $_" if $debug;
3503 if (/^(\d+):(\d+)\s+(\d+)/) { #hh:mm start time format ?
3504 $$counters_aref[($1*60 + $2)/$hist_interval] += $3 if $hist_opt;
3505 }
3506 elsif (/^(\d+)-(\d+)\s+(\d+)/) { #hh-hh start-end time format ?
3507 $$counters_aref[($1*60)/$hist_interval] += $3 if $hist_opt;
3508 }
3509 else { #Finished the table ?
3510 last;
3511 }
3512 }
3513}
059ec3d9
PH		 3514
3515
1b4fe9dd
PH		 3516#######################################################################
3517# update_relayed();
8e669ac1 3518#
1b4fe9dd 3519# update_relayed($count,$sender,$recipient);
8e669ac1 3520#
1b4fe9dd
PH		 3521# Adds an entry into the %relayed hash. Currently only used when
3522# merging reports.
3523#######################################################################
059ec3d9
PH		 3524sub update_relayed {
3525 my($count,$sender,$recipient) = @_;
3526
3527 #When generating the key, put in the 'H=' and 'A=' which can be used
3528 #in searches.
3529 my $key = "H=$sender => H=$recipient";
3530 $key =~ s/ ([^=\s]+\@\S+|<>)/ A=$1/g;
3531 if (!defined $relay_pattern || $key !~ /$relay_pattern/o) {
3532 $relayed{$key} = 0 if !defined $relayed{$key};
3533 $relayed{$key} += $count;
3534 }
3535 else {
3536 $relayed_unshown += $count;
3537 }
3538}
3539
3540
1b4fe9dd
PH		 3541#######################################################################
3542# add_to_totals();
8e669ac1 3543#
1b4fe9dd 3544# add_to_totals(\%totals,\@keys,$values);
8e669ac1 3545#
1b4fe9dd
PH		 3546# Given a line of space seperated values, add them into the provided hash using @keys
3547# as the hash keys.
8e669ac1 3548#
1b4fe9dd
PH		 3549# If the value contains a '%', then the value is set rather than added. Otherwise, we
3550# convert the value to bytes and gigs. The gigs get added to I<Key>-gigs.
3551#######################################################################
059ec3d9
PH		 3552sub add_to_totals {
3553 my($totals_href,$keys_aref,$values) = @_;
3554 my(@values) = split(/\s+/,$values);
a83c7e95
SC		 3555
3556 for(my $i = 0; $i < @values && $i < @$keys_aref; ++$i) {
3557 my $key = $keys_aref->[$i];
3558 if ($values[$i] =~ /%/) {
3559 $$totals_href{$key} = $values[$i];
059ec3d9
PH		 3560 }
3561 else {
3562 $$totals_href{$key} = 0 unless ($$totals_href{$key});
3563 $$totals_href{"$key-gigs"} = 0 unless ($$totals_href{"$key-gigs"});
a83c7e95
SC		 3564 un_round($values[$i], \$$totals_href{$key}, \$$totals_href{"$key-gigs"});
3565 print STDERR "Added $values[$i] to $key - $$totals_href{$key} , " . $$totals_href{"$key-gigs"} . "GB.\n" if $debug;
059ec3d9
PH		 3566 }
3567 }
3568}
3569
a83c7e95
SC		 3570
3571#######################################################################
3572# line_to_hash();
3573#
3574# line_to_hash(\%hash,\@keys,$line);
3575#
3576# Given a line of space seperated values, set them into the provided hash
3577# using @keys as the hash keys.
3578#######################################################################
3579sub line_to_hash {
3580 my($href,$keys_aref,$values) = @_;
3581 my(@values) = split(/\s+/,$values);
3582 for(my $i = 0; $i < @values && $i < @$keys_aref; ++$i) {
3583 $$href{$keys_aref->[$i]} = $values[$i];
3584 }
3585}
3586
3587
1b4fe9dd
PH		 3588#######################################################################
3589# get_report_total();
8e669ac1 3590#
1b4fe9dd 3591# $total = get_report_total(\%hash,$key);
8e669ac1 3592#
1b4fe9dd 3593# If %hash contains values split into Units and Gigs, we calculate and return
8e669ac1 3594#
1b4fe9dd
PH		 3595# $hash{$key} + 1024*1024*1024 * $hash{"${key}-gigs"}
3596#######################################################################
059ec3d9
PH		 3597sub get_report_total {
3598 no integer;
3599 my($hash_ref,$key) = @_;
3600 if ($$hash_ref{"${key}-gigs"}) {
3601 return $$hash_ref{$key} + $gig * $$hash_ref{"${key}-gigs"};
3602 }
3603 return $$hash_ref{$key} || 0;
3604}
3605
1b4fe9dd
PH		 3606#######################################################################
3607# html2txt();
8e669ac1 3608#
1b4fe9dd 3609# $text_line = html2txt($html_line);
8e669ac1 3610#
1b4fe9dd
PH		 3611# Convert a line from html to text. Currently we just convert HTML tags to spaces
3612# and convert &gt;, &lt;, and &nbsp; tags back.
3613#######################################################################
059ec3d9
PH		 3614sub html2txt {
3615 ($_) = @_;
3616
3617 # Convert HTML tags to spacing. Note that the reports may contain <Userid> and
3618 # <Userid@Domain> words, so explicitly specify the HTML tags we will remove
3619 # (the ones used by this program). If someone is careless enough to have their
3620 # Userid the same as an HTML tag, there's not much we can do about it.
608bc29d 3621 s/<\/?(html|head|title|body|h\d|ul|li|a\s+|table|tr|td|th|pre|hr|p|br)\b.*?>/ /g;
059ec3d9
PH		 3622
3623 s/\&lt\;/\</og; #Convert '&lt;' to '<'.
3624 s/\&gt\;/\>/og; #Convert '&gt;' to '>'.
3625 s/\&nbsp\;/ /og; #Convert '&nbsp;' to ' '.
3626 return($_);
3627}
3628
1b4fe9dd
PH		 3629#######################################################################
3630# get_next_arg();
8e669ac1 3631#
1b4fe9dd 3632# $arg = get_next_arg();
8e669ac1 3633#
1b4fe9dd
PH		 3634# Because eximstats arguments are often passed as variables,
3635# we can't rely on shell parsing to deal with quotes. This
3636# subroutine returns $ARGV[1] and does a shift. If $ARGV[1]
3637# starts with a quote (' or "), and doesn't end in one, then
3638# we append the next argument to it and shift again. We repeat
3639# until we've got all of the argument.
8e669ac1 3640#
1b4fe9dd
PH		 3641# This isn't perfect as all white space gets reduced to one space,
3642# but it's as good as we can get! If it's esential that spacing
3643# be preserved precisely, then you get that by not using shell
3644# variables.
3645#######################################################################
059ec3d9
PH		 3646sub get_next_arg {
3647 my $arg = '';
3648 my $matched_pattern = 0;
3649 while ($ARGV[1]) {
3650 $arg .= ' ' if $arg;
3651 $arg .= $ARGV[1]; shift(@ARGV);
3652 if ($arg !~ /^['"]/) {
3653 $matched_pattern = 1;
3654 last;
3655 }
3656 if ($arg =~ s/^(['"])(.*)\1$/$2/) {
3657 $matched_pattern = 1;
3658 last;
3659 }
3660 }
3661 die "Mismatched argument quotes - <$arg>.\n" unless $matched_pattern;
3662 return $arg;
3663}
3664
d5692f86
SC		 3665#######################################################################
3666# set_worksheet_line($ws_global, $startrow, $startcol, \@content, $format);
3667#
3668# set values to a sequence of cells in a row.
3669#
3670#######################################################################
3671sub set_worksheet_line {
3672 my ($worksheet, $row, $col, $content, $format) = @_;
3673
3674 foreach my $token (@$content)
3675 {
3676 $worksheet->write($row, $col++, $token, $format );
3677 }
059ec3d9 3678
d5692f86 3679}
059ec3d9 3680
608bc29d
SC		 3681#######################################################################
3682# @rcpt_times = parse_time_list($string);
3683#
3684# Parse a comma seperated list of time values in seconds given by
3685# the user and fill an array.
3686#
3687# Return a default list if $string is undefined.
3688# Return () if $string eq '0'.
3689#######################################################################
3690sub parse_time_list {
3691 my($string) = @_;
3692 if (! defined $string) {
3693 return(60, 5*60, 15*60, 30*60, 60*60, 3*60*60, 6*60*60, 12*60*60, 24*60*60);
3694 }
3695 my(@times) = split(/,/, $string);
3696 foreach my $q (@times) { $q = eval($q) + 0 }
3697 @times = sort { $a <=> $b } @times;
3698 @times = () if ($#times == 0 && $times[0] == 0);
3699 return(@times);
3700}
3701
3702
3703#######################################################################
3704# initialise_rcpt_times($protocol);
3705# Initialise an array of rcpt_times to 0 for the specified protocol.
3706#######################################################################
3707sub initialise_rcpt_times {
3708 my($protocol) = @_;
3709 for (my $i = 0; $i <= $#rcpt_times; ++$i) {
3710 $rcpt_times_bin{$protocol}[$i] = 0;
3711 }
3712 $rcpt_times_overflow{$protocol} = 0;
3713}
3714
3715
059ec3d9
PH		 3716##################################################
3717# Main Program #
3718##################################################
3719
3720
3721$last_timestamp = '';
3722$last_date = '';
3723$show_errors = 1;
3724$show_relay = 1;
3725$show_transport = 1;
3726$topcount = 50;
3727$local_league_table = 1;
3728$include_remote_users = 0;
d5692f86 3729$include_original_destination = 0;
059ec3d9
PH		 3730$hist_opt = 1;
3731$volume_rounding = 1;
3732$localtime_offset = calculate_localtime_offset(); # PH/FANF
3733
3734$charts = 0;
3735$charts_option_specified = 0;
3736$chartrel = ".";
3737$chartdir = ".";
3738
608bc29d
SC		 3739@queue_times = parse_time_list();
3740@rcpt_times = ();
3741@delivery_times = ();
059ec3d9
PH		 3742
3743$last_offset = '';
3744$offset_seconds = 0;
3745
d5692f86 3746$row=1;
d5692f86
SC		 3747$col=0;
3748$col_hist=0;
3749$run_hist=0;
3750my(%output_files); # What output files have been specified?
3751
059ec3d9
PH		 3752# Decode options
3753
608bc29d 3754while (@ARGV > 0 && substr($ARGV[0], 0, 1) eq '-') {
059ec3d9
PH		 3755 if ($ARGV[0] =~ /^\-h(\d+)$/) { $hist_opt = $1 }
3756 elsif ($ARGV[0] =~ /^\-ne$/) { $show_errors = 0 }
608bc29d 3757 elsif ($ARGV[0] =~ /^\-nr(.?)(.*)\1$/) {
059ec3d9 3758 if ($1 eq "") { $show_relay = 0 } else { $relay_pattern = $2 }
608bc29d
SC		 3759 }
3760 elsif ($ARGV[0] =~ /^\-q([,\d\+\-\*\/]+)$/) { @queue_times = parse_time_list($1) }
059ec3d9
PH		 3761 elsif ($ARGV[0] =~ /^-nt$/) { $show_transport = 0 }
3762 elsif ($ARGV[0] =~ /^\-nt(.?)(.*)\1$/)
3763 {
3764 if ($1 eq "") { $show_transport = 0 } else { $transport_pattern = $2 }
3765 }
3766 elsif ($ARGV[0] =~ /^-t(\d+)$/) { $topcount = $1 }
3767 elsif ($ARGV[0] =~ /^-tnl$/) { $local_league_table = 0 }
d5692f86
SC		 3768 elsif ($ARGV[0] =~ /^-txt=?(\S*)$/) { $txt_fh = get_filehandle($1,\%output_files) }
3769 elsif ($ARGV[0] =~ /^-html=?(\S*)$/) { $htm_fh = get_filehandle($1,\%output_files) }
3770 elsif ($ARGV[0] =~ /^-xls=?(\S*)$/) {
3771 if ($HAVE_Spreadsheet_WriteExcel) {
3772 $xls_fh = get_filehandle($1,\%output_files);
3773 }
3774 else {
3775 warn "WARNING: CPAN Module Spreadsheet::WriteExcel not installed. Obtain from www.cpan.org\n";
3776 }
3777 }
059ec3d9
PH		 3778 elsif ($ARGV[0] =~ /^-merge$/) { $merge_reports = 1 }
3779 elsif ($ARGV[0] =~ /^-charts$/) {
3780 $charts = 1;
3781 warn "WARNING: CPAN Module GD::Graph::pie not installed. Obtain from www.cpan.org\n" unless $HAVE_GD_Graph_pie;
3782 warn "WARNING: CPAN Module GD::Graph::linespoints not installed. Obtain from www.cpan.org\n" unless $HAVE_GD_Graph_linespoints;
3783 }
3784 elsif ($ARGV[0] =~ /^-chartdir$/) { $chartdir = $ARGV[1]; shift; $charts_option_specified = 1; }
3785 elsif ($ARGV[0] =~ /^-chartrel$/) { $chartrel = $ARGV[1]; shift; $charts_option_specified = 1; }
d5692f86
SC		 3786 elsif ($ARGV[0] =~ /^-include_original_destination$/) { $include_original_destination = 1 }
3787 elsif ($ARGV[0] =~ /^-cache$/) { } #Not currently used.
059ec3d9
PH		 3788 elsif ($ARGV[0] =~ /^-byhost$/) { $do_sender{Host} = 1 }
3789 elsif ($ARGV[0] =~ /^-bydomain$/) { $do_sender{Domain} = 1 }
3790 elsif ($ARGV[0] =~ /^-byemail$/) { $do_sender{Email} = 1 }
3791 elsif ($ARGV[0] =~ /^-byemaildomain$/) { $do_sender{Edomain} = 1 }
3792 elsif ($ARGV[0] =~ /^-byedomain$/) { $do_sender{Edomain} = 1 }
a83c7e95 3793 elsif ($ARGV[0] =~ /^-emptyok$/) { $emptyOK = 1 }
059ec3d9 3794 elsif ($ARGV[0] =~ /^-nvr$/) { $volume_rounding = 0 }
608bc29d
SC		 3795 elsif ($ARGV[0] =~ /^-show_rt([,\d\+\-\*\/]+)?$/) { @rcpt_times = parse_time_list($1) }
3796 elsif ($ARGV[0] =~ /^-show_dt([,\d\+\-\*\/]+)?$/) { @delivery_times = parse_time_list($1) }
059ec3d9
PH		 3797 elsif ($ARGV[0] =~ /^-d$/) { $debug = 1 }
3798 elsif ($ARGV[0] =~ /^--?h(elp)?$/){ help() }
3799 elsif ($ARGV[0] =~ /^-t_remote_users$/) { $include_remote_users = 1 }
3800 elsif ($ARGV[0] =~ /^-pattern$/)
3801 {
3802 push(@user_descriptions,get_next_arg());
3803 push(@user_patterns,get_next_arg());
3804 }
3805 elsif ($ARGV[0] =~ /^-utc$/)
3806 {
3807 # We don't need this value if the log is in UTC.
3808 $localtime_offset = undef;
3809 }
3810 else
3811 {
3812 print STDERR "Eximstats: Unknown or malformed option $ARGV[0]\n";
3813 help();
3814 }
3815 shift;
3816 }
3817
d5692f86
SC		 3818 # keep old default behaviour
3819 if (! ($xls_fh or $htm_fh or $txt_fh)) {
3820 $txt_fh = \*STDOUT;
3821 }
3822
059ec3d9
PH		 3823 # Check that all the charts options are specified.
3824 warn "-charts option not specified. Use -help for help.\n" if ($charts_option_specified && ! $charts);
3825
3826 # Default to display tables by sending Host.
3827 $do_sender{Host} = 1 unless ($do_sender{Domain} || $do_sender{Email} || $do_sender{Edomain});
3828
d5692f86 3829 # prepare xls Excel Workbook
a83c7e95 3830 if (defined $xls_fh) {
d5692f86
SC		 3831
3832 # Create a new Excel workbook
3833 $workbook = Spreadsheet::WriteExcel->new($xls_fh);
3834
3835 # Add worksheets
3836 $ws_global = $workbook->addworksheet('Exim Statistik');
3837 # show $ws_global as initial sheet
3838 $ws_global->set_first_sheet();
3839 $ws_global->activate();
3840
3841 if ($show_relay) {
3842 $ws_relayed = $workbook->addworksheet('Relayed Messages');
3843 $ws_relayed->set_column(1, 2, 80);
3844 }
d5692f86
SC		 3845 if ($show_errors) {
3846 $ws_errors = $workbook->addworksheet('Errors');
3847 }
3848
3849
3850 # set column widths
3851 $ws_global->set_column(0, 2, 20); # Columns B-D width set to 30
3852 $ws_global->set_column(3, 3, 15); # Columns B-D width set to 30
3853 $ws_global->set_column(4, 4, 25); # Columns B-D width set to 30
3854
3855 # Define Formats
3856 $f_default = $workbook->add_format();
3857
3858 $f_header1 = $workbook->add_format();
3859 $f_header1->set_bold();
3860 #$f_header1->set_color('red');
3861 $f_header1->set_size('15');
3862 $f_header1->set_valign();
3863 # $f_header1->set_align('center');
3864 # $ws_global->write($row++, 2, "Testing Headers 1", $f_header1);
3865
3866 $f_header2 = $workbook->add_format();
3867 $f_header2->set_bold();
3868 $f_header2->set_size('12');
3869 $f_header2->set_valign();
3870 # $ws_global->write($row++, 2, "Testing Headers 2", $f_header2);
3871
a83c7e95
SC		 3872 # Create another header2 for use in merged cells.
3873 $f_header2_m = $workbook->add_format();
3874 $f_header2_m->set_bold();
3875 $f_header2_m->set_size('8');
3876 $f_header2_m->set_valign();
3877 $f_header2_m->set_align('center');
3878
d5692f86
SC		 3879 $f_percent = $workbook->add_format();
3880 $f_percent->set_num_format('0.0%');
3881
3882 $f_headertab = $workbook->add_format();
3883 $f_headertab->set_bold();
3884 $f_headertab->set_valign();
3885 # $ws_global->write($row++, 2, "Testing Headers tab", $f_headertab);
3886
3887 }
3888
059ec3d9 3889
608bc29d 3890# Initialise the queue/delivery/rcpt time counters.
059ec3d9 3891for (my $i = 0; $i <= $#queue_times; $i++) {
608bc29d
SC		 3892 $qt_all_bin[$i] = 0;
3893 $qt_remote_bin[$i] = 0;
3894}
3895for (my $i = 0; $i <= $#delivery_times; $i++) {
3896 $dt_all_bin[$i] = 0;
3897 $dt_remote_bin[$i] = 0;
059ec3d9 3898}
608bc29d 3899initialise_rcpt_times('all');
059ec3d9 3900
059ec3d9 3901
608bc29d 3902# Compute the number of slots for the histogram
059ec3d9
PH		 3903if ($hist_opt > 0)
3904 {
3905 if ($hist_opt > 60 || 60 % $hist_opt != 0)
3906 {
d5692f86 3907 print STDERR "Eximstats: -h must specify a factor of 60\n";
059ec3d9
PH		 3908 exit 1;
3909 }
d5692f86
SC		 3910 $hist_interval = 60/$hist_opt; #Interval in minutes.
3911 $hist_number = (24*60)/$hist_interval; #Number of intervals per day.
059ec3d9
PH		 3912 @received_interval_count = (0) x $hist_number;
3913 @delivered_interval_count = (0) x $hist_number;
608bc29d
SC		 3914 my $user_pattern_index = 0;
3915 for (my $user_pattern_index = 0; $user_pattern_index <= $#user_patterns; ++$user_pattern_index) {
3916 @{$user_pattern_interval_count[$user_pattern_index]} = (0) x $hist_number;
3917 }
3918 @dt_all_bin = (0) x $hist_number;
3919 @dt_remote_bin = (0) x $hist_number;
81aad8c9 3920}
059ec3d9
PH		 3921
3922#$queue_unknown = 0;
3923
3924$total_received_data = 0;
3925$total_received_data_gigs = 0;
3926$total_received_count = 0;
3927
3928$total_delivered_data = 0;
3929$total_delivered_data_gigs = 0;
a83c7e95
SC		 3930$total_delivered_messages = 0;
3931$total_delivered_addresses = 0;
059ec3d9 3932
608bc29d
SC		 3933$qt_all_overflow = 0;
3934$qt_remote_overflow = 0;
3935$dt_all_overflow = 0;
3936$dt_remote_overflow = 0;
059ec3d9
PH		 3937$delayed_count = 0;
3938$relayed_unshown = 0;
608bc29d 3939$message_errors = 0;
059ec3d9
PH		 3940$begin = "9999-99-99 99:99:99";
3941$end = "0000-00-00 00:00:00";
3942my($section,$type);
a83c7e95 3943foreach $section ('Received','Delivered','Rejects','Ham','Spam') {
059ec3d9
PH		 3944 foreach $type ('Volume','Messages','Delayed','Failed','Hosts','Domains','Emails','Edomains') {
3945 $report_totals{$section}{$type} = 0;
3946 }
3947}
3948
3949# Generate our parser.
3950my $parser = generate_parser();
3951
3952
3953
3954if (@ARGV) {
3955 # Scan the input files and collect the data
3956 foreach my $file (@ARGV) {
3957 if ($file =~ /\.gz/) {
3958 unless (open(FILE,"gunzip -c $file |")) {
d5692f86
SC		 3959 print STDERR "Failed to gunzip -c $file: $!";
3960 next;
059ec3d9
PH		 3961 }
3962 }
3963 elsif ($file =~ /\.Z/) {
3964 unless (open(FILE,"uncompress -c $file |")) {
d5692f86
SC		 3965 print STDERR "Failed to uncompress -c $file: $!";
3966 next;
059ec3d9
PH		 3967 }
3968 }
3969 else {
3970 unless (open(FILE,$file)) {
d5692f86
SC		 3971 print STDERR "Failed to read $file: $!";
3972 next;
059ec3d9
PH		 3973 }
3974 }
3975 #Now parse the filehandle, updating the global variables.
3976 parse($parser,\*FILE);
3977 close FILE;
3978 }
3979}
3980else {
3981 #No files provided. Parse STDIN, updating the global variables.
3982 parse($parser,\*STDIN);
3983}
3984
3985
a83c7e95 3986if ($begin eq "9999-99-99 99:99:99" && ! $emptyOK) {
d5692f86 3987 print STDERR "**** No valid log lines read\n";
059ec3d9
PH		 3988 exit 1;
3989}
3990
3991# Output our results.
3992print_header();
3993print_grandtotals();
3994
3995# Print counts of user specified patterns if required.
3996print_user_patterns() if @user_patterns;
3997
a83c7e95
SC		 3998# Print rejection reasons.
3999# print_rejects();
4000
059ec3d9
PH		 4001# Print totals by transport if required.
4002print_transport() if $show_transport;
4003
4004# Print the deliveries per interval as a histogram, unless configured not to.
4005# First find the maximum in one interval and scale accordingly.
4006if ($hist_opt > 0) {
608bc29d
SC		 4007 print_histogram("Messages received", 'message', @received_interval_count);
4008 print_histogram("Deliveries", 'delivery', @delivered_interval_count);
059ec3d9
PH		 4009}
4010
4011# Print times on queue if required.
4012if ($#queue_times >= 0) {
608bc29d
SC		 4013 print_duration_table("Time spent on the queue", "all messages", \@queue_times, \@qt_all_bin,$qt_all_overflow);
4014 print_duration_table("Time spent on the queue", "messages with at least one remote delivery", \@queue_times, \@qt_remote_bin,$qt_remote_overflow);
4015}
4016
4017# Print delivery times if required.
4018if ($#delivery_times >= 0) {
4019 print_duration_table("Delivery times", "all messages", \@delivery_times, \@dt_all_bin,$dt_all_overflow);
4020 print_duration_table("Delivery times", "messages with at least one remote delivery", \@delivery_times, \@dt_remote_bin,$dt_remote_overflow);
4021}
4022
4023# Print rcpt times if required.
4024if ($#rcpt_times >= 0) {
4025 foreach my $protocol ('all', grep(!/^all$/, sort keys %rcpt_times_bin)) {
4026 print_duration_table("Receipt times", "$protocol messages", \@rcpt_times, $rcpt_times_bin{$protocol}, $rcpt_times_overflow{$protocol});
4027 }
059ec3d9
PH		 4028}
4029
4030# Print relay information if required.
4031print_relay() if $show_relay;
4032
4033# Print the league tables, if topcount isn't zero.
4034if ($topcount > 0) {
a83c7e95
SC		 4035 my($ws_rej, $ws_top50, $ws_rej_row, $ws_top50_row);
4036 $ws_rej_row = $ws_top50_row = 0;
4037 if ($xls_fh) {
4038 $ws_top50 = $workbook->addworksheet('Deliveries');
4039 $ws_rej = $workbook->addworksheet('Rejections') if (%rejected_count_by_reason || %rejected_count_by_ip || %spam_count_by_ip);
4040 }
4041
4042 print_league_table("mail rejection reason", \%rejected_count_by_reason, undef, undef, undef, $ws_rej, \$ws_rej_row) if %rejected_count_by_reason;
4043
059ec3d9
PH		 4044 foreach ('Host','Domain','Email','Edomain') {
4045 next unless $do_sender{$_};
a83c7e95 4046 print_league_table("sending \l$_", $received_count{$_}, undef, $received_data{$_},$received_data_gigs{$_}, $ws_top50, \$ws_top50_row);
059ec3d9
PH		 4047 }
4048
a83c7e95
SC		 4049 print_league_table("local sender", \%received_count_user, undef,
4050 \%received_data_user,\%received_data_gigs_user, $ws_top50, \$ws_top50_row) if (($local_league_table || $include_remote_users) && %received_count_user);
059ec3d9
PH		 4051 foreach ('Host','Domain','Email','Edomain') {
4052 next unless $do_sender{$_};
a83c7e95 4053 print_league_table("\l$_ destination", $delivered_messages{$_}, $delivered_addresses{$_}, $delivered_data{$_},$delivered_data_gigs{$_}, $ws_top50, \$ws_top50_row);
059ec3d9 4054 }
a83c7e95
SC		 4055 print_league_table("local destination", \%delivered_messages_user, \%delivered_addresses_user, \%delivered_data_user,\%delivered_data_gigs_user, $ws_top50, \$ws_top50_row) if (($local_league_table || $include_remote_users) && %delivered_messages_user);
4056
4057 print_league_table("rejected ip", \%rejected_count_by_ip, undef, undef, undef, $ws_rej, \$ws_rej_row) if %rejected_count_by_ip;
4058 print_league_table("non-rejected spamming ip", \%spam_count_by_ip, undef, undef, undef, $ws_rej, \$ws_rej_row) if %spam_count_by_ip;
4059
059ec3d9
PH		 4060}
4061
4062# Print the error statistics if required.
4063print_errors() if $show_errors;
4064
d5692f86
SC		 4065print $htm_fh "</body>\n</html>\n" if $htm_fh;
4066
4067
a83c7e95 4068$txt_fh->close if $txt_fh && ref $txt_fh;
d5692f86
SC		 4069$htm_fh->close if $htm_fh;
4070
4071if ($xls_fh) {
4072 # close Excel Workbook
4073 $ws_global->set_first_sheet();
4074 # FIXME: whyever - activate does not work :-/
4075 $ws_global->activate();
4076 $workbook->close();
059ec3d9
PH		 4077}
4078
d5692f86 4079
059ec3d9 4080# End of eximstats
Unnamed repository; edit this file 'description' to name the repository.