From: Jeremy Harris Date: Sat, 10 Jan 2015 21:39:44 +0000 (+0000) Subject: Add queue_size variable. Bug 1406 X-Git-Tag: exim-4_94_RC0~101 X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=04403ab0b5778126d98d5c9dc8064405688129e0;p=exim.git Add queue_size variable. Bug 1406 --- diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 888eb7f60..bba71b76d 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -12854,6 +12854,13 @@ or if not set, the value of &$qualify_domain$&. .cindex queues named The name of the spool queue in use; empty for the default queue. +.vitem &$queue_size$& +.vindex "&$queue_size$&" +.cindex "queue" "size of" +.cindex "spool" "number of messages" +This variable contains the number of messages queued. +It is evaluated on demand, but no more often than once every minute. + .vitem &$r_...$& .vindex &$r_...$& .cindex router variables diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 919a56ea5..3cef43c32 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -35,9 +35,11 @@ Version 4.94 9. The ACL control "queue_only" can also be spelled "queue", and now takes an option "first_pass_route" to do the same as a "-odqs" on the command line. + 9. Items specified for the router and transport headers_remove option can use a trailing asterisk to specify globbing. +10. New "queue_size" variable. @@ -371,7 +373,6 @@ Version 4.86 14. Main option "dns_trust_aa" for trusting your local nameserver at the same level as DNSSEC. - Version 4.85 ------------ diff --git a/src/src/daemon.c b/src/src/daemon.c index b6c27ffe5..a6980e95e 100644 --- a/src/src/daemon.c +++ b/src/src/daemon.c @@ -983,6 +983,7 @@ daemon_notifier_socket(void) int fd; const uschar * where; struct sockaddr_un sun = {.sun_family = AF_UNIX}; +int len; DEBUG(D_any) debug_printf("creating notifier socket\n"); @@ -996,10 +997,12 @@ if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0))) < 0) (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); #endif -snprintf(sun.sun_path, sizeof(sun.sun_path), "%s/%s", - spool_directory, NOTIFIER_SOCKET_NAME); +sun.sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ +len = offsetof(struct sockaddr_un, sun_path) + 1 + + snprintf(sun.sun_path+1, sizeof(sun.sun_path)-1, "%s", NOTIFIER_SOCKET_NAME); + where = US"bind"; -if (bind(fd, (const struct sockaddr *)&sun, sizeof(sun)) < 0) +if (bind(fd, (const struct sockaddr *)&sun, len) < 0) goto bad; where = US"SO_PASSCRED"; @@ -1023,9 +1026,10 @@ static BOOL daemon_notification(void) { uschar buf[256], cbuf[256]; +struct sockaddr_un sun; struct iovec iov = {.iov_base = buf, .iov_len = sizeof(buf)-1}; -struct msghdr msg = { .msg_name = NULL, - .msg_namelen = 0, +struct msghdr msg = { .msg_name = &sun, + .msg_namelen = sizeof(sun), .msg_iov = &iov, .msg_iovlen = 1, .msg_control = cbuf, @@ -1038,6 +1042,14 @@ buf[sizeof(buf)-1] = 0; if ((sz = recvmsg(daemon_notifier_fd, &msg, 0)) <= 0) return FALSE; if (sz >= sizeof(buf)) return FALSE; +#ifdef notdef +debug_printf("addrlen %d\n", msg.msg_namelen); +#endif +DEBUG(D_queue_run) debug_printf("%s from addr%s '%s'\n", __FUNCTION__, + *sun.sun_path ? "" : " abstract", sun.sun_path+ (*sun.sun_path ? 0 : 1)); + +/* Refuse to handle the item unless the peer has good credentials */ + for (struct cmsghdr * cp = CMSG_FIRSTHDR(&msg); cp; cp = CMSG_NXTHDR(&msg, cp)) @@ -1064,6 +1076,20 @@ switch (buf[0]) memcpy(queuerun_msgid, buf+1, MESSAGE_ID_LENGTH+1); return TRUE; #endif /*EXPERIMENTAL_QUEUE_RAMP*/ + + case NOTIFY_QUEUE_SIZE_REQ: + { + uschar buf[16]; + int len = snprintf(CS buf, sizeof(buf), "%u", queue_count_cached()); + + DEBUG(D_queue_run) + debug_printf("%s: queue size request: %s\n", __FUNCTION__, buf); + + if (sendto(daemon_notifier_fd, buf, len, 0, &sun, msg.msg_namelen) < 0) + log_write(0, LOG_MAIN|LOG_PANIC, + "%s: sendto: %s\n", __FUNCTION__, strerror(errno)); + return FALSE; + } } return FALSE; } diff --git a/src/src/exim.c b/src/src/exim.c index a8f3c2248..dfd6df76c 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -4363,7 +4363,7 @@ if (list_queue) if (count_queue) { set_process_info("counting the queue"); - queue_count(); + fprintf(stdout, "%u\n", queue_count()); exit(EXIT_SUCCESS); } diff --git a/src/src/expand.c b/src/src/expand.c index 7986bbd14..cd4522afb 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -466,6 +466,7 @@ typedef struct { static uschar * fn_recipients(void); typedef uschar * stringptr_fn_t(void); +static uschar * fn_queue_size(void); /* This table must be kept in alphabetical order. */ @@ -669,6 +670,7 @@ static var_entry var_table[] = { { "qualify_domain", vtype_stringptr, &qualify_domain_sender }, { "qualify_recipient", vtype_stringptr, &qualify_domain_recipient }, { "queue_name", vtype_stringptr, &queue_name }, + { "queue_size", vtype_string_func, &fn_queue_size }, { "rcpt_count", vtype_int, &rcpt_count }, { "rcpt_defer_count", vtype_int, &rcpt_defer_count }, { "rcpt_fail_count", vtype_int, &rcpt_fail_count }, @@ -1740,6 +1742,65 @@ return g ? g->s : NULL; } +/************************************************* +* Return size of queue * +*************************************************/ +/* Ask the daemon for the queue size */ + +static uschar * +fn_queue_size(void) +{ +struct sockaddr_un sun = {.sun_family = AF_UNIX}; +uschar buf[16]; +int fd; +ssize_t len; +const uschar * where; + +if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) < 0) + { + DEBUG(D_expand) debug_printf(" socket: %s\n", strerror(errno)); + return NULL; + } + +#define ABSTRACT_CLIENT +#ifdef ABSTRACT_CLIENT +sun.sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ +len = offsetof(struct sockaddr_un, sun_path) + 1 + + snprintf(sun.sun_path+1, sizeof(sun.sun_path)-1, "exim_%d", getpid()); +#else +len = offsetof(struct sockaddr_un, sun_path) + + snprintf(sun.sun_path, sizeof(sun.sun_path), "%s/p_%d", + spool_directory, getpid()); +#endif + +if (bind(fd, &sun, len) < 0) { where = US"bind"; goto bad; } + +#ifdef notdef +debug_printf("local%s '%s'\n", *sun.sun_path ? "" : " abstract", + sun.sun_path+ (*sun.sun_path ? 0 : 1)); +#endif + +sun.sun_path[0] = 0; /* Abstract local socket addr - Linux-specific? */ +len = offsetof(struct sockaddr_un, sun_path) + 1 + + snprintf(sun.sun_path+1, sizeof(sun.sun_path)-1, "%s", NOTIFIER_SOCKET_NAME); + +if (connect(fd, &sun, len) < 0) { where = US"connect"; goto bad; } + +buf[0] = NOTIFY_QUEUE_SIZE_REQ; +if (send(fd, buf, 1, 0) < 0) { where = US"send"; goto bad; } + +if ((len = recv(fd, buf, sizeof(buf), 0)) < 0) { where = US"recv"; goto bad; } + +close(fd); +return string_copyn(buf, len); + +bad: + close(fd); + DEBUG(D_expand) debug_printf(" %s: %s\n", where, strerror(errno)); + return NULL; +} + + /************************************************* * Find value of a variable * *************************************************/ diff --git a/src/src/functions.h b/src/src/functions.h index 9716a02b4..be929a710 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -363,7 +363,8 @@ extern int vaguely_random_number_fallback(int); extern BOOL queue_action(uschar *, int, uschar **, int, int); extern void queue_check_only(void); -extern void queue_count(void); +extern unsigned queue_count(void); +extern unsigned queue_count_cached(void); extern void queue_list(int, uschar **, int); #ifdef EXPERIMENTAL_QUEUE_RAMP extern void queue_notify_daemon(const uschar * hostname); diff --git a/src/src/globals.c b/src/src/globals.c index a06aa5c1f..a5711c73b 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -1208,6 +1208,8 @@ int queue_only_load = -1; uschar *queue_run_max = US"5"; pid_t queue_run_pid = (pid_t)0; int queue_run_pipe = -1; +unsigned queue_size = 0; +time_t queue_size_next = 0; uschar *queue_smtp_domains = NULL; uint32_t random_seed = 0; diff --git a/src/src/globals.h b/src/src/globals.h index 760863ac3..b570078c3 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -805,6 +805,8 @@ extern uschar *queue_only_file; /* Queue if file exists/not-exists */ extern BOOL queue_only_override; /* Allow override from command line */ extern BOOL queue_run_in_order; /* As opposed to random */ extern uschar *queue_run_max; /* Max queue runners */ +extern unsigned queue_size; /* items in queue */ +extern time_t queue_size_next; /* next time to evaluate queue_size */ extern uschar *queue_smtp_domains; /* Ditto, for these domains */ extern unsigned int random_seed; /* Seed for random numbers */ diff --git a/src/src/macros.h b/src/src/macros.h index 7f50dbb89..93756d8a1 100644 --- a/src/src/macros.h +++ b/src/src/macros.h @@ -1101,6 +1101,7 @@ should not be one active. */ #define NOTIFIER_SOCKET_NAME "exim_daemon_notify" -#define NOTIFY_MSG_QRUN 1 /* Notify message types */ +#define NOTIFY_MSG_QRUN 1 /* Notify message types */ +#define NOTIFY_QUEUE_SIZE_REQ 2 /* End of macros.h */ diff --git a/src/src/queue.c b/src/src/queue.c index 5f75470e0..ac7aad1a0 100644 --- a/src/src/queue.c +++ b/src/src/queue.c @@ -767,26 +767,39 @@ if (!recurse) /* Called as a result of -bpc Arguments: none -Returns: nothing +Returns: count */ -void +unsigned queue_count(void) { int subcount; -int count = 0; +unsigned count = 0; uschar subdirs[64]; -for (queue_filename *f = queue_get_spool_list( +for (queue_filename * f = queue_get_spool_list( -1, /* entire queue */ subdirs, /* for holding sub list */ &subcount, /* for subcount */ FALSE); /* not random */ f; f = f->next) count++; -fprintf(stdout, "%d\n", count); +return count; } +#define QUEUE_SIZE_AGE 60 /* update rate for queue_size */ + +unsigned +queue_count_cached(void) +{ +time_t now; +if ((now = time(NULL)) >= queue_size_next) + { + queue_size = queue_count(); + queue_size_next = now + (f.running_in_test_harness ? 3 : QUEUE_SIZE_AGE); + } +return queue_size; +} /************************************************ * List extra deliveries * @@ -1511,11 +1524,12 @@ memcpy(buf+1, msgid, MESSAGE_ID_LENGTH+1); if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) >= 0) { struct sockaddr_un sun = {.sun_family = AF_UNIX}; + int len = offsetof(struct sockaddr_un, sun_path) + 1 + + snprintf(sun.sun_path+1, sizeof(sun.sun_path)-1, "%s", + NOTIFIER_SOCKET_NAME); + sun.sun_path[0] = 0; - snprintf(sun.sun_path, sizeof(sun.sun_path), "%s/%s", - spool_directory, NOTIFIER_SOCKET_NAME); - - if (sendto(fd, buf, sizeof(buf), 0, &sun, sizeof(sun)) < 0) + if (sendto(fd, buf, sizeof(buf), 0, &sun, len) < 0) DEBUG(D_queue_run) debug_printf("%s: sendto %s\n", __FUNCTION__, strerror(errno)); close(fd); diff --git a/test/confs/0621 b/test/confs/0621 new file mode 100644 index 000000000..32d906203 --- /dev/null +++ b/test/confs/0621 @@ -0,0 +1,22 @@ +# Exim test configuration 0621 + +.include DIR/aux-var/std_conf_prefix + + +# ----- Main settings ----- + +domainlist local_domains = test.ex +qualify_domain = test.ex +queue_only + +log_selector = +millisec + +acl_not_smtp = on_rcpt + +begin acl + +on_rcpt: + accept logwrite = queue_size: $queue_size + + +# End diff --git a/test/log/0621 b/test/log/0621 new file mode 100644 index 000000000..9f7acbfd2 --- /dev/null +++ b/test/log/0621 @@ -0,0 +1,11 @@ +2017-07-30 18:51:05.712 10HmaX-0005vi-00 queue_size: 0 +2017-07-30 18:51:05.712 10HmaX-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss +2017-07-30 18:51:05.712 10HmaY-0005vi-00 queue_size: 0 +2017-07-30 18:51:05.712 10HmaY-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 queue_size: 0 +2017-07-30 18:51:05.712 10HmaZ-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss +2017-07-30 18:51:05.712 10HmbA-0005vi-00 queue_size: 3 +2017-07-30 18:51:05.712 10HmbA-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss + +******** SERVER ******** +2017-07-30 18:51:05.712 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port PORT_D diff --git a/test/runtest b/test/runtest index d9db2ccc8..0dc5ec967 100755 --- a/test/runtest +++ b/test/runtest @@ -1211,6 +1211,9 @@ RESET_AFTER_EXTRA_LINE_READ: # are unset, because dane ain't always there. next if /in\shosts_require_dane\?\sno\s\(option\sunset\)/x; + # daemon notifier socket + s/^\d+ creating notifier socket$/ppppp creating notifier socket/; + # DISABLE_OCSP next if /in hosts_requ(est|ire)_ocsp\? (no|yes)/; diff --git a/test/scripts/0000-Basic/0621 b/test/scripts/0000-Basic/0621 new file mode 100644 index 000000000..53512ade4 --- /dev/null +++ b/test/scripts/0000-Basic/0621 @@ -0,0 +1,19 @@ +# $queue_size +# +exim -bd -DSERVER=server -oX PORT_D +**** +# +exim a@test.ex +**** +exim a@test.ex +**** +exim a@test.ex +Subject: msg 3 +**** +sleep 4 +exim a@test.ex +Subject: msg 4 +**** +# +killdaemon +no_msglog_check diff --git a/test/stderr/0433 b/test/stderr/0433 index d0f69f0cb..6164b0f41 100644 --- a/test/stderr/0433 +++ b/test/stderr/0433 @@ -7,6 +7,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on all interfaces (IPv4) port 1225 pid written to TESTSUITE/spool/exim-daemon.pid changed uid/gid: running as a daemon @@ -23,6 +24,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv4) port 1226 pid written to TESTSUITE/spool/exim-daemon.pid @@ -40,6 +42,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on 127.0.0.1 port 1228 listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv4) port 1226 @@ -58,6 +61,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv4) port 1226 listening on 127.0.0.1 port 1228 @@ -78,6 +82,7 @@ dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME daemon_smtp_port overridden by -oX: <: 1227 +creating notifier socket listening on 127.0.0.1 port 1228 listening on all interfaces (IPv4) port 1227 pid written to TESTSUITE/spool/exim-daemon.pid @@ -97,6 +102,7 @@ dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME local_interfaces overridden by -oX: <; 0.0.0.0.1225; 0.0.0.0.1226 +creating notifier socket listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv4) port 1226 pid written to TESTSUITE/spool/exim-daemon.pid diff --git a/test/stderr/0438 b/test/stderr/0438 index bac5bc1a6..b4afb0a5b 100644 --- a/test/stderr/0438 +++ b/test/stderr/0438 @@ -9,6 +9,7 @@ dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME daemon_smtp_port overridden by -oX: <: 1225 +creating notifier socket listening on all interfaces (IPv4) port 1225 pid written to TESTSUITE/spool/exim-daemon.pid changed uid/gid: running as a daemon @@ -27,6 +28,7 @@ dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME daemon_smtp_port overridden by -oX: <: 1225 +creating notifier socket listening on all interfaces (IPv4) port 1225 pid written to TESTSUITE/spool/exim-daemon.anotherpid changed uid/gid: running as a daemon @@ -45,6 +47,7 @@ dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME daemon_smtp_port overridden by -oX: <: 1225 +creating notifier socket listening on all interfaces (IPv4) port 1225 pid written to TESTSUITE/spool/mypidfile changed uid/gid: running as a daemon diff --git a/test/stderr/0609 b/test/stderr/0609 index 8cb2b5b91..d8b91de15 100644 --- a/test/stderr/0609 +++ b/test/stderr/0609 @@ -6,6 +6,7 @@ admin user dropping to exim gid; retaining priv uid ppppp daemon_smtp_port overridden by -oX: ppppp <: 1225 +ppppp creating notifier socket ppppp listening on all interfaces (IPv4) port 1225 ppppp pid written to TESTSUITE/spool/exim-daemon.pid ppppp LOG: MAIN diff --git a/test/stderr/1007 b/test/stderr/1007 index ad5f748f9..95a96a4d0 100644 --- a/test/stderr/1007 +++ b/test/stderr/1007 @@ -7,6 +7,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on all interfaces (IPv6) port 1225 listening on all interfaces (IPv4) port 1225 pid written to TESTSUITE/spool/exim-daemon.pid @@ -24,6 +25,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on all interfaces (IPv6) port 1225 listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv6) port 1226 @@ -43,6 +45,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on all interfaces (IPv6) port 1225 listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv6) port 1226 @@ -63,6 +66,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on all interfaces (IPv6) port 1225 listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv6) port 1226 @@ -83,6 +87,7 @@ configuration file is TESTSUITE/test-config admin user dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME +ppppp creating notifier socket listening on ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6 port 1225 listening on ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6 port 1226 pid written to TESTSUITE/spool/exim-daemon.pid @@ -102,6 +107,7 @@ dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME daemon_smtp_port overridden by -oX: <: 1227 +creating notifier socket listening on all interfaces (IPv6) port 1227 listening on all interfaces (IPv4) port 1227 listening on 127.0.0.1 port 1228 @@ -124,6 +130,7 @@ daemon_smtp_port overridden by -oX: <; 1227; 1225 local_interfaces overridden by -oX: <; 127.0.0.1 +creating notifier socket listening on 127.0.0.1 port 1227 listening on 127.0.0.1 port 1225 pid written to TESTSUITE/spool/exim-daemon.pid @@ -143,6 +150,7 @@ dropping to exim gid; retaining priv uid originator: uid=CALLER_UID gid=CALLER_GID login=CALLER name=CALLER_NAME local_interfaces overridden by -oX: <; ::0.1225; 0.0.0.0.1225; 0.0.0.0.1226 +creating notifier socket listening on all interfaces (IPv6) port 1225 listening on all interfaces (IPv4) port 1225 listening on all interfaces (IPv4) port 1226 diff --git a/test/stderr/2201 b/test/stderr/2201 index 23d3d6a5f..2d1386af6 100644 --- a/test/stderr/2201 +++ b/test/stderr/2201 @@ -193,6 +193,7 @@ admin user dropping to exim gid; retaining priv uid ppppp daemon_smtp_port overridden by -oX: ppppp <: 1225 +ppppp creating notifier socket ppppp listening on all interfaces (IPv4) port 1225 ppppp pid written to TESTSUITE/spool/exim-daemon.pid ppppp LOG: MAIN