--- /dev/null
+First off, you MUST have BIND 9.x or later compiled and installed. You
+need the lwres libraries and bind libraries (and include files) and you
+will also need lwresd. Once you have it compiled and installed, try running
+lwresd. On solaris I had to use the -i flag to specify where to write
+the pid file, and on linux I didn't. (shrug) Go figure. But you gotta get
+lwresd running before moving on. Don't panic, this is FAR easier to get
+going than a full-blown BIND server (grin).
+
+Building bulkquery
+
+First, edit the Makefile and uncomment out the section of CC, CFLAGS, and
+LDFLAGS settings for your operating system and set DESTDIR to the directory
+you want to install it in.
+
+Then, edit bulkquery.c and edit the MAXTHREADS and DEFTTL values. You probably
+don't need to fiddle with anything else and the defaults for MAXTHREADS and
+DEFTTL should be fine.
+
+MAXTHREADS is the maximum number of threads you will spawn. The more you
+spawn, the harder you will work your DNS server (and the RBLs) and the
+more resources you'll use up on your own machine -- AND the faster the
+queries will complete. If you set this too high your operating system might
+not permit bulkquery to run. The default of 50 works fine on Solaris and
+and Linux.
+
+DEFTTL is the default time to live value. This is sort of a misnomer, since
+the REAL time to live values will be obtained in the replies for queries
+that were found. But for the queries that were NOT found, DEFTTL will be
+used. This is so that the non-existence of an IP in one of the RBLs will be
+cached by the filters plugin for DEFTTL seconds.
+
+After looking at bulkquery.c, just run "make" to compile it. If it compiles
+without errors, you can run "make install" (you may need to be root to
+write to DESTDIR). You can test it with the included sample input file
+bq.in
+
+If all you get back are entries with the default TTL value (ie. 7200), then
+something isn't working. Make sure you have lwresd running. You should also
+make sure the machine you're running it on CAN make DNS queries (grin). So
+check your /etc/resolv.conf file (on UNIX machines anyway). I'll include
+the file bq.out to show what the output of bulkquery looked like at the time
+of this writing (assuming everything was working).
--- /dev/null
+
+DESTDIR=/usr/local/bin
+
+# Solaris
+#CC=gcc
+#CFLAGS=-I/usr/local/tools/bind920/bind/include -I/usr/local/tools/bind920/include
+#LDFLAGS=-L/usr/local/tools/bind920/lib -llwres -lsocket -lnsl -lpthread
+
+# Linux
+CC=gcc
+CFLAGS=
+LDFLAGS= -lpthread -llwres
+
+all: bulkquery
+
+bulkquery: bulkquery.c
+ $(CC) $(CFLAGS) -o bulkquery bulkquery.c $(LDFLAGS)
+
+clean:
+ rm bulkquery
+
+install:
+ cp bulkquery $(DESTDIR)/bulkquery
+ chmod 755 $(DESTDIR)/bulkquery
--- /dev/null
+
+bulkquery - A program by Brent Bice - 02/2002
+
+ Bulkquery is a multi-threaded program that makes a large number of
+DNS queries as fast as possible. It's tailored for making DNS queries for
+specific IP addresses against specific DNS based RBLs.
+
+ "Why bother?"
+I've been asked this several times. While running squirrelmail on a server
+with very little bandwidth and very high latency (a dial-up account - snicker)
+I noticed that while my filters plugin was making gethostbyname() function
+calls, my dial-up line was mostly idle. This was, apparently, because all
+the DNS function calls were being made in series. Once a DNS query had been
+sent, no others were sent until a response had been received or until a
+certain timeout period had occurred. And some of the RBLs were quite a
+bit faster than the others. This seemed pretty inefficient. "Why not make
+more DNS queries while waiting for the replies," I thought.
+
+ After several attempts to make a multi-threaded program to make the queries
+using the standard DNS library calls, I realized why. The library functions
+weren't thread-safe and most of the replies would be lost. Luckily, a
+friend of a friend told me about the lwres API in Bind 9.x and told me that
+IT was definitely thread-safe and very fast.
+
+ Boy was SHE right!
+
+ So, how much faster is it? It will depend on your bandwidth and the
+latency of your connection. But for those of us trying to make do with less,
+bulkquery is a LOT faster. On a dial-up connection, I can make queries
+anywhere from 6 to 10 times faster! On my T1 at work, the queries are
+usually between 2 and 7 times faster. The bottom line of all of this is
+that even on my T1 at work, the filters plugin can query ALL of the RBLs
+for all the new email in my INBOX in less than half the time it takes using
+the PHP gethostbyname() function calls.
--- /dev/null
+.bl.spamcop.net
+.relays.osirusoft.com
+----------
+2.0.0.127
+3.0.0.127
+4.0.0.127
+5.0.0.127
+6.0.0.127
+7.0.0.127
--- /dev/null
+7.0.0.127.relays.osirusoft.com, 7.0.0.127.relays.osirusoft.com, 7200
+6.0.0.127.relays.osirusoft.com,127.0.0.6,42807
+5.0.0.127.relays.osirusoft.com, 5.0.0.127.relays.osirusoft.com, 7200
+4.0.0.127.relays.osirusoft.com,127.0.0.4,42807
+6.0.0.127.bl.spamcop.net, 6.0.0.127.bl.spamcop.net, 7200
+7.0.0.127.bl.spamcop.net, 7.0.0.127.bl.spamcop.net, 7200
+3.0.0.127.relays.osirusoft.com,127.0.0.3,42808
+4.0.0.127.bl.spamcop.net, 4.0.0.127.bl.spamcop.net, 7200
+5.0.0.127.bl.spamcop.net, 5.0.0.127.bl.spamcop.net, 7200
+2.0.0.127.bl.spamcop.net,127.0.0.2,1542
+2.0.0.127.relays.osirusoft.com,127.0.0.2,42808
+3.0.0.127.bl.spamcop.net, 3.0.0.127.bl.spamcop.net, 7200
--- /dev/null
+
+#include <arpa/nameser.h>
+#include <lwres/lwres.h>
+#include <strings.h>
+#include <stdio.h>
+#include <pthread.h>
+
+
+#define BUFLEN 1024
+#define MAXSTR 80
+#define MAXTHREADS 50
+#define MAXRBLS 40
+#define DEFTTL 7200
+
+extern int errno;
+extern int h_errno;
+
+
+struct ipnode;
+typedef struct ipnode *iplist;
+struct ipnode {
+ char *IP;
+ iplist next;
+};
+
+iplist IPs;
+
+pthread_mutex_t *mutexp;
+pthread_mutex_t *mutexoutput;
+
+char *dnsrbls[MAXRBLS];
+int numrbls, numthreads, numqueries;
+
+void do_queries () {
+ iplist tIP;
+ lwres_context_t *ctx = NULL;
+ lwres_grbnresponse_t *response = NULL;
+ int n, i;
+
+
+ pthread_mutex_lock(mutexp);
+ tIP = IPs;
+ if (IPs != NULL) {
+ IPs = tIP->next;
+ }
+ pthread_mutex_unlock(mutexp);
+
+ while (tIP != NULL) {
+//fprintf (stderr, "making query %s\n", tIP->IP); fflush(stderr);
+ if (lwres_context_create(&ctx, NULL, NULL, NULL, 0) != 0) {
+ fprintf (stderr, "Couldn't create context\n");
+ return;
+ } else {
+ lwres_conf_parse(ctx, lwres_resolv_conf);
+ //pthread_mutex_lock(mutexoutput);
+ n = lwres_getrdatabyname(ctx, tIP->IP, ns_c_in, ns_t_a, 0, &response);
+ //pthread_mutex_unlock(mutexoutput);
+ if (n == LWRES_R_SUCCESS) {
+ printf ("%s,%d.%d.%d.%d,%d\n", tIP->IP,
+ response->rdatas[0][0], response->rdatas[0][1],
+ response->rdatas[0][2], response->rdatas[0][3],
+ response->ttl);
+ //fprintf (stderr, "freeing response\n"); fflush(stderr);
+ lwres_grbnresponse_free(ctx, &response);
+ } else {
+ //fprintf (stderr, "Nothing found\n");
+ printf ("%s, %s, %d\n", tIP->IP, tIP->IP, DEFTTL);
+ }
+ //fprintf (stderr, "freeing context\n"); fflush(stderr);
+ lwres_context_destroy(&ctx);
+ //fprintf (stderr, "done freeing\n"); fflush(stderr);
+ }
+
+ pthread_mutex_lock(mutexp);
+ tIP = IPs;
+ if (IPs != NULL) {
+ IPs = tIP->next;
+ }
+ pthread_mutex_unlock(mutexp);
+ }
+}
+
+void GetRBLs() {
+ char instr[MAXSTR];
+ numrbls = 0;
+ while ((fgets(instr, MAXSTR, stdin) != NULL) && (numrbls < MAXRBLS)) {
+ instr[strlen(instr)-1] = 0; // strip off newline
+ if (strncmp(instr, "----------", 10) == 0) {
+ return;
+ }
+ dnsrbls[numrbls] = (char *) malloc(strlen(instr)+1);
+ if (dnsrbls[numrbls] == NULL) {
+ fprintf (stderr, "Couldn't allocate memory for %d DNS RBLs\n", numrbls);
+ exit (10);
+ } else {
+ strcpy (dnsrbls[numrbls], instr);
+ numrbls++;
+ }
+ }
+}
+
+
+main () {
+ pthread_t threads[MAXTHREADS];
+ char instr[MAXSTR];
+ iplist tIP;
+ int loop1;
+
+ GetRBLs();
+
+// for (loop1=0; loop1<numrbls; loop1++)
+// fprintf (stderr, "%s\n", dnsrbls[loop1]);
+// fprintf (stderr, "----------\n");
+
+ numqueries = 0;
+ IPs = NULL;
+ while (fgets(instr, MAXSTR, stdin) != NULL) {
+ instr[strlen(instr)-1] = 0;
+ for (loop1 = 0; loop1 < numrbls; loop1++) {
+ tIP = (iplist)malloc(sizeof(struct ipnode));
+ tIP->IP = (char *)malloc(strlen(instr)+strlen(dnsrbls[loop1])+2);
+ strcpy (tIP->IP, instr);
+ strcat (tIP->IP, dnsrbls[loop1]);
+ tIP->next = IPs;
+ IPs = tIP;
+ numqueries++;
+ }
+ }
+
+// fprintf (stderr, "%d queries to make\n", numqueries);
+// tIP = IPs;
+// while (tIP != NULL) {
+// fprintf (stderr, "%s\n", tIP->IP);
+// tIP = tIP->next;
+// }
+// fprintf (stderr, "done\n");
+// exit (0);
+
+ mutexp=(pthread_mutex_t *) malloc(sizeof(pthread_mutex_t));
+ pthread_mutex_init(mutexp, NULL);
+ mutexoutput=(pthread_mutex_t *) malloc(sizeof(pthread_mutex_t));
+ pthread_mutex_init(mutexoutput, NULL);
+
+ numthreads = 0; // number of threads created successfully
+ for (loop1 = 0; ((loop1<MAXTHREADS) && (loop1<numqueries)); loop1++) {
+ if (pthread_create(&threads[loop1], NULL,
+ (void *) do_queries, NULL) != 0) {
+ fprintf (stderr, "Couldn't make more than %d threads\n", numthreads);
+ break;
+ } else {
+ numthreads++;
+ }
+ }
+
+ for (loop1 = 0; loop1 < numthreads ; loop1++) {
+ pthread_join(threads[loop1], NULL);
+ }
+
+//do_queries();
+
+ exit (0);
+}
+
projects / squirrelmail.git / commitdiff