Use the BDB "environment" so that a database config file is not looked for. Bug...
[exim.git] / src / src / dbfn.c
1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
4
5 /* Copyright (c) University of Cambridge 1995 - 2015 */
6 /* See the file NOTICE for conditions of use and distribution. */
7
8
9 #include "exim.h"
10
11
12 /* Functions for accessing Exim's hints database, which consists of a number of
13 different DBM files. This module does not contain code for reading DBM files
14 for (e.g.) alias expansion. That is all contained within the general search
15 functions. As Exim now has support for several DBM interfaces, all the relevant
16 functions are called as macros.
17
18 All the data in Exim's database is in the nature of *hints*. Therefore it
19 doesn't matter if it gets destroyed by accident. These functions are not
20 supposed to implement a "safe" database.
21
22 Keys are passed in as C strings, and the terminating zero *is* used when
23 building the dbm files. This just makes life easier when scanning the files
24 sequentially.
25
26 Synchronization is required on the database files, and this is achieved by
27 means of locking on independent lock files. (Earlier attempts to lock on the
28 DBM files themselves were never completely successful.) Since callers may in
29 general want to do more than one read or write while holding the lock, there
30 are separate open and close functions. However, the calling modules should
31 arrange to hold the locks for the bare minimum of time. */
32
33
34
35 /*************************************************
36 * Berkeley DB error callback *
37 *************************************************/
38
39 /* For Berkeley DB >= 2, we can define a function to be called in case of DB
40 errors. This should help with debugging strange DB problems, e.g. getting "File
41 exists" when you try to open a db file. The API for this function was changed
42 at DB release 4.3. */
43
44 #if defined(USE_DB) && defined(DB_VERSION_STRING)
45 void
46 #if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3)
47 dbfn_bdb_error_callback(const DB_ENV *dbenv, const char *pfx, const char *msg)
48 {
49 dbenv = dbenv;
50 #else
51 dbfn_bdb_error_callback(const char *pfx, char *msg)
52 {
53 #endif
54 pfx = pfx;
55 log_write(0, LOG_MAIN, "Berkeley DB error: %s", msg);
56 }
57 #endif
58
59
60
61
62 /*************************************************
63 * Open and lock a database file *
64 *************************************************/
65
66 /* Used for accessing Exim's hints databases.
67
68 Arguments:
69 name The single-component name of one of Exim's database files.
70 flags Either O_RDONLY or O_RDWR, indicating the type of open required;
71 O_RDWR implies "create if necessary"
72 dbblock Points to an open_db block to be filled in.
73 lof If TRUE, write to the log for actual open failures (locking failures
74 are always logged).
75
76 Returns: NULL if the open failed, or the locking failed. After locking
77 failures, errno is zero.
78
79 On success, dbblock is returned. This contains the dbm pointer and
80 the fd of the locked lock file.
81
82 There are some calls that use O_RDWR|O_CREAT for the flags. Having discovered
83 this in December 2005, I'm not sure if this is correct or not, but for the
84 moment I haven't changed them.
85 */
86
87 open_db *
88 dbfn_open(uschar *name, int flags, open_db *dbblock, BOOL lof)
89 {
90 int rc, save_errno;
91 BOOL read_only = flags == O_RDONLY;
92 BOOL created = FALSE;
93 flock_t lock_data;
94 uschar dirname[256], filename[256];
95
96 /* The first thing to do is to open a separate file on which to lock. This
97 ensures that Exim has exclusive use of the database before it even tries to
98 open it. Early versions tried to lock on the open database itself, but that
99 gave rise to mysterious problems from time to time - it was suspected that some
100 DB libraries "do things" on their open() calls which break the interlocking.
101 The lock file is never written to, but we open it for writing so we can get a
102 write lock if required. If it does not exist, we create it. This is done
103 separately so we know when we have done it, because when running as root we
104 need to change the ownership - see the bottom of this function. We also try to
105 make the directory as well, just in case. We won't be doing this many times
106 unnecessarily, because usually the lock file will be there. If the directory
107 exists, there is no error. */
108
109 snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
110 snprintf(CS filename, sizeof(filename), "%s/%s.lockfile", dirname, name);
111
112 if ((dbblock->lockfd = Uopen(filename, O_RDWR, EXIMDB_LOCKFILE_MODE)) < 0)
113 {
114 created = TRUE;
115 (void)directory_make(spool_directory, US"db", EXIMDB_DIRECTORY_MODE, TRUE);
116 dbblock->lockfd = Uopen(filename, O_RDWR|O_CREAT, EXIMDB_LOCKFILE_MODE);
117 }
118
119 if (dbblock->lockfd < 0)
120 {
121 log_write(0, LOG_MAIN, "%s",
122 string_open_failed(errno, "database lock file %s", filename));
123 errno = 0; /* Indicates locking failure */
124 return NULL;
125 }
126
127 /* Now we must get a lock on the opened lock file; do this with a blocking
128 lock that times out. */
129
130 lock_data.l_type = read_only? F_RDLCK : F_WRLCK;
131 lock_data.l_whence = lock_data.l_start = lock_data.l_len = 0;
132
133 DEBUG(D_hints_lookup|D_retry|D_route|D_deliver)
134 debug_printf("locking %s\n", filename);
135
136 sigalrm_seen = FALSE;
137 alarm(EXIMDB_LOCK_TIMEOUT);
138 rc = fcntl(dbblock->lockfd, F_SETLKW, &lock_data);
139 alarm(0);
140
141 if (sigalrm_seen) errno = ETIMEDOUT;
142 if (rc < 0)
143 {
144 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to get %s lock for %s: %s",
145 read_only ? "read" : "write", filename,
146 errno == ETIMEDOUT ? "timed out" : strerror(errno));
147 (void)close(dbblock->lockfd);
148 errno = 0; /* Indicates locking failure */
149 return NULL;
150 }
151
152 DEBUG(D_hints_lookup) debug_printf("locked %s\n", filename);
153
154 /* At this point we have an opened and locked separate lock file, that is,
155 exclusive access to the database, so we can go ahead and open it. If we are
156 expected to create it, don't do so at first, again so that we can detect
157 whether we need to change its ownership (see comments about the lock file
158 above.) There have been regular reports of crashes while opening hints
159 databases - often this is caused by non-matching db.h and the library. To make
160 it easy to pin this down, there are now debug statements on either side of the
161 open call. */
162
163 snprintf(CS filename, sizeof(filename), "%s/%s", dirname, name);
164 EXIM_DBOPEN(filename, dirname, flags, EXIMDB_MODE, &(dbblock->dbptr));
165
166 if (!dbblock->dbptr && errno == ENOENT && flags == O_RDWR)
167 {
168 DEBUG(D_hints_lookup)
169 debug_printf("%s appears not to exist: trying to create\n", filename);
170 created = TRUE;
171 EXIM_DBOPEN(filename, dirname, flags|O_CREAT, EXIMDB_MODE, &(dbblock->dbptr));
172 }
173
174 save_errno = errno;
175
176 /* If we are running as root and this is the first access to the database, its
177 files will be owned by root. We want them to be owned by exim. We detect this
178 situation by noting above when we had to create the lock file or the database
179 itself. Because the different dbm libraries use different extensions for their
180 files, I don't know of any easier way of arranging this than scanning the
181 directory for files with the appropriate base name. At least this deals with
182 the lock file at the same time. Also, the directory will typically have only
183 half a dozen files, so the scan will be quick.
184
185 This code is placed here, before the test for successful opening, because there
186 was a case when a file was created, but the DBM library still returned NULL
187 because of some problem. It also sorts out the lock file if that was created
188 but creation of the database file failed. */
189
190 if (created && geteuid() == root_uid)
191 {
192 DIR *dd;
193 struct dirent *ent;
194 uschar *lastname = Ustrrchr(filename, '/') + 1;
195 int namelen = Ustrlen(name);
196
197 *lastname = 0;
198 dd = opendir(CS filename);
199
200 while ((ent = readdir(dd)))
201 if (Ustrncmp(ent->d_name, name, namelen) == 0)
202 {
203 struct stat statbuf;
204 Ustrcpy(lastname, ent->d_name);
205 if (Ustat(filename, &statbuf) >= 0 && statbuf.st_uid != exim_uid)
206 {
207 DEBUG(D_hints_lookup) debug_printf("ensuring %s is owned by exim\n", filename);
208 if (Uchown(filename, exim_uid, exim_gid))
209 DEBUG(D_hints_lookup) debug_printf("failed setting %s to owned by exim\n", filename);
210 }
211 }
212
213 closedir(dd);
214 }
215
216 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
217 log the event - also for debugging - but debug only if the file just doesn't
218 exist. */
219
220 if (!dbblock->dbptr)
221 {
222 if (lof && save_errno != ENOENT)
223 log_write(0, LOG_MAIN, "%s", string_open_failed(save_errno, "DB file %s",
224 filename));
225 else
226 DEBUG(D_hints_lookup)
227 debug_printf("%s\n", CS string_open_failed(save_errno, "DB file %s",
228 filename));
229 (void)close(dbblock->lockfd);
230 errno = save_errno;
231 return NULL;
232 }
233
234 DEBUG(D_hints_lookup)
235 debug_printf("opened hints database %s: flags=%s\n", filename,
236 flags == O_RDONLY ? "O_RDONLY"
237 : flags == O_RDWR ? "O_RDWR"
238 : flags == (O_RDWR|O_CREAT) ? "O_RDWR|O_CREAT"
239 : "??");
240
241 /* Pass back the block containing the opened database handle and the open fd
242 for the lock. */
243
244 return dbblock;
245 }
246
247
248
249
250 /*************************************************
251 * Unlock and close a database file *
252 *************************************************/
253
254 /* Closing a file automatically unlocks it, so after closing the database, just
255 close the lock file.
256
257 Argument: a pointer to an open database block
258 Returns: nothing
259 */
260
261 void
262 dbfn_close(open_db *dbblock)
263 {
264 EXIM_DBCLOSE(dbblock->dbptr);
265 (void)close(dbblock->lockfd);
266 DEBUG(D_hints_lookup) debug_printf("closed hints database and lockfile\n");
267 }
268
269
270
271
272 /*************************************************
273 * Read from database file *
274 *************************************************/
275
276 /* Passing back the pointer unchanged is useless, because there is
277 no guarantee of alignment. Since all the records used by Exim need
278 to be properly aligned to pick out the timestamps, etc., we might as
279 well do the copying centrally here.
280
281 Most calls don't need the length, so there is a macro called dbfn_read which
282 has two arguments; it calls this function adding NULL as the third.
283
284 Arguments:
285 dbblock a pointer to an open database block
286 key the key of the record to be read
287 length a pointer to an int into which to return the length, if not NULL
288
289 Returns: a pointer to the retrieved record, or
290 NULL if the record is not found
291 */
292
293 void *
294 dbfn_read_with_length(open_db *dbblock, const uschar *key, int *length)
295 {
296 void *yield;
297 EXIM_DATUM key_datum, result_datum;
298 int klen = Ustrlen(key) + 1;
299 uschar * key_copy = store_get(klen);
300
301 memcpy(key_copy, key, klen);
302
303 DEBUG(D_hints_lookup) debug_printf("dbfn_read: key=%s\n", key);
304
305 EXIM_DATUM_INIT(key_datum); /* Some DBM libraries require the datum */
306 EXIM_DATUM_INIT(result_datum); /* to be cleared before use. */
307 EXIM_DATUM_DATA(key_datum) = CS key_copy;
308 EXIM_DATUM_SIZE(key_datum) = klen;
309
310 if (!EXIM_DBGET(dbblock->dbptr, key_datum, result_datum)) return NULL;
311
312 yield = store_get(EXIM_DATUM_SIZE(result_datum));
313 memcpy(yield, EXIM_DATUM_DATA(result_datum), EXIM_DATUM_SIZE(result_datum));
314 if (length != NULL) *length = EXIM_DATUM_SIZE(result_datum);
315
316 EXIM_DATUM_FREE(result_datum); /* Some DBM libs require freeing */
317 return yield;
318 }
319
320
321
322 /*************************************************
323 * Write to database file *
324 *************************************************/
325
326 /*
327 Arguments:
328 dbblock a pointer to an open database block
329 key the key of the record to be written
330 ptr a pointer to the record to be written
331 length the length of the record to be written
332
333 Returns: the yield of the underlying dbm or db "write" function. If this
334 is dbm, the value is zero for OK.
335 */
336
337 int
338 dbfn_write(open_db *dbblock, const uschar *key, void *ptr, int length)
339 {
340 EXIM_DATUM key_datum, value_datum;
341 dbdata_generic *gptr = (dbdata_generic *)ptr;
342 int klen = Ustrlen(key) + 1;
343 uschar * key_copy = store_get(klen);
344
345 memcpy(key_copy, key, klen);
346 gptr->time_stamp = time(NULL);
347
348 DEBUG(D_hints_lookup) debug_printf("dbfn_write: key=%s\n", key);
349
350 EXIM_DATUM_INIT(key_datum); /* Some DBM libraries require the datum */
351 EXIM_DATUM_INIT(value_datum); /* to be cleared before use. */
352 EXIM_DATUM_DATA(key_datum) = CS key_copy;
353 EXIM_DATUM_SIZE(key_datum) = klen;
354 EXIM_DATUM_DATA(value_datum) = CS ptr;
355 EXIM_DATUM_SIZE(value_datum) = length;
356 return EXIM_DBPUT(dbblock->dbptr, key_datum, value_datum);
357 }
358
359
360
361 /*************************************************
362 * Delete record from database file *
363 *************************************************/
364
365 /*
366 Arguments:
367 dbblock a pointer to an open database block
368 key the key of the record to be deleted
369
370 Returns: the yield of the underlying dbm or db "delete" function.
371 */
372
373 int
374 dbfn_delete(open_db *dbblock, const uschar *key)
375 {
376 int klen = Ustrlen(key) + 1;
377 uschar * key_copy = store_get(klen);
378
379 memcpy(key_copy, key, klen);
380 EXIM_DATUM key_datum;
381 EXIM_DATUM_INIT(key_datum); /* Some DBM libraries require clearing */
382 EXIM_DATUM_DATA(key_datum) = CS key_copy;
383 EXIM_DATUM_SIZE(key_datum) = klen;
384 return EXIM_DBDEL(dbblock->dbptr, key_datum);
385 }
386
387
388
389 /*************************************************
390 * Scan the keys of a database file *
391 *************************************************/
392
393 /*
394 Arguments:
395 dbblock a pointer to an open database block
396 start TRUE if starting a new scan
397 FALSE if continuing with the current scan
398 cursor a pointer to a pointer to a cursor anchor, for those dbm libraries
399 that use the notion of a cursor
400
401 Returns: the next record from the file, or
402 NULL if there are no more
403 */
404
405 uschar *
406 dbfn_scan(open_db *dbblock, BOOL start, EXIM_CURSOR **cursor)
407 {
408 EXIM_DATUM key_datum, value_datum;
409 uschar *yield;
410 value_datum = value_datum; /* dummy; not all db libraries use this */
411
412 /* Some dbm require an initialization */
413
414 if (start) EXIM_DBCREATE_CURSOR(dbblock->dbptr, cursor);
415
416 EXIM_DATUM_INIT(key_datum); /* Some DBM libraries require the datum */
417 EXIM_DATUM_INIT(value_datum); /* to be cleared before use. */
418
419 yield = (EXIM_DBSCAN(dbblock->dbptr, key_datum, value_datum, start, *cursor))?
420 US EXIM_DATUM_DATA(key_datum) : NULL;
421
422 /* Some dbm require a termination */
423
424 if (!yield) EXIM_DBDELETE_CURSOR(*cursor);
425 return yield;
426 }
427
428
429
430 /*************************************************
431 **************************************************
432 * Stand-alone test program *
433 **************************************************
434 *************************************************/
435
436 #ifdef STAND_ALONE
437
438 int
439 main(int argc, char **cargv)
440 {
441 open_db dbblock[8];
442 int max_db = sizeof(dbblock)/sizeof(open_db);
443 int current = -1;
444 int showtime = 0;
445 int i;
446 dbdata_wait *dbwait = NULL;
447 uschar **argv = USS cargv;
448 uschar buffer[256];
449 uschar structbuffer[1024];
450
451 if (argc != 2)
452 {
453 printf("Usage: test_dbfn directory\n");
454 printf("The subdirectory called \"db\" in the given directory is used for\n");
455 printf("the files used in this test program.\n");
456 return 1;
457 }
458
459 /* Initialize */
460
461 spool_directory = argv[1];
462 debug_selector = D_all - D_memory;
463 debug_file = stderr;
464 big_buffer = malloc(big_buffer_size);
465
466 for (i = 0; i < max_db; i++) dbblock[i].dbptr = NULL;
467
468 printf("\nExim's db functions tester: interface type is %s\n", EXIM_DBTYPE);
469 printf("DBM library: ");
470
471 #ifdef DB_VERSION_STRING
472 printf("Berkeley DB: %s\n", DB_VERSION_STRING);
473 #elif defined(BTREEVERSION) && defined(HASHVERSION)
474 #ifdef USE_DB
475 printf("probably Berkeley DB version 1.8x (native mode)\n");
476 #else
477 printf("probably Berkeley DB version 1.8x (compatibility mode)\n");
478 #endif
479 #elif defined(_DBM_RDONLY) || defined(dbm_dirfno)
480 printf("probably ndbm\n");
481 #elif defined(USE_TDB)
482 printf("using tdb\n");
483 #else
484 #ifdef USE_GDBM
485 printf("probably GDBM (native mode)\n");
486 #else
487 printf("probably GDBM (compatibility mode)\n");
488 #endif
489 #endif
490
491 /* Test the functions */
492
493 printf("\nTest the functions\n> ");
494
495 while (Ufgets(buffer, 256, stdin) != NULL)
496 {
497 int len = Ustrlen(buffer);
498 int count = 1;
499 clock_t start = 1;
500 clock_t stop = 0;
501 uschar *cmd = buffer;
502 while (len > 0 && isspace((uschar)buffer[len-1])) len--;
503 buffer[len] = 0;
504
505 if (isdigit((uschar)*cmd))
506 {
507 count = Uatoi(cmd);
508 while (isdigit((uschar)*cmd)) cmd++;
509 while (isspace((uschar)*cmd)) cmd++;
510 }
511
512 if (Ustrncmp(cmd, "open", 4) == 0)
513 {
514 int i;
515 open_db *odb;
516 uschar *s = cmd + 4;
517 while (isspace((uschar)*s)) s++;
518
519 for (i = 0; i < max_db; i++)
520 if (dbblock[i].dbptr == NULL) break;
521
522 if (i >= max_db)
523 {
524 printf("Too many open databases\n> ");
525 continue;
526 }
527
528 start = clock();
529 odb = dbfn_open(s, O_RDWR, dbblock + i, TRUE);
530 stop = clock();
531
532 if (odb)
533 {
534 current = i;
535 printf("opened %d\n", current);
536 }
537 /* Other error cases will have written messages */
538 else if (errno == ENOENT)
539 {
540 printf("open failed: %s%s\n", strerror(errno),
541 #ifdef USE_DB
542 " (or other Berkeley DB error)"
543 #else
544 ""
545 #endif
546 );
547 }
548 }
549
550 else if (Ustrncmp(cmd, "write", 5) == 0)
551 {
552 int rc = 0;
553 uschar *key = cmd + 5;
554 uschar *data;
555
556 if (current < 0)
557 {
558 printf("No current database\n");
559 continue;
560 }
561
562 while (isspace((uschar)*key)) key++;
563 data = key;
564 while (*data != 0 && !isspace((uschar)*data)) data++;
565 *data++ = 0;
566 while (isspace((uschar)*data)) data++;
567
568 dbwait = (dbdata_wait *)(&structbuffer);
569 Ustrcpy(dbwait->text, data);
570
571 start = clock();
572 while (count-- > 0)
573 rc = dbfn_write(dbblock + current, key, dbwait,
574 Ustrlen(data) + sizeof(dbdata_wait));
575 stop = clock();
576 if (rc != 0) printf("Failed: %s\n", strerror(errno));
577 }
578
579 else if (Ustrncmp(cmd, "read", 4) == 0)
580 {
581 uschar *key = cmd + 4;
582 if (current < 0)
583 {
584 printf("No current database\n");
585 continue;
586 }
587 while (isspace((uschar)*key)) key++;
588 start = clock();
589 while (count-- > 0)
590 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock+ current, key, NULL);
591 stop = clock();
592 printf("%s\n", (dbwait == NULL)? "<not found>" : CS dbwait->text);
593 }
594
595 else if (Ustrncmp(cmd, "delete", 6) == 0)
596 {
597 uschar *key = cmd + 6;
598 if (current < 0)
599 {
600 printf("No current database\n");
601 continue;
602 }
603 while (isspace((uschar)*key)) key++;
604 dbfn_delete(dbblock + current, key);
605 }
606
607 else if (Ustrncmp(cmd, "scan", 4) == 0)
608 {
609 EXIM_CURSOR *cursor;
610 BOOL startflag = TRUE;
611 uschar *key;
612 uschar keybuffer[256];
613 if (current < 0)
614 {
615 printf("No current database\n");
616 continue;
617 }
618 start = clock();
619 while ((key = dbfn_scan(dbblock + current, startflag, &cursor)) != NULL)
620 {
621 startflag = FALSE;
622 Ustrcpy(keybuffer, key);
623 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock + current,
624 keybuffer, NULL);
625 printf("%s: %s\n", keybuffer, dbwait->text);
626 }
627 stop = clock();
628 printf("End of scan\n");
629 }
630
631 else if (Ustrncmp(cmd, "close", 5) == 0)
632 {
633 uschar *s = cmd + 5;
634 while (isspace((uschar)*s)) s++;
635 i = Uatoi(s);
636 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n"); else
637 {
638 start = clock();
639 dbfn_close(dbblock + i);
640 stop = clock();
641 dbblock[i].dbptr = NULL;
642 if (i == current) current = -1;
643 }
644 }
645
646 else if (Ustrncmp(cmd, "file", 4) == 0)
647 {
648 uschar *s = cmd + 4;
649 while (isspace((uschar)*s)) s++;
650 i = Uatoi(s);
651 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n");
652 else current = i;
653 }
654
655 else if (Ustrncmp(cmd, "time", 4) == 0)
656 {
657 showtime = ~showtime;
658 printf("Timing %s\n", showtime? "on" : "off");
659 }
660
661 else if (Ustrcmp(cmd, "q") == 0 || Ustrncmp(cmd, "quit", 4) == 0) break;
662
663 else if (Ustrncmp(cmd, "help", 4) == 0)
664 {
665 printf("close [<number>] close file [<number>]\n");
666 printf("delete <key> remove record from current file\n");
667 printf("file <number> make file <number> current\n");
668 printf("open <name> open db file\n");
669 printf("q[uit] exit program\n");
670 printf("read <key> read record from current file\n");
671 printf("scan scan current file\n");
672 printf("time time display on/off\n");
673 printf("write <key> <rest-of-line> write record to current file\n");
674 }
675
676 else printf("Eh?\n");
677
678 if (showtime && stop >= start)
679 printf("start=%d stop=%d difference=%d\n", (int)start, (int)stop,
680 (int)(stop - start));
681
682 printf("> ");
683 }
684
685 for (i = 0; i < max_db; i++)
686 {
687 if (dbblock[i].dbptr != NULL)
688 {
689 printf("\nClosing %d", i);
690 dbfn_close(dbblock + i);
691 }
692 }
693
694 printf("\n");
695 return 0;
696 }
697
698 #endif
699
700 /* End of dbfn.c */