src/src/transports/tf_maildir.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2017 */
   6 /* See the file NOTICE for conditions of use and distribution. */
   7
   8 /* Functions in support of the use of maildirsize files for handling quotas in
   9 maildir directories. Some of the rules are a bit baroque:
  10
  11 http://www.inter7.com/courierimap/README.maildirquota.html
  12
  13 We try to follow most of that, except that the directories to skip for quota
  14 calculations are not hard wired in, but are supplied as a regex. */
  15
  16
  17 #include "../exim.h"
  18 #include "appendfile.h"
  19 #include "tf_maildir.h"
  20
  21 #define MAX_FILE_SIZE  5120
  22
  23
  24
  25 /*************************************************
  26 *      Ensure maildir directories exist          *
  27 *************************************************/
  28
  29 /* This function is called at the start of a maildir delivery, to ensure that
  30 all the relevant directories exist. It also creates a maildirfolder file if the
  31 base directory matches a given pattern.
  32
  33 Argument:
  34   path              the base directory name
  35   addr              the address item (for setting an error message)
  36   create_directory  true if we are allowed to create missing directories
  37   dirmode           the mode for created directories
  38   maildirfolder_create_regex
  39                     the pattern to match for maildirfolder creation
  40
  41 Returns:            TRUE on success; FALSE on failure
  42 */
  43
  44 BOOL maildir_ensure_directories(uschar *path, address_item *addr,
  45   BOOL create_directory, int dirmode, uschar *maildirfolder_create_regex)
  46 {
  47 int i;
  48 struct stat statbuf;
  49 const char *subdirs[] = { "/tmp", "/new", "/cur" };
  50
  51 DEBUG(D_transport)
  52   debug_printf("ensuring maildir directories exist in %s\n", path);
  53
  54 /* First ensure that the path we have is a directory; if it does not exist,
  55 create it. Then make sure the tmp, new & cur subdirs of the maildir are
  56 there. If not, fail. This aborts the delivery (even though the cur subdir is
  57 not actually needed for delivery). Handle all 4 directory tests/creates in a
  58 loop so that code can be shared. */
  59
  60 for (i = 0; i < 4; i++)
  61   {
  62   int j;
  63   const uschar *dir, *mdir;
  64
  65   if (i == 0)
  66     {
  67     mdir = CUS"";
  68     dir = path;
  69     }
  70   else
  71     {
  72     mdir = CUS subdirs[i-1];
  73     dir = mdir + 1;
  74     }
  75
  76   /* Check an existing path is a directory. This is inside a loop because
  77   there is a potential race condition when creating the directory - some
  78   other process may get there first. Give up after trying several times,
  79   though. */
  80
  81   for (j = 0; j < 10; j++)
  82     {
  83     if (Ustat(dir, &statbuf) == 0)
  84       {
  85       if (S_ISDIR(statbuf.st_mode)) break;   /* out of the race loop */
  86       addr->message = string_sprintf("%s%s is not a directory", path,
  87         mdir);
  88       addr->basic_errno = ERRNO_NOTDIRECTORY;
  89       return FALSE;
  90       }
  91
  92     /* Try to make if non-existent and configured to do so */
  93
  94     if (errno == ENOENT && create_directory)
  95       {
  96       if (!directory_make(NULL, dir, dirmode, FALSE))
  97         {
  98         if (errno == EEXIST) continue;     /* repeat the race loop */
  99         addr->message = string_sprintf("cannot create %s%s", path, mdir);
 100         addr->basic_errno = errno;
 101         return FALSE;
 102         }
 103       DEBUG(D_transport)
 104         debug_printf("created directory %s%s\n", path, mdir);
 105       break;   /* out of the race loop */
 106       }
 107
 108     /* stat() error other than ENOENT, or ENOENT and not creatable */
 109
 110     addr->message = string_sprintf("stat() error for %s%s: %s", path, mdir,
 111       strerror(errno));
 112     addr->basic_errno = errno;
 113     return FALSE;
 114     }
 115
 116   /* If we went round the loop 10 times, the directory was flickering in
 117   and out of existence like someone in a malfunctioning Star Trek
 118   transporter. */
 119
 120   if (j >= 10)
 121     {
 122     addr->message = string_sprintf("existence of %s%s unclear\n", path,
 123       mdir);
 124     addr->basic_errno = errno;
 125     addr->special_action = SPECIAL_FREEZE;
 126     return FALSE;
 127     }
 128
 129   /* First time through the directories loop, cd to the main directory */
 130
 131   if (i == 0 && Uchdir(path) != 0)
 132     {
 133     addr->message = string_sprintf ("cannot chdir to %s", path);
 134     addr->basic_errno = errno;
 135     return FALSE;
 136     }
 137   }
 138
 139 /* If the basic path matches maildirfolder_create_regex, we are dealing with
 140 a subfolder, and should ensure that a maildirfolder file exists. */
 141
 142 if (maildirfolder_create_regex != NULL)
 143   {
 144   const uschar *error;
 145   int offset;
 146   const pcre *regex;
 147
 148   DEBUG(D_transport) debug_printf("checking for maildirfolder requirement\n");
 149
 150   regex = pcre_compile(CS maildirfolder_create_regex, PCRE_COPT,
 151     (const char **)&error, &offset, NULL);
 152
 153   if (regex == NULL)
 154     {
 155     addr->message = string_sprintf("appendfile: regular expression "
 156       "error: %s at offset %d while compiling %s", error, offset,
 157       maildirfolder_create_regex);
 158     return FALSE;
 159     }
 160
 161   if (pcre_exec(regex, NULL, CS path, Ustrlen(path), 0, 0, NULL, 0) >= 0)
 162     {
 163     uschar *fname = string_sprintf("%s/maildirfolder", path);
 164     if (Ustat(fname, &statbuf) == 0)
 165       {
 166       DEBUG(D_transport) debug_printf("maildirfolder already exists\n");
 167       }
 168     else
 169       {
 170       int fd = Uopen(fname, O_WRONLY|O_APPEND|O_CREAT, 0600);
 171       if (fd < 0)
 172         {
 173         addr->message = string_sprintf("appendfile: failed to create "
 174           "maildirfolder file in %s directory: %s", path, strerror(errno));
 175         return FALSE;
 176         }
 177       (void)close(fd);
 178       DEBUG(D_transport) debug_printf("created maildirfolder file\n");
 179       }
 180     }
 181   else
 182     {
 183     DEBUG(D_transport) debug_printf("maildirfolder file not required\n");
 184     }
 185   }
 186
 187 return TRUE;   /* Everything exists that should exist */
 188 }
 189
 190
 191
 192
 193 /*************************************************
 194 *       Update maildirsizefile for new file      *
 195 *************************************************/
 196
 197 /* This function is called to add a new line to the file, recording the length
 198 of the newly added message. There isn't much we can do on failure...
 199
 200 Arguments:
 201   fd           the open file descriptor
 202   size         the size of the message
 203
 204 Returns:       nothing
 205 */
 206
 207 void
 208 maildir_record_length(int fd, int size)
 209 {
 210 int len;
 211 uschar buffer[256];
 212 sprintf(CS buffer, "%d 1\n", size);
 213 len = Ustrlen(buffer);
 214 if (lseek(fd, 0, SEEK_END) >= 0)
 215   {
 216   len = write(fd, buffer, len);
 217   DEBUG(D_transport)
 218     debug_printf("added '%.*s' to maildirsize file\n", len-1, buffer);
 219   }
 220 }
 221
 222
 223
 224 /*************************************************
 225 *          Find the size of a maildir            *
 226 *************************************************/
 227
 228 /* This function is called when we have to recalculate the size of a maildir by
 229 scanning all the files and directories therein. There are rules and conventions
 230 about which files or directories are included. We support this by the use of a
 231 regex to match directories that are to be included.
 232
 233 Maildirs can only be one level deep. However, this function recurses, so it
 234 might cope with deeper nestings. We use the existing check_dir_size() function
 235 to add up the sizes of the files in a directory that contains messages.
 236
 237 The function returns the most recent timestamp encountered. It can also be run
 238 in a dummy mode in which it does not scan for sizes, but just returns the
 239 timestamp.
 240
 241 Arguments:
 242   path            the path to the maildir
 243   filecount       where to store the count of messages
 244   latest          where to store the latest timestamp encountered
 245   regex           a regex for getting files sizes from file names
 246   dir_regex       a regex for matching directories to be included
 247   timestamp_only  don't actually compute any sizes
 248
 249 Returns:      the sum of the sizes of the messages
 250 */
 251
 252 off_t
 253 maildir_compute_size(uschar *path, int *filecount, time_t *latest,
 254   const pcre *regex, const pcre *dir_regex, BOOL timestamp_only)
 255 {
 256 DIR *dir;
 257 off_t sum = 0;
 258 struct dirent *ent;
 259 struct stat statbuf;
 260
 261 dir = opendir(CS path);
 262 if (dir == NULL) return 0;
 263
 264 while ((ent = readdir(dir)) != NULL)
 265   {
 266   uschar *name = US ent->d_name;
 267   uschar buffer[1024];
 268
 269   if (Ustrcmp(name, ".") == 0 || Ustrcmp(name, "..") == 0) continue;
 270
 271   /* We are normally supplied with a regex for choosing which directories to
 272   scan. We do the regex match first, because that avoids a stat() for names
 273   we aren't interested in. */
 274
 275   if (dir_regex != NULL &&
 276       pcre_exec(dir_regex, NULL, CS name, Ustrlen(name), 0, 0, NULL, 0) < 0)
 277     {
 278     DEBUG(D_transport)
 279       debug_printf("skipping %s/%s: dir_regex does not match\n", path, name);
 280     continue;
 281     }
 282
 283   /* The name is OK; stat it. */
 284
 285   if (!string_format(buffer, sizeof(buffer), "%s/%s", path, name))
 286     {
 287     DEBUG(D_transport)
 288       debug_printf("maildir_compute_size: name too long: dir=%s name=%s\n",
 289         path, name);
 290     continue;
 291     }
 292
 293   if (Ustat(buffer, &statbuf) < 0)
 294     {
 295     DEBUG(D_transport)
 296       debug_printf("maildir_compute_size: stat error %d for %s: %s\n", errno,
 297         buffer, strerror(errno));
 298     continue;
 299     }
 300
 301   if ((statbuf.st_mode & S_IFMT) != S_IFDIR)
 302     {
 303     DEBUG(D_transport)
 304       debug_printf("skipping %s/%s: not a directory\n", path, name);
 305     continue;
 306     }
 307
 308   /* Keep the latest timestamp encountered */
 309
 310   if (statbuf.st_mtime > *latest) *latest = statbuf.st_mtime;
 311
 312   /* If this is a maildir folder, call this function recursively. */
 313
 314   if (name[0] == '.')
 315     {
 316     sum += maildir_compute_size(buffer, filecount, latest, regex, dir_regex,
 317       timestamp_only);
 318     }
 319
 320   /* Otherwise it must be a folder that contains messages (e.g. new or cur), so
 321   we need to get its size, unless all we are interested in is the timestamp. */
 322
 323   else if (!timestamp_only)
 324     {
 325     sum += check_dir_size(buffer, filecount, regex);
 326     }
 327   }
 328
 329 closedir(dir);
 330 DEBUG(D_transport)
 331   {
 332   if (timestamp_only)
 333     debug_printf("maildir_compute_size (timestamp_only): %ld\n",
 334     (long int) *latest);
 335   else
 336     debug_printf("maildir_compute_size: path=%s\n  sum=" OFF_T_FMT
 337       " filecount=%d timestamp=%ld\n",
 338       path, sum, *filecount, (long int) *latest);
 339   }
 340 return sum;
 341 }
 342
 343
 344
 345 /*************************************************
 346 *        Create or update maildirsizefile        *
 347 *************************************************/
 348
 349 /* This function is called before a delivery if the option to use
 350 maildirsizefile is enabled. Its function is to create the file if it does not
 351 exist, or to update it if that is necessary.
 352
 353 The logic in this function follows the rules that are described in
 354
 355   http://www.inter7.com/courierimap/README.maildirquota.html
 356
 357 Or, at least, it is supposed to!
 358
 359 Arguments:
 360   path             the path to the maildir directory; this is already backed-up
 361                      to the parent if the delivery directory is a maildirfolder
 362   ob               the appendfile options block
 363   regex            a compiled regex for getting a file's size from its name
 364   dir_regex        a compiled regex for selecting maildir directories
 365   returned_size    where to return the current size of the maildir, even if
 366                      the maildirsizefile is removed because of a race
 367
 368 Returns:           >=0  a file descriptor for an open maildirsize file
 369                    -1   there was an error opening or accessing the file
 370                    -2   the file was removed because of a race
 371 */
 372
 373 int
 374 maildir_ensure_sizefile(uschar *path, appendfile_transport_options_block *ob,
 375   const pcre *regex, const pcre *dir_regex, off_t *returned_size,
 376   int *returned_filecount)
 377 {
 378 int count, fd;
 379 off_t cached_quota = 0;
 380 int cached_quota_filecount = 0;
 381 int filecount = 0;
 382 int linecount = 0;
 383 off_t size = 0;
 384 uschar *filename;
 385 uschar buffer[MAX_FILE_SIZE];
 386 uschar *ptr = buffer;
 387 uschar *endptr;
 388
 389 /* Try a few times to open or create the file, in case another process is doing
 390 the same thing. */
 391
 392 filename = string_sprintf("%s/maildirsize", path);
 393
 394 DEBUG(D_transport) debug_printf("looking for maildirsize in %s\n", path);
 395 fd = Uopen(filename, O_RDWR|O_APPEND, ob->mode ? ob->mode : 0600);
 396 if (fd < 0)
 397   {
 398   if (errno != ENOENT) return -1;
 399   DEBUG(D_transport)
 400     debug_printf("%s does not exist: recalculating\n", filename);
 401   goto RECALCULATE;
 402   }
 403
 404 /* The file has been successfully opened. Check that the cached quota value is
 405 still correct, and that the size of the file is still small enough. If so,
 406 compute the maildir size from the file. */
 407
 408 count = read(fd, buffer, sizeof(buffer));
 409 if (count >= sizeof(buffer))
 410   {
 411   DEBUG(D_transport)
 412     debug_printf("maildirsize file too big (%d): recalculating\n", count);
 413   goto RECALCULATE;
 414   }
 415 buffer[count] = 0;   /* Ensure string terminated */
 416
 417 /* Read the quota parameters from the first line of the data. */
 418
 419 DEBUG(D_transport)
 420   debug_printf("reading quota parameters from maildirsize data\n");
 421
 422 for (;;)
 423   {
 424   off_t n = (off_t)Ustrtod(ptr, &endptr);
 425
 426   /* Only two data items are currently defined; ignore any others that
 427   may be present. The spec is for a number followed by a letter. Anything
 428   else we reject and recalculate. */
 429
 430   if (*endptr == 'S') cached_quota = n;
 431     else if (*endptr == 'C') cached_quota_filecount = (int)n;
 432   if (!isalpha(*endptr++))
 433     {
 434     DEBUG(D_transport)
 435       debug_printf("quota parameter number not followed by letter in "
 436         "\"%.*s\": recalculating maildirsize\n", (int)(endptr - buffer),
 437         buffer);
 438     goto RECALCULATE;
 439     }
 440   if (*endptr == '\n' || *endptr == 0) break;
 441   if (*endptr++ != ',')
 442     {
 443     DEBUG(D_transport)
 444       debug_printf("quota parameter not followed by comma in "
 445         "\"%.*s\": recalculating maildirsize\n", (int)(endptr - buffer),
 446         buffer);
 447     goto RECALCULATE;
 448     }
 449   ptr = endptr;
 450   }
 451
 452 /* Check the cached values against the current settings */
 453
 454 if (cached_quota != ob->quota_value ||
 455     cached_quota_filecount != ob->quota_filecount_value)
 456   {
 457   DEBUG(D_transport)
 458     debug_printf("cached quota is out of date: recalculating\n"
 459       "  quota=" OFF_T_FMT " cached_quota=" OFF_T_FMT " filecount_quota=%d "
 460       "cached_quota_filecount=%d\n", ob->quota_value,
 461       cached_quota, ob->quota_filecount_value, cached_quota_filecount);
 462   goto RECALCULATE;
 463   }
 464
 465 /* Quota values agree; parse the rest of the data to get the sizes. At this
 466 stage, *endptr points either to 0 or to '\n'.  */
 467
 468 DEBUG(D_transport)
 469   debug_printf("computing maildir size from maildirsize data\n");
 470
 471 while (*endptr++ == '\n')
 472   {
 473   if (*endptr == 0) break;
 474   linecount++;
 475   ptr = endptr;
 476   size += (off_t)Ustrtod(ptr, &endptr);
 477   if (*endptr != ' ') break;
 478   ptr = endptr + 1;
 479   filecount += Ustrtol(ptr, &endptr, 10);
 480   }
 481
 482 /* If *endptr is zero, we have successfully parsed the file, and we now have
 483 the size of the mailbox as cached in the file. The "rules" say that if this
 484 value indicates that the mailbox is over quota, we must recalculate if there is
 485 more than one entry in the file, or if the file is older than 15 minutes. Also,
 486 just in case there are weird values in the file, recalculate if either of the
 487 values is negative. */
 488
 489 if (*endptr == 0)
 490   {
 491   if (size < 0 || filecount < 0)
 492     {
 493     DEBUG(D_transport) debug_printf("negative value in maildirsize "
 494       "(size=" OFF_T_FMT " count=%d): recalculating\n", size, filecount);
 495     goto RECALCULATE;
 496     }
 497
 498   if (ob->quota_value > 0 &&
 499       (size + (ob->quota_is_inclusive? message_size : 0) > ob->quota_value ||
 500         (ob->quota_filecount_value > 0 &&
 501           filecount + (ob->quota_is_inclusive ? 1:0) >
 502             ob->quota_filecount_value)
 503       ))
 504     {
 505     struct stat statbuf;
 506     if (linecount > 1)
 507       {
 508       DEBUG(D_transport) debug_printf("over quota and maildirsize has "
 509         "more than 1 entry: recalculating\n");
 510       goto RECALCULATE;
 511       }
 512
 513     if (fstat(fd, &statbuf) < 0) goto RECALCULATE;  /* Should never occur */
 514
 515     if (time(NULL) - statbuf.st_mtime > 15*60)
 516       {
 517       DEBUG(D_transport) debug_printf("over quota and maildirsize is older "
 518         "than 15 minutes: recalculating\n");
 519       goto RECALCULATE;
 520       }
 521     }
 522   }
 523
 524
 525 /* If *endptr is not zero, there was a syntax error in the file. */
 526
 527 else
 528   {
 529   int len;
 530   time_t old_latest, new_latest;
 531   uschar *tempname;
 532   struct timeval tv;
 533
 534   DEBUG(D_transport)
 535     {
 536     uschar *p = endptr;
 537     while (p > buffer && p[-1] != '\n') p--;
 538     endptr[1] = 0;
 539
 540     debug_printf("error in maildirsizefile: unexpected character %d in "
 541       "line %d (starting '%s'): recalculating\n",
 542       *endptr, linecount + 1, string_printing(p));
 543     }
 544
 545   /* Either there is no file, or the quota value has changed, or the file has
 546   got too big, or there was some format error in the file. Recalculate the size
 547   and write new contents to a temporary file; then rename it. After any
 548   error, just return -1 as the file descriptor. */
 549
 550   RECALCULATE:
 551
 552   if (fd >= 0) (void)close(fd);
 553   old_latest = 0;
 554   filecount = 0;
 555   size = maildir_compute_size(path, &filecount, &old_latest, regex, dir_regex,
 556     FALSE);
 557
 558   (void)gettimeofday(&tv, NULL);
 559   tempname = string_sprintf("%s/tmp/" TIME_T_FMT ".H%luP%lu.%s",
 560     path, tv.tv_sec, tv.tv_usec, (long unsigned) getpid(), primary_hostname);
 561
 562   fd = Uopen(tempname, O_RDWR|O_CREAT|O_EXCL, ob->mode ? ob->mode : 0600);
 563   if (fd >= 0)
 564     {
 565     (void)sprintf(CS buffer, OFF_T_FMT "S,%dC\n" OFF_T_FMT " %d\n",
 566       ob->quota_value, ob->quota_filecount_value, size, filecount);
 567     len = Ustrlen(buffer);
 568     if (write(fd, buffer, len) != len || Urename(tempname, filename) < 0)
 569       {
 570       (void)close(fd);
 571       fd = -1;
 572       }
 573     }
 574
 575   /* If any of the directories have been modified since the last timestamp we
 576   saw, we have to junk this maildirsize file. */
 577
 578   DEBUG(D_transport) debug_printf("checking subdirectory timestamps\n");
 579   new_latest = 0;
 580   (void)maildir_compute_size(path, NULL, &new_latest , NULL, dir_regex, TRUE);
 581   if (new_latest > old_latest)
 582     {
 583     DEBUG(D_transport) debug_printf("abandoning maildirsize because of "
 584       "a later subdirectory modification\n");
 585     (void)Uunlink(filename);
 586     (void)close(fd);
 587     fd = -2;
 588     }
 589   }
 590
 591 /* Return the sizes and the file descriptor, if any */
 592
 593 DEBUG(D_transport) debug_printf("returning maildir size=" OFF_T_FMT
 594   " filecount=%d\n", size, filecount);
 595 *returned_size = size;
 596 *returned_filecount = filecount;
 597 return fd;
 598 }
 599
 600 /* End of tf_maildir.c */