src/src/transports/tf_maildir.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2018 */
   6 /* Copyright (c) The Exim Maintainers 2020 */
   7 /* See the file NOTICE for conditions of use and distribution. */
   8
   9 /* Functions in support of the use of maildirsize files for handling quotas in
  10 maildir directories. Some of the rules are a bit baroque:
  11
  12 http://www.inter7.com/courierimap/README.maildirquota.html
  13
  14 We try to follow most of that, except that the directories to skip for quota
  15 calculations are not hard wired in, but are supplied as a regex. */
  16
  17
  18 #include "../exim.h"
  19 #include "appendfile.h"
  20 #include "tf_maildir.h"
  21
  22 #define MAX_FILE_SIZE  5120
  23
  24
  25
  26 /*************************************************
  27 *      Ensure maildir directories exist          *
  28 *************************************************/
  29
  30 /* This function is called at the start of a maildir delivery, to ensure that
  31 all the relevant directories exist. It also creates a maildirfolder file if the
  32 base directory matches a given pattern.
  33
  34 Argument:
  35   path              the base directory name
  36   addr              the address item (for setting an error message)
  37   create_directory  true if we are allowed to create missing directories
  38   dirmode           the mode for created directories
  39   maildirfolder_create_regex
  40                     the pattern to match for maildirfolder creation
  41
  42 Returns:            TRUE on success; FALSE on failure
  43 */
  44
  45 BOOL maildir_ensure_directories(uschar *path, address_item *addr,
  46   BOOL create_directory, int dirmode, uschar *maildirfolder_create_regex)
  47 {
  48 int i;
  49 struct stat statbuf;
  50 const char *subdirs[] = { "/tmp", "/new", "/cur" };
  51
  52 DEBUG(D_transport)
  53   debug_printf("ensuring maildir directories exist in %s\n", path);
  54
  55 /* First ensure that the path we have is a directory; if it does not exist,
  56 create it. Then make sure the tmp, new & cur subdirs of the maildir are
  57 there. If not, fail. This aborts the delivery (even though the cur subdir is
  58 not actually needed for delivery). Handle all 4 directory tests/creates in a
  59 loop so that code can be shared. */
  60
  61 for (i = 0; i < 4; i++)
  62   {
  63   int j;
  64   const uschar *dir, *mdir;
  65
  66   if (i == 0)
  67     {
  68     mdir = CUS"";
  69     dir = path;
  70     }
  71   else
  72     {
  73     mdir = CUS subdirs[i-1];
  74     dir = mdir + 1;
  75     }
  76
  77   /* Check an existing path is a directory. This is inside a loop because
  78   there is a potential race condition when creating the directory - some
  79   other process may get there first. Give up after trying several times,
  80   though. */
  81
  82   for (j = 0; j < 10; j++)
  83     {
  84     if (Ustat(dir, &statbuf) == 0)
  85       {
  86       if (S_ISDIR(statbuf.st_mode)) break;   /* out of the race loop */
  87       addr->message = string_sprintf("%s%s is not a directory", path,
  88         mdir);
  89       addr->basic_errno = ERRNO_NOTDIRECTORY;
  90       return FALSE;
  91       }
  92
  93     /* Try to make if non-existent and configured to do so */
  94
  95     if (errno == ENOENT && create_directory)
  96       {
  97       if (!directory_make(NULL, dir, dirmode, FALSE))
  98         {
  99         if (errno == EEXIST) continue;     /* repeat the race loop */
 100         addr->message = string_sprintf("cannot create %s%s", path, mdir);
 101         addr->basic_errno = errno;
 102         return FALSE;
 103         }
 104       DEBUG(D_transport)
 105         debug_printf("created directory %s%s\n", path, mdir);
 106       break;   /* out of the race loop */
 107       }
 108
 109     /* stat() error other than ENOENT, or ENOENT and not creatable */
 110
 111     addr->message = string_sprintf("stat() error for %s%s: %s", path, mdir,
 112       strerror(errno));
 113     addr->basic_errno = errno;
 114     return FALSE;
 115     }
 116
 117   /* If we went round the loop 10 times, the directory was flickering in
 118   and out of existence like someone in a malfunctioning Star Trek
 119   transporter. */
 120
 121   if (j >= 10)
 122     {
 123     addr->message = string_sprintf("existence of %s%s unclear\n", path,
 124       mdir);
 125     addr->basic_errno = errno;
 126     addr->special_action = SPECIAL_FREEZE;
 127     return FALSE;
 128     }
 129
 130   /* First time through the directories loop, cd to the main directory */
 131
 132   if (i == 0 && Uchdir(path) != 0)
 133     {
 134     addr->message = string_sprintf ("cannot chdir to %s", path);
 135     addr->basic_errno = errno;
 136     return FALSE;
 137     }
 138   }
 139
 140 /* If the basic path matches maildirfolder_create_regex, we are dealing with
 141 a subfolder, and should ensure that a maildirfolder file exists. */
 142
 143 if (maildirfolder_create_regex != NULL)
 144   {
 145   const uschar *error;
 146   int offset;
 147   const pcre *regex;
 148
 149   DEBUG(D_transport) debug_printf("checking for maildirfolder requirement\n");
 150
 151   if (!(regex = pcre_compile(CS maildirfolder_create_regex, PCRE_COPT,
 152     CCSS &error, &offset, NULL)))
 153     {
 154     addr->message = string_sprintf("appendfile: regular expression "
 155       "error: %s at offset %d while compiling %s", error, offset,
 156       maildirfolder_create_regex);
 157     return FALSE;
 158     }
 159
 160   if (pcre_exec(regex, NULL, CS path, Ustrlen(path), 0, 0, NULL, 0) >= 0)
 161     {
 162     uschar *fname = string_sprintf("%s/maildirfolder", path);
 163     if (Ustat(fname, &statbuf) == 0)
 164       {
 165       DEBUG(D_transport) debug_printf("maildirfolder already exists\n");
 166       }
 167     else
 168       {
 169       int fd = Uopen(fname, O_WRONLY|O_APPEND|O_CREAT, 0600);
 170       if (fd < 0)
 171         {
 172         addr->message = string_sprintf("appendfile: failed to create "
 173           "maildirfolder file in %s directory: %s", path, strerror(errno));
 174         return FALSE;
 175         }
 176       (void)close(fd);
 177       DEBUG(D_transport) debug_printf("created maildirfolder file\n");
 178       }
 179     }
 180   else
 181     {
 182     DEBUG(D_transport) debug_printf("maildirfolder file not required\n");
 183     }
 184   }
 185
 186 return TRUE;   /* Everything exists that should exist */
 187 }
 188
 189
 190
 191
 192 /*************************************************
 193 *       Update maildirsizefile for new file      *
 194 *************************************************/
 195
 196 /* This function is called to add a new line to the file, recording the length
 197 of the newly added message. There isn't much we can do on failure...
 198
 199 Arguments:
 200   fd           the open file descriptor
 201   size         the size of the message
 202
 203 Returns:       nothing
 204 */
 205
 206 void
 207 maildir_record_length(int fd, int size)
 208 {
 209 int len;
 210 uschar buffer[256];
 211 sprintf(CS buffer, "%d 1\n", size);
 212 len = Ustrlen(buffer);
 213 if (lseek(fd, 0, SEEK_END) >= 0)
 214   {
 215   len = write(fd, buffer, len);
 216   DEBUG(D_transport)
 217     debug_printf("added '%.*s' to maildirsize file\n", len-1, buffer);
 218   }
 219 }
 220
 221
 222
 223 /*************************************************
 224 *          Find the size of a maildir            *
 225 *************************************************/
 226
 227 /* This function is called when we have to recalculate the size of a maildir by
 228 scanning all the files and directories therein. There are rules and conventions
 229 about which files or directories are included. We support this by the use of a
 230 regex to match directories that are to be included.
 231
 232 Maildirs can only be one level deep. However, this function recurses, so it
 233 might cope with deeper nestings. We use the existing check_dir_size() function
 234 to add up the sizes of the files in a directory that contains messages.
 235
 236 The function returns the most recent timestamp encountered. It can also be run
 237 in a dummy mode in which it does not scan for sizes, but just returns the
 238 timestamp.
 239
 240 Arguments:
 241   path            the path to the maildir
 242   filecount       where to store the count of messages
 243   latest          where to store the latest timestamp encountered
 244   regex           a regex for getting files sizes from file names
 245   dir_regex       a regex for matching directories to be included
 246   timestamp_only  don't actually compute any sizes
 247
 248 Returns:      the sum of the sizes of the messages
 249 */
 250
 251 off_t
 252 maildir_compute_size(uschar *path, int *filecount, time_t *latest,
 253   const pcre *regex, const pcre *dir_regex, BOOL timestamp_only)
 254 {
 255 DIR *dir;
 256 off_t sum = 0;
 257
 258 if (!(dir = exim_opendir(path)))
 259   return 0;
 260
 261 for (struct dirent *ent; ent = readdir(dir); )
 262   {
 263   uschar * s, * name = US ent->d_name;
 264   struct stat statbuf;
 265
 266   if (Ustrcmp(name, ".") == 0 || Ustrcmp(name, "..") == 0) continue;
 267
 268   /* We are normally supplied with a regex for choosing which directories to
 269   scan. We do the regex match first, because that avoids a stat() for names
 270   we aren't interested in. */
 271
 272   if (dir_regex != NULL &&
 273       pcre_exec(dir_regex, NULL, CS name, Ustrlen(name), 0, 0, NULL, 0) < 0)
 274     {
 275     DEBUG(D_transport)
 276       debug_printf("skipping %s/%s: dir_regex does not match\n", path, name);
 277     continue;
 278     }
 279
 280   /* The name is OK; stat it. */
 281
 282   s = string_sprintf("%s/%s", path, name);
 283   if (Ustat(s, &statbuf) < 0)
 284     {
 285     DEBUG(D_transport)
 286       debug_printf("maildir_compute_size: stat error %d for %s: %s\n", errno,
 287         s, strerror(errno));
 288     continue;
 289     }
 290
 291   if ((statbuf.st_mode & S_IFMT) != S_IFDIR)
 292     {
 293     DEBUG(D_transport)
 294       debug_printf("skipping %s/%s: not a directory\n", s, name);
 295     continue;
 296     }
 297
 298   /* Keep the latest timestamp encountered */
 299
 300   if (statbuf.st_mtime > *latest) *latest = statbuf.st_mtime;
 301
 302   /* If this is a maildir folder, call this function recursively. */
 303
 304   if (name[0] == '.')
 305     sum += maildir_compute_size(s, filecount, latest, regex, dir_regex,
 306       timestamp_only);
 307
 308   /* Otherwise it must be a folder that contains messages (e.g. new or cur), so
 309   we need to get its size, unless all we are interested in is the timestamp. */
 310
 311   else if (!timestamp_only)
 312     sum += check_dir_size(s, filecount, regex);
 313   }
 314
 315 closedir(dir);
 316 DEBUG(D_transport)
 317   {
 318   if (timestamp_only)
 319     debug_printf("maildir_compute_size (timestamp_only): %ld\n",
 320     (long int) *latest);
 321   else
 322     debug_printf("maildir_compute_size: path=%s\n  sum=" OFF_T_FMT
 323       " filecount=%d timestamp=%ld\n",
 324       path, sum, *filecount, (long int) *latest);
 325   }
 326 return sum;
 327 }
 328
 329
 330
 331 /*************************************************
 332 *        Create or update maildirsizefile        *
 333 *************************************************/
 334
 335 /* This function is called before a delivery if the option to use
 336 maildirsizefile is enabled. Its function is to create the file if it does not
 337 exist, or to update it if that is necessary.
 338
 339 The logic in this function follows the rules that are described in
 340
 341   http://www.inter7.com/courierimap/README.maildirquota.html
 342
 343 Or, at least, it is supposed to!
 344
 345 Arguments:
 346   path             the path to the maildir directory; this is already backed-up
 347                      to the parent if the delivery directory is a maildirfolder
 348   ob               the appendfile options block
 349   regex            a compiled regex for getting a file's size from its name
 350   dir_regex        a compiled regex for selecting maildir directories
 351   returned_size    where to return the current size of the maildir, even if
 352                      the maildirsizefile is removed because of a race
 353
 354 Returns:           >=0  a file descriptor for an open maildirsize file
 355                    -1   there was an error opening or accessing the file
 356                    -2   the file was removed because of a race
 357 */
 358
 359 int
 360 maildir_ensure_sizefile(uschar *path, appendfile_transport_options_block *ob,
 361   const pcre *regex, const pcre *dir_regex, off_t *returned_size,
 362   int *returned_filecount)
 363 {
 364 int count, fd;
 365 off_t cached_quota = 0;
 366 int cached_quota_filecount = 0;
 367 int filecount = 0;
 368 int linecount = 0;
 369 off_t size = 0;
 370 uschar *filename;
 371 uschar buffer[MAX_FILE_SIZE];
 372 uschar *ptr = buffer;
 373 uschar *endptr;
 374
 375 /* Try a few times to open or create the file, in case another process is doing
 376 the same thing. */
 377
 378 filename = string_sprintf("%s/maildirsize", path);
 379
 380 DEBUG(D_transport) debug_printf("looking for maildirsize in %s\n", path);
 381 if ((fd = Uopen(filename, O_RDWR|O_APPEND, ob->mode ? ob->mode : 0600)) < 0)
 382   {
 383   if (errno != ENOENT) return -1;
 384   DEBUG(D_transport)
 385     debug_printf("%s does not exist: recalculating\n", filename);
 386   goto RECALCULATE;
 387   }
 388
 389 /* The file has been successfully opened. Check that the cached quota value is
 390 still correct, and that the size of the file is still small enough. If so,
 391 compute the maildir size from the file. */
 392
 393 if ((count = read(fd, buffer, sizeof(buffer))) >= sizeof(buffer))
 394   {
 395   DEBUG(D_transport)
 396     debug_printf("maildirsize file too big (%d): recalculating\n", count);
 397   goto RECALCULATE;
 398   }
 399 buffer[count] = 0;   /* Ensure string terminated */
 400
 401 /* Read the quota parameters from the first line of the data. */
 402
 403 DEBUG(D_transport)
 404   debug_printf("reading quota parameters from maildirsize data\n");
 405
 406 for (;;)
 407   {
 408   off_t n = (off_t)Ustrtod(ptr, &endptr);
 409
 410   /* Only two data items are currently defined; ignore any others that
 411   may be present. The spec is for a number followed by a letter. Anything
 412   else we reject and recalculate. */
 413
 414   if (*endptr == 'S') cached_quota = n;
 415     else if (*endptr == 'C') cached_quota_filecount = (int)n;
 416   if (!isalpha(*endptr++))
 417     {
 418     DEBUG(D_transport)
 419       debug_printf("quota parameter number not followed by letter in "
 420         "\"%.*s\": recalculating maildirsize\n", (int)(endptr - buffer),
 421         buffer);
 422     goto RECALCULATE;
 423     }
 424   if (*endptr == '\n' || *endptr == 0) break;
 425   if (*endptr++ != ',')
 426     {
 427     DEBUG(D_transport)
 428       debug_printf("quota parameter not followed by comma in "
 429         "\"%.*s\": recalculating maildirsize\n", (int)(endptr - buffer),
 430         buffer);
 431     goto RECALCULATE;
 432     }
 433   ptr = endptr;
 434   }
 435
 436 /* Check the cached values against the current settings */
 437
 438 if (cached_quota != ob->quota_value ||
 439     cached_quota_filecount != ob->quota_filecount_value)
 440   {
 441   DEBUG(D_transport)
 442     debug_printf("cached quota is out of date: recalculating\n"
 443       "  quota=" OFF_T_FMT " cached_quota=" OFF_T_FMT " filecount_quota=%d "
 444       "cached_quota_filecount=%d\n", ob->quota_value,
 445       cached_quota, ob->quota_filecount_value, cached_quota_filecount);
 446   goto RECALCULATE;
 447   }
 448
 449 /* Quota values agree; parse the rest of the data to get the sizes. At this
 450 stage, *endptr points either to 0 or to '\n'.  */
 451
 452 DEBUG(D_transport)
 453   debug_printf("computing maildir size from maildirsize data\n");
 454
 455 while (*endptr++ == '\n')
 456   {
 457   if (*endptr == 0) break;
 458   linecount++;
 459   ptr = endptr;
 460   size += (off_t)Ustrtod(ptr, &endptr);
 461   if (*endptr != ' ') break;
 462   ptr = endptr + 1;
 463   filecount += Ustrtol(ptr, &endptr, 10);
 464   }
 465
 466 /* If *endptr is zero, we have successfully parsed the file, and we now have
 467 the size of the mailbox as cached in the file. The "rules" say that if this
 468 value indicates that the mailbox is over quota, we must recalculate if there is
 469 more than one entry in the file, or if the file is older than 15 minutes. Also,
 470 just in case there are weird values in the file, recalculate if either of the
 471 values is negative. */
 472
 473 if (*endptr == 0)
 474   {
 475   if (size < 0 || filecount < 0)
 476     {
 477     DEBUG(D_transport) debug_printf("negative value in maildirsize "
 478       "(size=" OFF_T_FMT " count=%d): recalculating\n", size, filecount);
 479     goto RECALCULATE;
 480     }
 481
 482   if (ob->quota_value > 0 &&
 483       (size + (ob->quota_is_inclusive? message_size : 0) > ob->quota_value ||
 484         (ob->quota_filecount_value > 0 &&
 485           filecount + (ob->quota_is_inclusive ? 1:0) >
 486             ob->quota_filecount_value)
 487       ))
 488     {
 489     struct stat statbuf;
 490     if (linecount > 1)
 491       {
 492       DEBUG(D_transport) debug_printf("over quota and maildirsize has "
 493         "more than 1 entry: recalculating\n");
 494       goto RECALCULATE;
 495       }
 496
 497     if (fstat(fd, &statbuf) < 0) goto RECALCULATE;  /* Should never occur */
 498
 499     if (time(NULL) - statbuf.st_mtime > 15*60)
 500       {
 501       DEBUG(D_transport) debug_printf("over quota and maildirsize is older "
 502         "than 15 minutes: recalculating\n");
 503       goto RECALCULATE;
 504       }
 505     }
 506   }
 507
 508
 509 /* If *endptr is not zero, there was a syntax error in the file. */
 510
 511 else
 512   {
 513   int len;
 514   time_t old_latest, new_latest;
 515   uschar *tempname;
 516   struct timeval tv;
 517
 518   DEBUG(D_transport)
 519     {
 520     uschar *p = endptr;
 521     while (p > buffer && p[-1] != '\n') p--;
 522     endptr[1] = 0;
 523
 524     debug_printf("error in maildirsizefile: unexpected character %d in "
 525       "line %d (starting '%s'): recalculating\n",
 526       *endptr, linecount + 1, string_printing(p));
 527     }
 528
 529   /* Either there is no file, or the quota value has changed, or the file has
 530   got too big, or there was some format error in the file. Recalculate the size
 531   and write new contents to a temporary file; then rename it. After any
 532   error, just return -1 as the file descriptor. */
 533
 534   RECALCULATE:
 535
 536   if (fd >= 0) (void)close(fd);
 537   old_latest = 0;
 538   filecount = 0;
 539   size = maildir_compute_size(path, &filecount, &old_latest, regex, dir_regex,
 540     FALSE);
 541
 542   (void)gettimeofday(&tv, NULL);
 543   tempname = string_sprintf("%s/tmp/" TIME_T_FMT ".H%luP%lu.%s",
 544     path, tv.tv_sec, tv.tv_usec, (long unsigned) getpid(), primary_hostname);
 545
 546   fd = Uopen(tempname, O_RDWR|O_CREAT|O_EXCL, ob->mode ? ob->mode : 0600);
 547   if (fd >= 0)
 548     {
 549     (void)sprintf(CS buffer, OFF_T_FMT "S,%dC\n" OFF_T_FMT " %d\n",
 550       ob->quota_value, ob->quota_filecount_value, size, filecount);
 551     len = Ustrlen(buffer);
 552     if (write(fd, buffer, len) != len || Urename(tempname, filename) < 0)
 553       {
 554       (void)close(fd);
 555       fd = -1;
 556       }
 557     }
 558
 559   /* If any of the directories have been modified since the last timestamp we
 560   saw, we have to junk this maildirsize file. */
 561
 562   DEBUG(D_transport) debug_printf("checking subdirectory timestamps\n");
 563   new_latest = 0;
 564   (void)maildir_compute_size(path, NULL, &new_latest , NULL, dir_regex, TRUE);
 565   if (new_latest > old_latest)
 566     {
 567     DEBUG(D_transport) debug_printf("abandoning maildirsize because of "
 568       "a later subdirectory modification\n");
 569     (void)Uunlink(filename);
 570     (void)close(fd);
 571     fd = -2;
 572     }
 573   }
 574
 575 /* Return the sizes and the file descriptor, if any */
 576
 577 DEBUG(D_transport) debug_printf("returning maildir size=" OFF_T_FMT
 578   " filecount=%d\n", size, filecount);
 579 *returned_size = size;
 580 *returned_filecount = filecount;
 581 return fd;
 582 }
 583
 584 /* End of tf_maildir.c */