Taint: invert the taint-check implementation control #define
[exim.git] / src / src / store.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
f3ebb786 6/* Copyright (c) The Exim maintainers 2019 */
059ec3d9
PH
7/* See the file NOTICE for conditions of use and distribution. */
8
9/* Exim gets and frees all its store through these functions. In the original
10implementation there was a lot of mallocing and freeing of small bits of store.
11The philosophy has now changed to a scheme which includes the concept of
12"stacking pools" of store. For the short-lived processes, there isn't any real
13need to do any garbage collection, but the stack concept allows quick resetting
14in places where this seems sensible.
15
16Obviously the long-running processes (the daemon, the queue runner, and eximon)
17must take care not to eat store.
18
19The following different types of store are recognized:
20
21. Long-lived, large blocks: This is implemented by retaining the original
22 malloc/free functions, and it used for permanent working buffers and for
23 getting blocks to cut up for the other types.
24
25. Long-lived, small blocks: This is used for blocks that have to survive until
26 the process exits. It is implemented as a stacking pool (POOL_PERM). This is
27 functionally the same as store_malloc(), except that the store can't be
28 freed, but I expect it to be more efficient for handling small blocks.
29
30. Short-lived, short blocks: Most of the dynamic store falls into this
31 category. It is implemented as a stacking pool (POOL_MAIN) which is reset
32 after accepting a message when multiple messages are received by a single
33 process. Resetting happens at some other times as well, usually fairly
34 locally after some specific processing that needs working store.
35
36. There is a separate pool (POOL_SEARCH) that is used only for lookup storage.
37 This means it can be freed when search_tidyup() is called to close down all
38 the lookup caching.
f3ebb786
JH
39
40. Orthogonal to the three pool types, there are two classes of memory: untainted
41 and tainted. The latter is used for values derived from untrusted input, and
42 the string-expansion mechanism refuses to operate on such values (obviously,
43 it can expand an untainted value to return a tainted result). The classes
adc4ecf9 44 are implemented by duplicating the three pool types. Pool resets are requested
f3ebb786 45 against the nontainted sibling and apply to both siblings.
adc4ecf9
JH
46
47 Only memory blocks requested for tainted use are regarded as tainted; anything
48 else (including stack auto variables) is untainted. Care is needed when coding
49 to not copy untrusted data into untainted memory, as downstream taint-checks
50 would be avoided.
51
52 Intermediate layers (eg. the string functions) can test for taint, and use this
53 for ensuringn that results have proper state. For example the
54 string_vformat_trc() routing supporting the string_sprintf() interface will
55 recopy a string being built into a tainted allocation if it meets a %s for a
56 tainted argument.
57
58 Internally we currently use malloc for nontainted pools, and mmap for tainted
59 pools. The disparity is for speed of testing the taintedness of pointers;
60 because Linux appears to use distinct non-overlapping address allocations for
61 mmap vs. everything else, which means only two pointer-compares suffice for the
62 test. Other OS' cannot use that optimisation, and a more lengthy test against
63 the limits of tainted-pool allcations has to be done.
059ec3d9
PH
64*/
65
66
67#include "exim.h"
438257ba
PP
68/* keep config.h before memcheck.h, for NVALGRIND */
69#include "config.h"
70
f3ebb786 71#include <sys/mman.h>
7f36d675 72#include "memcheck.h"
059ec3d9
PH
73
74
75/* We need to know how to align blocks of data for general use. I'm not sure
76how to get an alignment factor in general. In the current world, a value of 8
77is probably right, and this is sizeof(double) on some systems and sizeof(void
78*) on others, so take the larger of those. Since everything in this expression
79is a constant, the compiler should optimize it to a simple constant wherever it
80appears (I checked that gcc does do this). */
81
82#define alignment \
f3ebb786 83 (sizeof(void *) > sizeof(double) ? sizeof(void *) : sizeof(double))
059ec3d9
PH
84
85/* store_reset() will not free the following block if the last used block has
86less than this much left in it. */
87
88#define STOREPOOL_MIN_SIZE 256
89
90/* Structure describing the beginning of each big block. */
91
92typedef struct storeblock {
93 struct storeblock *next;
94 size_t length;
95} storeblock;
96
97/* Just in case we find ourselves on a system where the structure above has a
98length that is not a multiple of the alignment, set up a macro for the padded
99length. */
100
101#define ALIGNED_SIZEOF_STOREBLOCK \
102 (((sizeof(storeblock) + alignment - 1) / alignment) * alignment)
103
f3ebb786
JH
104/* Size of block to get from malloc to carve up into smaller ones. This
105must be a multiple of the alignment. We assume that 8192 is going to be
106suitably aligned. */
107
108#define STORE_BLOCK_SIZE (8192 - ALIGNED_SIZEOF_STOREBLOCK)
109
059ec3d9
PH
110/* Variables holding data for the local pools of store. The current pool number
111is held in store_pool, which is global so that it can be changed from outside.
112Setting the initial length values to -1 forces a malloc for the first call,
113even if the length is zero (which is used for getting a point to reset to). */
114
f3ebb786 115int store_pool = POOL_MAIN;
059ec3d9 116
f3ebb786
JH
117#define NPOOLS 6
118static storeblock *chainbase[NPOOLS];
119static storeblock *current_block[NPOOLS];
120static void *next_yield[NPOOLS];
121static int yield_length[NPOOLS] = { -1, -1, -1, -1, -1, -1 };
122
123/* The limits of the tainted pools. Tracking these on new allocations enables
124a fast is_tainted implementation. We assume the kernel only allocates mmaps using
125one side or the other of data+heap, not both. */
126
6d5f5caf
JH
127void * tainted_base = (void *)-1;
128void * tainted_top = (void *)0;
059ec3d9
PH
129
130/* pool_malloc holds the amount of memory used by the store pools; this goes up
131and down as store is reset or released. nonpool_malloc is the total got by
132malloc from other calls; this doesn't go down because it is just freed by
133pointer. */
134
f3ebb786
JH
135static int pool_malloc;
136static int nonpool_malloc;
059ec3d9
PH
137
138/* This variable is set by store_get() to its yield, and by store_reset() to
139NULL. This enables string_cat() to optimize its store handling for very long
140strings. That's why the variable is global. */
141
f3ebb786
JH
142void *store_last_get[NPOOLS];
143
144/* These are purely for stats-gathering */
145
146static int nbytes[NPOOLS]; /* current bytes allocated */
147static int maxbytes[NPOOLS]; /* max number reached */
148static int nblocks[NPOOLS]; /* current number of blocks allocated */
149static int maxblocks[NPOOLS];
150static int n_nonpool_blocks; /* current number of direct store_malloc() blocks */
151static int max_nonpool_blocks;
152static int max_pool_malloc; /* max value for pool_malloc */
153static int max_nonpool_malloc; /* max value for nonpool_malloc */
154
155
81a559c8 156#ifndef COMPILE_UTILITY
f3ebb786
JH
157static const uschar * pooluse[NPOOLS] = {
158[POOL_MAIN] = US"main",
159[POOL_PERM] = US"perm",
160[POOL_SEARCH] = US"search",
161[POOL_TAINT_MAIN] = US"main",
162[POOL_TAINT_PERM] = US"perm",
163[POOL_TAINT_SEARCH] = US"search",
164};
165static const uschar * poolclass[NPOOLS] = {
166[POOL_MAIN] = US"untainted",
167[POOL_PERM] = US"untainted",
168[POOL_SEARCH] = US"untainted",
169[POOL_TAINT_MAIN] = US"tainted",
170[POOL_TAINT_PERM] = US"tainted",
171[POOL_TAINT_SEARCH] = US"tainted",
172};
81a559c8 173#endif
f3ebb786
JH
174
175
176static void * store_mmap(int, const char *, int);
177static void * internal_store_malloc(int, const char *, int);
65766f1b
JH
178static void internal_untainted_free(void *, const char *, int linenumber);
179static void internal_tainted_free(storeblock *, const char *, int linenumber);
f3ebb786
JH
180
181/******************************************************************************/
182
f096bccc 183#ifndef TAINT_CHECK_FAST
14ca5d2a
JH
184/* Slower version check, for use when platform intermixes malloc and mmap area
185addresses. */
186
187BOOL
188is_tainted_fn(const void * p)
189{
190storeblock * b;
191int pool;
192
193for (pool = 0; pool < nelem(chainbase); pool++)
194 if ((b = current_block[pool]))
195 {
196 char * bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
197 if (CS p >= bc && CS p <= bc + b->length) goto hit;
198 }
199
200for (pool = 0; pool < nelem(chainbase); pool++)
201 for (b = chainbase[pool]; b; b = b->next)
202 {
203 char * bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
204 if (CS p >= bc && CS p <= bc + b->length) goto hit;
205 }
206return FALSE;
207
208hit:
209return pool >= POOL_TAINT_BASE;
210}
f096bccc 211#endif
14ca5d2a
JH
212
213
f3ebb786
JH
214void
215die_tainted(const uschar * msg, const uschar * func, int line)
216{
217log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Taint mismatch, %s: %s %d\n",
218 msg, func, line);
219}
059ec3d9
PH
220
221
222/*************************************************
223* Get a block from the current pool *
224*************************************************/
225
226/* Running out of store is a total disaster. This function is called via the
227macro store_get(). It passes back a block of store within the current big
228block, getting a new one if necessary. The address is saved in
229store_last_was_get.
230
231Arguments:
adc4ecf9
JH
232 size amount wanted, bytes
233 tainted class: set to true for untrusted data (eg. from smtp input)
f3ebb786
JH
234 func function from which called
235 linenumber line number in source file
059ec3d9
PH
236
237Returns: pointer to store (panic on malloc failure)
238*/
239
240void *
f3ebb786 241store_get_3(int size, BOOL tainted, const char *func, int linenumber)
059ec3d9 242{
f3ebb786
JH
243int pool = tainted ? store_pool + POOL_TAINT_BASE : store_pool;
244
059ec3d9
PH
245/* Round up the size to a multiple of the alignment. Although this looks a
246messy statement, because "alignment" is a constant expression, the compiler can
247do a reasonable job of optimizing, especially if the value of "alignment" is a
248power of two. I checked this with -O2, and gcc did very well, compiling it to 4
249instructions on a Sparc (alignment = 8). */
250
251if (size % alignment != 0) size += alignment - (size % alignment);
252
253/* If there isn't room in the current block, get a new one. The minimum
254size is STORE_BLOCK_SIZE, and we would expect this to be the norm, since
255these functions are mostly called for small amounts of store. */
256
f3ebb786 257if (size > yield_length[pool])
059ec3d9 258 {
f3ebb786 259 int length = size <= STORE_BLOCK_SIZE ? STORE_BLOCK_SIZE : size;
059ec3d9 260 int mlength = length + ALIGNED_SIZEOF_STOREBLOCK;
f3ebb786 261 storeblock * newblock;
059ec3d9
PH
262
263 /* Sometimes store_reset() may leave a block for us; check if we can use it */
264
f3ebb786 265 if ( (newblock = current_block[pool])
64073d9c
JH
266 && (newblock = newblock->next)
267 && newblock->length < length
268 )
059ec3d9 269 {
64073d9c 270 /* Give up on this block, because it's too small */
f3ebb786
JH
271 nblocks[pool]--;
272 if (pool < POOL_TAINT_BASE)
65766f1b 273 internal_untainted_free(newblock, func, linenumber);
f3ebb786 274 else
65766f1b 275 internal_tainted_free(newblock, func, linenumber);
64073d9c 276 newblock = NULL;
059ec3d9
PH
277 }
278
279 /* If there was no free block, get a new one */
280
64073d9c 281 if (!newblock)
059ec3d9 282 {
f3ebb786
JH
283 if ((nbytes[pool] += mlength) > maxbytes[pool])
284 maxbytes[pool] = nbytes[pool];
285 if ((pool_malloc += mlength) > max_pool_malloc) /* Used in pools */
286 max_pool_malloc = pool_malloc;
287 nonpool_malloc -= mlength; /* Exclude from overall total */
288 if (++nblocks[pool] > maxblocks[pool])
289 maxblocks[pool] = nblocks[pool];
290
291 newblock = tainted
292 ? store_mmap(mlength, func, linenumber)
293 : internal_store_malloc(mlength, func, linenumber);
059ec3d9
PH
294 newblock->next = NULL;
295 newblock->length = length;
f3ebb786
JH
296
297 if (!chainbase[pool])
298 chainbase[pool] = newblock;
64073d9c 299 else
f3ebb786 300 current_block[pool]->next = newblock;
059ec3d9
PH
301 }
302
f3ebb786
JH
303 current_block[pool] = newblock;
304 yield_length[pool] = newblock->length;
305 next_yield[pool] =
306 (void *)(CS current_block[pool] + ALIGNED_SIZEOF_STOREBLOCK);
307 (void) VALGRIND_MAKE_MEM_NOACCESS(next_yield[pool], yield_length[pool]);
059ec3d9
PH
308 }
309
310/* There's (now) enough room in the current block; the yield is the next
311pointer. */
312
f3ebb786 313store_last_get[pool] = next_yield[pool];
059ec3d9
PH
314
315/* Cut out the debugging stuff for utilities, but stop picky compilers from
316giving warnings. */
317
318#ifdef COMPILE_UTILITY
f3ebb786 319func = func;
059ec3d9
PH
320linenumber = linenumber;
321#else
322DEBUG(D_memory)
f3ebb786
JH
323 debug_printf("---%d Get %6p %5d %-14s %4d\n", pool,
324 store_last_get[pool], size, func, linenumber);
059ec3d9
PH
325#endif /* COMPILE_UTILITY */
326
f3ebb786 327(void) VALGRIND_MAKE_MEM_UNDEFINED(store_last_get[pool], size);
059ec3d9
PH
328/* Update next pointer and number of bytes left in the current block. */
329
f3ebb786
JH
330next_yield[pool] = (void *)(CS next_yield[pool] + size);
331yield_length[pool] -= size;
332return store_last_get[pool];
059ec3d9
PH
333}
334
335
336
337/*************************************************
338* Get a block from the PERM pool *
339*************************************************/
340
341/* This is just a convenience function, useful when just a single block is to
342be obtained.
343
344Arguments:
345 size amount wanted
f3ebb786
JH
346 func function from which called
347 linenumber line number in source file
059ec3d9
PH
348
349Returns: pointer to store (panic on malloc failure)
350*/
351
352void *
f3ebb786 353store_get_perm_3(int size, BOOL tainted, const char *func, int linenumber)
059ec3d9
PH
354{
355void *yield;
356int old_pool = store_pool;
357store_pool = POOL_PERM;
f3ebb786 358yield = store_get_3(size, tainted, func, linenumber);
059ec3d9
PH
359store_pool = old_pool;
360return yield;
361}
362
363
364
365/*************************************************
366* Extend a block if it is at the top *
367*************************************************/
368
369/* While reading strings of unknown length, it is often the case that the
370string is being read into the block at the top of the stack. If it needs to be
f3ebb786 371extended, it is more efficient just to extend within the top block rather than
059ec3d9
PH
372allocate a new block and then have to copy the data. This function is provided
373for the use of string_cat(), but of course can be used elsewhere too.
f3ebb786 374The block itself is not expanded; only the top allocation from it.
059ec3d9
PH
375
376Arguments:
377 ptr pointer to store block
378 oldsize current size of the block, as requested by user
379 newsize new size required
f3ebb786 380 func function from which called
059ec3d9
PH
381 linenumber line number in source file
382
383Returns: TRUE if the block is at the top of the stack and has been
384 extended; FALSE if it isn't at the top of the stack, or cannot
385 be extended
386*/
387
388BOOL
f3ebb786
JH
389store_extend_3(void *ptr, BOOL tainted, int oldsize, int newsize,
390 const char *func, int linenumber)
059ec3d9 391{
f3ebb786 392int pool = tainted ? store_pool + POOL_TAINT_BASE : store_pool;
059ec3d9
PH
393int inc = newsize - oldsize;
394int rounded_oldsize = oldsize;
395
f3ebb786
JH
396/* Check that the block being extended was already of the required taint status;
397refuse to extend if not. */
398
399if (is_tainted(ptr) != tainted)
400 return FALSE;
401
059ec3d9
PH
402if (rounded_oldsize % alignment != 0)
403 rounded_oldsize += alignment - (rounded_oldsize % alignment);
404
f3ebb786
JH
405if (CS ptr + rounded_oldsize != CS (next_yield[pool]) ||
406 inc > yield_length[pool] + rounded_oldsize - oldsize)
059ec3d9
PH
407 return FALSE;
408
409/* Cut out the debugging stuff for utilities, but stop picky compilers from
410giving warnings. */
411
412#ifdef COMPILE_UTILITY
f3ebb786 413func = func;
059ec3d9
PH
414linenumber = linenumber;
415#else
416DEBUG(D_memory)
f3ebb786
JH
417 debug_printf("---%d Ext %6p %5d %-14s %4d\n", pool, ptr, newsize,
418 func, linenumber);
059ec3d9
PH
419#endif /* COMPILE_UTILITY */
420
421if (newsize % alignment != 0) newsize += alignment - (newsize % alignment);
f3ebb786
JH
422next_yield[pool] = CS ptr + newsize;
423yield_length[pool] -= newsize - rounded_oldsize;
4d8bb202 424(void) VALGRIND_MAKE_MEM_UNDEFINED(ptr + oldsize, inc);
059ec3d9
PH
425return TRUE;
426}
427
428
429
430
431/*************************************************
432* Back up to a previous point on the stack *
433*************************************************/
434
435/* This function resets the next pointer, freeing any subsequent whole blocks
f3ebb786
JH
436that are now unused. Call with a cookie obtained from store_mark() only; do
437not call with a pointer returned by store_get(). Both the untainted and tainted
438pools corresposding to store_pool are reset.
059ec3d9
PH
439
440Arguments:
f3ebb786
JH
441 r place to back up to
442 func function from which called
059ec3d9
PH
443 linenumber line number in source file
444
445Returns: nothing
446*/
447
f3ebb786
JH
448static void
449internal_store_reset(void * ptr, int pool, const char *func, int linenumber)
059ec3d9 450{
cf0812d5 451storeblock * bb;
f3ebb786 452storeblock * b = current_block[pool];
cf0812d5 453char * bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
f3ebb786
JH
454int newlength, count;
455#ifndef COMPILE_UTILITY
456int oldmalloc = pool_malloc;
457#endif
059ec3d9
PH
458
459/* Last store operation was not a get */
460
f3ebb786 461store_last_get[pool] = NULL;
059ec3d9
PH
462
463/* See if the place is in the current block - as it often will be. Otherwise,
464search for the block in which it lies. */
465
cf0812d5 466if (CS ptr < bc || CS ptr > bc + b->length)
059ec3d9 467 {
f3ebb786 468 for (b = chainbase[pool]; b; b = b->next)
059ec3d9 469 {
cf0812d5
JH
470 bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
471 if (CS ptr >= bc && CS ptr <= bc + b->length) break;
059ec3d9 472 }
cf0812d5 473 if (!b)
438257ba 474 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "internal error: store_reset(%p) "
f3ebb786 475 "failed: pool=%d %-14s %4d", ptr, pool, func, linenumber);
059ec3d9
PH
476 }
477
478/* Back up, rounding to the alignment if necessary. When testing, flatten
479the released memory. */
480
cf0812d5 481newlength = bc + b->length - CS ptr;
059ec3d9 482#ifndef COMPILE_UTILITY
65a32f85 483if (debug_store)
2c9f7ff8 484 {
f3ebb786 485 assert_no_variables(ptr, newlength, func, linenumber);
8768d548 486 if (f.running_in_test_harness)
64073d9c
JH
487 {
488 (void) VALGRIND_MAKE_MEM_DEFINED(ptr, newlength);
489 memset(ptr, 0xF0, newlength);
490 }
2c9f7ff8 491 }
059ec3d9 492#endif
4d8bb202 493(void) VALGRIND_MAKE_MEM_NOACCESS(ptr, newlength);
f3ebb786
JH
494next_yield[pool] = CS ptr + (newlength % alignment);
495count = yield_length[pool];
496count = (yield_length[pool] = newlength - (newlength % alignment)) - count;
497current_block[pool] = b;
498
499/* Free any subsequent block. Do NOT free the first
500successor, if our current block has less than 256 bytes left. This should
501prevent us from flapping memory. However, keep this block only when it has
502the default size. */
503
504if ( yield_length[pool] < STOREPOOL_MIN_SIZE
505 && b->next
506 && b->next->length == STORE_BLOCK_SIZE)
7f36d675 507 {
059ec3d9 508 b = b->next;
cf0812d5 509#ifndef COMPILE_UTILITY
65a32f85 510 if (debug_store)
cf0812d5 511 assert_no_variables(b, b->length + ALIGNED_SIZEOF_STOREBLOCK,
f3ebb786 512 func, linenumber);
cf0812d5
JH
513#endif
514 (void) VALGRIND_MAKE_MEM_NOACCESS(CS b + ALIGNED_SIZEOF_STOREBLOCK,
4d8bb202 515 b->length - ALIGNED_SIZEOF_STOREBLOCK);
7f36d675 516 }
059ec3d9
PH
517
518bb = b->next;
519b->next = NULL;
520
cf0812d5 521while ((b = bb))
059ec3d9 522 {
f3ebb786 523 int siz = b->length + ALIGNED_SIZEOF_STOREBLOCK;
cf0812d5 524#ifndef COMPILE_UTILITY
65a32f85 525 if (debug_store)
cf0812d5 526 assert_no_variables(b, b->length + ALIGNED_SIZEOF_STOREBLOCK,
f3ebb786 527 func, linenumber);
cf0812d5 528#endif
059ec3d9 529 bb = bb->next;
f3ebb786
JH
530 nbytes[pool] -= siz;
531 pool_malloc -= siz;
532 nblocks[pool]--;
533 if (pool < POOL_TAINT_BASE)
65766f1b 534 internal_untainted_free(b, func, linenumber);
f3ebb786 535 else
65766f1b 536 internal_tainted_free(b, func, linenumber);
059ec3d9
PH
537 }
538
539/* Cut out the debugging stuff for utilities, but stop picky compilers from
540giving warnings. */
541
542#ifdef COMPILE_UTILITY
f3ebb786 543func = func;
059ec3d9
PH
544linenumber = linenumber;
545#else
546DEBUG(D_memory)
f3ebb786
JH
547 debug_printf("---%d Rst %6p %5d %-14s %4d %d\n", pool, ptr,
548 count + oldmalloc - pool_malloc,
549 func, linenumber, pool_malloc);
550#endif /* COMPILE_UTILITY */
551}
552
553
554rmark
555store_reset_3(rmark r, int pool, const char *func, int linenumber)
556{
557void ** ptr = r;
558
559if (pool >= POOL_TAINT_BASE)
560 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
561 "store_reset called for pool %d: %s %d\n", pool, func, linenumber);
562if (!r)
563 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
564 "store_reset called with bad mark: %s %d\n", func, linenumber);
565
566internal_store_reset(*ptr, pool + POOL_TAINT_BASE, func, linenumber);
567internal_store_reset(ptr, pool, func, linenumber);
568return NULL;
569}
570
571
572
573/* Free tail-end unused allocation. This lets us allocate a big chunk
574early, for cases when we only discover later how much was really needed.
575
576Can be called with a value from store_get(), or an offset after such. Only
577the tainted or untainted pool that serviced the store_get() will be affected.
578
579This is mostly a cut-down version of internal_store_reset().
580XXX needs rationalising
581*/
582
583void
584store_release_above_3(void *ptr, const char *func, int linenumber)
585{
586/* Search all pools' "current" blocks. If it isn't one of those,
587ignore it (it usually will be). */
588
589for (int pool = 0; pool < nelem(current_block); pool++)
059ec3d9 590 {
f3ebb786
JH
591 storeblock * b = current_block[pool];
592 char * bc;
593 int count, newlength;
594
595 if (!b)
596 continue;
597
598 bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
599 if (CS ptr < bc || CS ptr > bc + b->length)
600 continue;
601
602 /* Last store operation was not a get */
603
604 store_last_get[pool] = NULL;
605
606 /* Back up, rounding to the alignment if necessary. When testing, flatten
607 the released memory. */
608
609 newlength = bc + b->length - CS ptr;
610#ifndef COMPILE_UTILITY
611 if (debug_store)
612 {
613 assert_no_variables(ptr, newlength, func, linenumber);
614 if (f.running_in_test_harness)
615 {
616 (void) VALGRIND_MAKE_MEM_DEFINED(ptr, newlength);
617 memset(ptr, 0xF0, newlength);
618 }
619 }
620#endif
621 (void) VALGRIND_MAKE_MEM_NOACCESS(ptr, newlength);
622 next_yield[pool] = CS ptr + (newlength % alignment);
623 count = yield_length[pool];
624 count = (yield_length[pool] = newlength - (newlength % alignment)) - count;
625
626 /* Cut out the debugging stuff for utilities, but stop picky compilers from
627 giving warnings. */
628
629#ifdef COMPILE_UTILITY
630 func = func;
631 linenumber = linenumber;
632#else
633 DEBUG(D_memory)
634 debug_printf("---%d Rel %6p %5d %-14s %4d %d\n", pool, ptr, count,
635 func, linenumber, pool_malloc);
636#endif
637 return;
059ec3d9 638 }
f3ebb786
JH
639#ifndef COMPILE_UTILITY
640DEBUG(D_memory)
641 debug_printf("non-last memory release try: %s %d\n", func, linenumber);
642#endif
059ec3d9
PH
643}
644
645
646
f3ebb786
JH
647rmark
648store_mark_3(const char *func, int linenumber)
649{
650void ** p;
651
652if (store_pool >= POOL_TAINT_BASE)
653 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
654 "store_mark called for pool %d: %s %d\n", store_pool, func, linenumber);
655
656/* Stash a mark for the tainted-twin release, in the untainted twin. Return
657a cookie (actually the address in the untainted pool) to the caller.
658Reset uses the cookie to recover the t-mark, winds back the tainted pool with it
659and winds back the untainted pool with the cookie. */
660
661p = store_get_3(sizeof(void *), FALSE, func, linenumber);
662*p = store_get_3(0, TRUE, func, linenumber);
663return p;
664}
665
666
059ec3d9
PH
667
668
669/************************************************
670* Release store *
671************************************************/
672
459fca58
JH
673/* This function checks that the pointer it is given is the first thing in a
674block, and if so, releases that block.
059ec3d9
PH
675
676Arguments:
677 block block of store to consider
f3ebb786 678 func function from which called
059ec3d9
PH
679 linenumber line number in source file
680
681Returns: nothing
682*/
683
459fca58 684static void
f3ebb786 685store_release_3(void * block, int pool, const char * func, int linenumber)
059ec3d9 686{
059ec3d9
PH
687/* It will never be the first block, so no need to check that. */
688
f3ebb786 689for (storeblock * b = chainbase[pool]; b; b = b->next)
059ec3d9 690 {
459fca58
JH
691 storeblock * bb = b->next;
692 if (bb && CS block == CS bb + ALIGNED_SIZEOF_STOREBLOCK)
059ec3d9 693 {
f3ebb786 694 int siz = bb->length + ALIGNED_SIZEOF_STOREBLOCK;
059ec3d9 695 b->next = bb->next;
f3ebb786
JH
696 nbytes[pool] -= siz;
697 pool_malloc -= siz;
698 nblocks[pool]--;
059ec3d9
PH
699
700 /* Cut out the debugging stuff for utilities, but stop picky compilers
701 from giving warnings. */
702
459fca58 703#ifdef COMPILE_UTILITY
f3ebb786 704 func = func;
059ec3d9 705 linenumber = linenumber;
459fca58 706#else
059ec3d9 707 DEBUG(D_memory)
f3ebb786
JH
708 debug_printf("-Release %6p %-20s %4d %d\n", (void *)bb, func,
709 linenumber, pool_malloc);
459fca58 710
8768d548 711 if (f.running_in_test_harness)
059ec3d9 712 memset(bb, 0xF0, bb->length+ALIGNED_SIZEOF_STOREBLOCK);
459fca58 713#endif /* COMPILE_UTILITY */
059ec3d9
PH
714
715 free(bb);
716 return;
717 }
718 }
719}
720
721
459fca58
JH
722/************************************************
723* Move store *
724************************************************/
725
726/* Allocate a new block big enough to expend to the given size and
727copy the current data into it. Free the old one if possible.
728
729This function is specifically provided for use when reading very
730long strings, e.g. header lines. When the string gets longer than a
731complete block, it gets copied to a new block. It is helpful to free
732the old block iff the previous copy of the string is at its start,
733and therefore the only thing in it. Otherwise, for very long strings,
734dead store can pile up somewhat disastrously. This function checks that
735the pointer it is given is the first thing in a block, and that nothing
736has been allocated since. If so, releases that block.
737
738Arguments:
739 block
740 newsize
741 len
742
743Returns: new location of data
744*/
745
746void *
f3ebb786
JH
747store_newblock_3(void * block, BOOL tainted, int newsize, int len,
748 const char * func, int linenumber)
459fca58 749{
f3ebb786
JH
750int pool = tainted ? store_pool + POOL_TAINT_BASE : store_pool;
751BOOL release_ok = !tainted && store_last_get[pool] == block;
752uschar * newtext;
753
aaabfafe 754#ifndef MACRO_PREDEF
f3ebb786
JH
755if (is_tainted(block) != tainted)
756 die_tainted(US"store_newblock", CUS func, linenumber);
aaabfafe 757#endif
459fca58 758
f3ebb786 759newtext = store_get(newsize, tainted);
459fca58 760memcpy(newtext, block, len);
f3ebb786 761if (release_ok) store_release_3(block, pool, func, linenumber);
459fca58
JH
762return (void *)newtext;
763}
764
765
059ec3d9
PH
766
767
f3ebb786
JH
768/******************************************************************************/
769static void *
770store_alloc_tail(void * yield, int size, const char * func, int line,
771 const uschar * type)
772{
773if ((nonpool_malloc += size) > max_nonpool_malloc)
774 max_nonpool_malloc = nonpool_malloc;
775
776/* Cut out the debugging stuff for utilities, but stop picky compilers from
777giving warnings. */
778
779#ifdef COMPILE_UTILITY
780func = func; line = line; type = type;
781#else
782
783/* If running in test harness, spend time making sure all the new store
784is not filled with zeros so as to catch problems. */
785
786if (f.running_in_test_harness)
787 memset(yield, 0xF0, (size_t)size);
788DEBUG(D_memory) debug_printf("--%6s %6p %5d bytes\t%-14s %4d\tpool %5d nonpool %5d\n",
789 type, yield, size, func, line, pool_malloc, nonpool_malloc);
790#endif /* COMPILE_UTILITY */
791
792return yield;
793}
794
795/*************************************************
796* Mmap store *
797*************************************************/
798
799static void *
800store_mmap(int size, const char * func, int line)
801{
802void * yield, * top;
803
804if (size < 16) size = 16;
805
806if (!(yield = mmap(NULL, (size_t)size,
807 PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)))
808 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to mmap %d bytes of memory: "
809 "called from line %d of %s", size, line, func);
810
811if (yield < tainted_base) tainted_base = yield;
6d95688d 812if ((top = US yield + size) > tainted_top) tainted_top = top;
f3ebb786
JH
813
814return store_alloc_tail(yield, size, func, line, US"Mmap");
815}
816
059ec3d9
PH
817/*************************************************
818* Malloc store *
819*************************************************/
820
821/* Running out of store is a total disaster for exim. Some malloc functions
822do not run happily on very small sizes, nor do they document this fact. This
823function is called via the macro store_malloc().
824
825Arguments:
826 size amount of store wanted
f3ebb786 827 func function from which called
059ec3d9
PH
828 linenumber line number in source file
829
830Returns: pointer to gotten store (panic on failure)
831*/
832
f3ebb786
JH
833static void *
834internal_store_malloc(int size, const char *func, int linenumber)
059ec3d9 835{
f3ebb786 836void * yield;
059ec3d9
PH
837
838if (size < 16) size = 16;
059ec3d9 839
40c90bca 840if (!(yield = malloc((size_t)size)))
059ec3d9 841 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to malloc %d bytes of memory: "
f3ebb786 842 "called from line %d in %s", size, linenumber, func);
059ec3d9 843
f3ebb786
JH
844return store_alloc_tail(yield, size, func, linenumber, US"Malloc");
845}
059ec3d9 846
f3ebb786
JH
847void *
848store_malloc_3(int size, const char *func, int linenumber)
849{
850if (n_nonpool_blocks++ > max_nonpool_blocks)
851 max_nonpool_blocks = n_nonpool_blocks;
852return internal_store_malloc(size, func, linenumber);
059ec3d9
PH
853}
854
855
856/************************************************
857* Free store *
858************************************************/
859
860/* This function is called by the macro store_free().
861
862Arguments:
863 block block of store to free
f3ebb786 864 func function from which called
059ec3d9
PH
865 linenumber line number in source file
866
867Returns: nothing
868*/
869
f3ebb786 870static void
65766f1b 871internal_untainted_free(void * block, const char * func, int linenumber)
059ec3d9
PH
872{
873#ifdef COMPILE_UTILITY
f3ebb786 874func = func;
059ec3d9
PH
875linenumber = linenumber;
876#else
877DEBUG(D_memory)
f3ebb786 878 debug_printf("----Free %6p %-20s %4d\n", block, func, linenumber);
059ec3d9
PH
879#endif /* COMPILE_UTILITY */
880free(block);
881}
882
f3ebb786 883void
65766f1b 884store_free_3(void * block, const char * func, int linenumber)
f3ebb786
JH
885{
886n_nonpool_blocks--;
65766f1b
JH
887internal_untainted_free(block, func, linenumber);
888}
889
890/******************************************************************************/
891static void
892internal_tainted_free(storeblock * block, const char * func, int linenumber)
893{
894#ifdef COMPILE_UTILITY
895func = func;
896linenumber = linenumber;
897#else
898DEBUG(D_memory)
899 debug_printf("---Unmap %6p %-20s %4d\n", block, func, linenumber);
900#endif
901munmap((void *)block, block->length + ALIGNED_SIZEOF_STOREBLOCK);
f3ebb786
JH
902}
903
904/******************************************************************************/
905/* Stats output on process exit */
906void
907store_exit(void)
908{
909#ifndef COMPILE_UTILITY
910DEBUG(D_memory)
911 {
912 debug_printf("----Exit nonpool max: %3d kB in %d blocks\n",
913 (max_nonpool_malloc+1023)/1024, max_nonpool_blocks);
914 debug_printf("----Exit npools max: %3d kB\n", max_pool_malloc/1024);
915 for (int i = 0; i < NPOOLS; i++)
916 debug_printf("----Exit pool %d max: %3d kB in %d blocks\t%s %s\n",
917 i, maxbytes[i]/1024, maxblocks[i], poolclass[i], pooluse[i]);
918 }
919#endif
920}
921
059ec3d9 922/* End of store.c */