Taint: speed up slow-mode is_tainted
[exim.git] / src / src / store.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
f3ebb786 6/* Copyright (c) The Exim maintainers 2019 */
059ec3d9
PH
7/* See the file NOTICE for conditions of use and distribution. */
8
9/* Exim gets and frees all its store through these functions. In the original
10implementation there was a lot of mallocing and freeing of small bits of store.
11The philosophy has now changed to a scheme which includes the concept of
12"stacking pools" of store. For the short-lived processes, there isn't any real
13need to do any garbage collection, but the stack concept allows quick resetting
14in places where this seems sensible.
15
16Obviously the long-running processes (the daemon, the queue runner, and eximon)
17must take care not to eat store.
18
19The following different types of store are recognized:
20
21. Long-lived, large blocks: This is implemented by retaining the original
22 malloc/free functions, and it used for permanent working buffers and for
23 getting blocks to cut up for the other types.
24
25. Long-lived, small blocks: This is used for blocks that have to survive until
26 the process exits. It is implemented as a stacking pool (POOL_PERM). This is
27 functionally the same as store_malloc(), except that the store can't be
28 freed, but I expect it to be more efficient for handling small blocks.
29
30. Short-lived, short blocks: Most of the dynamic store falls into this
31 category. It is implemented as a stacking pool (POOL_MAIN) which is reset
32 after accepting a message when multiple messages are received by a single
33 process. Resetting happens at some other times as well, usually fairly
34 locally after some specific processing that needs working store.
35
36. There is a separate pool (POOL_SEARCH) that is used only for lookup storage.
37 This means it can be freed when search_tidyup() is called to close down all
38 the lookup caching.
f3ebb786
JH
39
40. Orthogonal to the three pool types, there are two classes of memory: untainted
41 and tainted. The latter is used for values derived from untrusted input, and
42 the string-expansion mechanism refuses to operate on such values (obviously,
43 it can expand an untainted value to return a tainted result). The classes
adc4ecf9 44 are implemented by duplicating the three pool types. Pool resets are requested
f3ebb786 45 against the nontainted sibling and apply to both siblings.
adc4ecf9
JH
46
47 Only memory blocks requested for tainted use are regarded as tainted; anything
48 else (including stack auto variables) is untainted. Care is needed when coding
49 to not copy untrusted data into untainted memory, as downstream taint-checks
50 would be avoided.
51
adc4ecf9
JH
52 Internally we currently use malloc for nontainted pools, and mmap for tainted
53 pools. The disparity is for speed of testing the taintedness of pointers;
54 because Linux appears to use distinct non-overlapping address allocations for
55 mmap vs. everything else, which means only two pointer-compares suffice for the
56 test. Other OS' cannot use that optimisation, and a more lengthy test against
57 the limits of tainted-pool allcations has to be done.
2fd4074d
JH
58
59 Intermediate layers (eg. the string functions) can test for taint, and use this
60 for ensurinng that results have proper state. For example the
61 string_vformat_trc() routing supporting the string_sprintf() interface will
62 recopy a string being built into a tainted allocation if it meets a %s for a
63 tainted argument. Any intermediate-layer function that (can) return a new
64 allocation should behave this way; returning a tainted result if any tainted
65 content is used. Users of functions that modify existing allocations should
66 check if a tainted source and an untainted destination is used, and fail instead
67 (sprintf() being the classic case).
059ec3d9
PH
68*/
69
70
71#include "exim.h"
438257ba
PP
72/* keep config.h before memcheck.h, for NVALGRIND */
73#include "config.h"
74
f3ebb786 75#include <sys/mman.h>
7f36d675 76#include "memcheck.h"
059ec3d9
PH
77
78
79/* We need to know how to align blocks of data for general use. I'm not sure
80how to get an alignment factor in general. In the current world, a value of 8
81is probably right, and this is sizeof(double) on some systems and sizeof(void
82*) on others, so take the larger of those. Since everything in this expression
83is a constant, the compiler should optimize it to a simple constant wherever it
84appears (I checked that gcc does do this). */
85
86#define alignment \
f3ebb786 87 (sizeof(void *) > sizeof(double) ? sizeof(void *) : sizeof(double))
059ec3d9
PH
88
89/* store_reset() will not free the following block if the last used block has
90less than this much left in it. */
91
92#define STOREPOOL_MIN_SIZE 256
93
94/* Structure describing the beginning of each big block. */
95
96typedef struct storeblock {
97 struct storeblock *next;
98 size_t length;
99} storeblock;
100
101/* Just in case we find ourselves on a system where the structure above has a
102length that is not a multiple of the alignment, set up a macro for the padded
103length. */
104
105#define ALIGNED_SIZEOF_STOREBLOCK \
106 (((sizeof(storeblock) + alignment - 1) / alignment) * alignment)
107
f3ebb786
JH
108/* Size of block to get from malloc to carve up into smaller ones. This
109must be a multiple of the alignment. We assume that 8192 is going to be
110suitably aligned. */
111
112#define STORE_BLOCK_SIZE (8192 - ALIGNED_SIZEOF_STOREBLOCK)
113
059ec3d9
PH
114/* Variables holding data for the local pools of store. The current pool number
115is held in store_pool, which is global so that it can be changed from outside.
116Setting the initial length values to -1 forces a malloc for the first call,
117even if the length is zero (which is used for getting a point to reset to). */
118
f3ebb786 119int store_pool = POOL_MAIN;
059ec3d9 120
f3ebb786
JH
121#define NPOOLS 6
122static storeblock *chainbase[NPOOLS];
123static storeblock *current_block[NPOOLS];
124static void *next_yield[NPOOLS];
125static int yield_length[NPOOLS] = { -1, -1, -1, -1, -1, -1 };
126
127/* The limits of the tainted pools. Tracking these on new allocations enables
128a fast is_tainted implementation. We assume the kernel only allocates mmaps using
129one side or the other of data+heap, not both. */
130
6d5f5caf
JH
131void * tainted_base = (void *)-1;
132void * tainted_top = (void *)0;
059ec3d9
PH
133
134/* pool_malloc holds the amount of memory used by the store pools; this goes up
135and down as store is reset or released. nonpool_malloc is the total got by
136malloc from other calls; this doesn't go down because it is just freed by
137pointer. */
138
f3ebb786
JH
139static int pool_malloc;
140static int nonpool_malloc;
059ec3d9
PH
141
142/* This variable is set by store_get() to its yield, and by store_reset() to
143NULL. This enables string_cat() to optimize its store handling for very long
144strings. That's why the variable is global. */
145
f3ebb786
JH
146void *store_last_get[NPOOLS];
147
148/* These are purely for stats-gathering */
149
150static int nbytes[NPOOLS]; /* current bytes allocated */
151static int maxbytes[NPOOLS]; /* max number reached */
152static int nblocks[NPOOLS]; /* current number of blocks allocated */
153static int maxblocks[NPOOLS];
154static int n_nonpool_blocks; /* current number of direct store_malloc() blocks */
155static int max_nonpool_blocks;
156static int max_pool_malloc; /* max value for pool_malloc */
157static int max_nonpool_malloc; /* max value for nonpool_malloc */
158
159
81a559c8 160#ifndef COMPILE_UTILITY
f3ebb786
JH
161static const uschar * pooluse[NPOOLS] = {
162[POOL_MAIN] = US"main",
163[POOL_PERM] = US"perm",
164[POOL_SEARCH] = US"search",
165[POOL_TAINT_MAIN] = US"main",
166[POOL_TAINT_PERM] = US"perm",
167[POOL_TAINT_SEARCH] = US"search",
168};
169static const uschar * poolclass[NPOOLS] = {
170[POOL_MAIN] = US"untainted",
171[POOL_PERM] = US"untainted",
172[POOL_SEARCH] = US"untainted",
173[POOL_TAINT_MAIN] = US"tainted",
174[POOL_TAINT_PERM] = US"tainted",
175[POOL_TAINT_SEARCH] = US"tainted",
176};
81a559c8 177#endif
f3ebb786
JH
178
179
180static void * store_mmap(int, const char *, int);
181static void * internal_store_malloc(int, const char *, int);
65766f1b
JH
182static void internal_untainted_free(void *, const char *, int linenumber);
183static void internal_tainted_free(storeblock *, const char *, int linenumber);
f3ebb786
JH
184
185/******************************************************************************/
186
f096bccc 187#ifndef TAINT_CHECK_FAST
2fd4074d
JH
188/* Test if a pointer refers to tainted memory.
189
190Slower version check, for use when platform intermixes malloc and mmap area
191addresses. Test against the current-block of all tainted pools first, then all
192blocks of all tainted pools.
193
194Return: TRUE iff tainted
195*/
14ca5d2a
JH
196
197BOOL
198is_tainted_fn(const void * p)
199{
200storeblock * b;
201int pool;
202
2fd4074d 203for (pool = POOL_TAINT_BASE; pool < nelem(chainbase); pool++)
14ca5d2a
JH
204 if ((b = current_block[pool]))
205 {
206 char * bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
2fd4074d 207 if (CS p >= bc && CS p <= bc + b->length) return TRUE;
14ca5d2a
JH
208 }
209
2fd4074d 210for (pool = POOL_TAINT_BASE; pool < nelem(chainbase); pool++)
14ca5d2a
JH
211 for (b = chainbase[pool]; b; b = b->next)
212 {
213 char * bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
2fd4074d 214 if (CS p >= bc && CS p <= bc + b->length) return TRUE;
14ca5d2a
JH
215 }
216return FALSE;
14ca5d2a 217}
f096bccc 218#endif
14ca5d2a
JH
219
220
f3ebb786
JH
221void
222die_tainted(const uschar * msg, const uschar * func, int line)
223{
224log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Taint mismatch, %s: %s %d\n",
225 msg, func, line);
226}
059ec3d9
PH
227
228
229/*************************************************
230* Get a block from the current pool *
231*************************************************/
232
233/* Running out of store is a total disaster. This function is called via the
234macro store_get(). It passes back a block of store within the current big
235block, getting a new one if necessary. The address is saved in
236store_last_was_get.
237
238Arguments:
adc4ecf9
JH
239 size amount wanted, bytes
240 tainted class: set to true for untrusted data (eg. from smtp input)
f3ebb786
JH
241 func function from which called
242 linenumber line number in source file
059ec3d9
PH
243
244Returns: pointer to store (panic on malloc failure)
245*/
246
247void *
f3ebb786 248store_get_3(int size, BOOL tainted, const char *func, int linenumber)
059ec3d9 249{
f3ebb786
JH
250int pool = tainted ? store_pool + POOL_TAINT_BASE : store_pool;
251
059ec3d9
PH
252/* Round up the size to a multiple of the alignment. Although this looks a
253messy statement, because "alignment" is a constant expression, the compiler can
254do a reasonable job of optimizing, especially if the value of "alignment" is a
255power of two. I checked this with -O2, and gcc did very well, compiling it to 4
256instructions on a Sparc (alignment = 8). */
257
258if (size % alignment != 0) size += alignment - (size % alignment);
259
260/* If there isn't room in the current block, get a new one. The minimum
261size is STORE_BLOCK_SIZE, and we would expect this to be the norm, since
262these functions are mostly called for small amounts of store. */
263
f3ebb786 264if (size > yield_length[pool])
059ec3d9 265 {
f3ebb786 266 int length = size <= STORE_BLOCK_SIZE ? STORE_BLOCK_SIZE : size;
059ec3d9 267 int mlength = length + ALIGNED_SIZEOF_STOREBLOCK;
f3ebb786 268 storeblock * newblock;
059ec3d9
PH
269
270 /* Sometimes store_reset() may leave a block for us; check if we can use it */
271
f3ebb786 272 if ( (newblock = current_block[pool])
64073d9c
JH
273 && (newblock = newblock->next)
274 && newblock->length < length
275 )
059ec3d9 276 {
64073d9c 277 /* Give up on this block, because it's too small */
f3ebb786
JH
278 nblocks[pool]--;
279 if (pool < POOL_TAINT_BASE)
65766f1b 280 internal_untainted_free(newblock, func, linenumber);
f3ebb786 281 else
65766f1b 282 internal_tainted_free(newblock, func, linenumber);
64073d9c 283 newblock = NULL;
059ec3d9
PH
284 }
285
286 /* If there was no free block, get a new one */
287
64073d9c 288 if (!newblock)
059ec3d9 289 {
f3ebb786
JH
290 if ((nbytes[pool] += mlength) > maxbytes[pool])
291 maxbytes[pool] = nbytes[pool];
292 if ((pool_malloc += mlength) > max_pool_malloc) /* Used in pools */
293 max_pool_malloc = pool_malloc;
294 nonpool_malloc -= mlength; /* Exclude from overall total */
295 if (++nblocks[pool] > maxblocks[pool])
296 maxblocks[pool] = nblocks[pool];
297
298 newblock = tainted
299 ? store_mmap(mlength, func, linenumber)
300 : internal_store_malloc(mlength, func, linenumber);
059ec3d9
PH
301 newblock->next = NULL;
302 newblock->length = length;
f3ebb786
JH
303
304 if (!chainbase[pool])
305 chainbase[pool] = newblock;
64073d9c 306 else
f3ebb786 307 current_block[pool]->next = newblock;
059ec3d9
PH
308 }
309
f3ebb786
JH
310 current_block[pool] = newblock;
311 yield_length[pool] = newblock->length;
312 next_yield[pool] =
313 (void *)(CS current_block[pool] + ALIGNED_SIZEOF_STOREBLOCK);
314 (void) VALGRIND_MAKE_MEM_NOACCESS(next_yield[pool], yield_length[pool]);
059ec3d9
PH
315 }
316
317/* There's (now) enough room in the current block; the yield is the next
318pointer. */
319
f3ebb786 320store_last_get[pool] = next_yield[pool];
059ec3d9
PH
321
322/* Cut out the debugging stuff for utilities, but stop picky compilers from
323giving warnings. */
324
325#ifdef COMPILE_UTILITY
f3ebb786 326func = func;
059ec3d9
PH
327linenumber = linenumber;
328#else
329DEBUG(D_memory)
f3ebb786
JH
330 debug_printf("---%d Get %6p %5d %-14s %4d\n", pool,
331 store_last_get[pool], size, func, linenumber);
059ec3d9
PH
332#endif /* COMPILE_UTILITY */
333
f3ebb786 334(void) VALGRIND_MAKE_MEM_UNDEFINED(store_last_get[pool], size);
059ec3d9
PH
335/* Update next pointer and number of bytes left in the current block. */
336
f3ebb786
JH
337next_yield[pool] = (void *)(CS next_yield[pool] + size);
338yield_length[pool] -= size;
339return store_last_get[pool];
059ec3d9
PH
340}
341
342
343
344/*************************************************
345* Get a block from the PERM pool *
346*************************************************/
347
348/* This is just a convenience function, useful when just a single block is to
349be obtained.
350
351Arguments:
352 size amount wanted
f3ebb786
JH
353 func function from which called
354 linenumber line number in source file
059ec3d9
PH
355
356Returns: pointer to store (panic on malloc failure)
357*/
358
359void *
f3ebb786 360store_get_perm_3(int size, BOOL tainted, const char *func, int linenumber)
059ec3d9
PH
361{
362void *yield;
363int old_pool = store_pool;
364store_pool = POOL_PERM;
f3ebb786 365yield = store_get_3(size, tainted, func, linenumber);
059ec3d9
PH
366store_pool = old_pool;
367return yield;
368}
369
370
371
372/*************************************************
373* Extend a block if it is at the top *
374*************************************************/
375
376/* While reading strings of unknown length, it is often the case that the
377string is being read into the block at the top of the stack. If it needs to be
f3ebb786 378extended, it is more efficient just to extend within the top block rather than
059ec3d9
PH
379allocate a new block and then have to copy the data. This function is provided
380for the use of string_cat(), but of course can be used elsewhere too.
f3ebb786 381The block itself is not expanded; only the top allocation from it.
059ec3d9
PH
382
383Arguments:
384 ptr pointer to store block
385 oldsize current size of the block, as requested by user
386 newsize new size required
f3ebb786 387 func function from which called
059ec3d9
PH
388 linenumber line number in source file
389
390Returns: TRUE if the block is at the top of the stack and has been
391 extended; FALSE if it isn't at the top of the stack, or cannot
392 be extended
393*/
394
395BOOL
f3ebb786
JH
396store_extend_3(void *ptr, BOOL tainted, int oldsize, int newsize,
397 const char *func, int linenumber)
059ec3d9 398{
f3ebb786 399int pool = tainted ? store_pool + POOL_TAINT_BASE : store_pool;
059ec3d9
PH
400int inc = newsize - oldsize;
401int rounded_oldsize = oldsize;
402
f3ebb786
JH
403/* Check that the block being extended was already of the required taint status;
404refuse to extend if not. */
405
406if (is_tainted(ptr) != tainted)
407 return FALSE;
408
059ec3d9
PH
409if (rounded_oldsize % alignment != 0)
410 rounded_oldsize += alignment - (rounded_oldsize % alignment);
411
f3ebb786
JH
412if (CS ptr + rounded_oldsize != CS (next_yield[pool]) ||
413 inc > yield_length[pool] + rounded_oldsize - oldsize)
059ec3d9
PH
414 return FALSE;
415
416/* Cut out the debugging stuff for utilities, but stop picky compilers from
417giving warnings. */
418
419#ifdef COMPILE_UTILITY
f3ebb786 420func = func;
059ec3d9
PH
421linenumber = linenumber;
422#else
423DEBUG(D_memory)
f3ebb786
JH
424 debug_printf("---%d Ext %6p %5d %-14s %4d\n", pool, ptr, newsize,
425 func, linenumber);
059ec3d9
PH
426#endif /* COMPILE_UTILITY */
427
428if (newsize % alignment != 0) newsize += alignment - (newsize % alignment);
f3ebb786
JH
429next_yield[pool] = CS ptr + newsize;
430yield_length[pool] -= newsize - rounded_oldsize;
4d8bb202 431(void) VALGRIND_MAKE_MEM_UNDEFINED(ptr + oldsize, inc);
059ec3d9
PH
432return TRUE;
433}
434
435
436
437
438/*************************************************
439* Back up to a previous point on the stack *
440*************************************************/
441
442/* This function resets the next pointer, freeing any subsequent whole blocks
f3ebb786
JH
443that are now unused. Call with a cookie obtained from store_mark() only; do
444not call with a pointer returned by store_get(). Both the untainted and tainted
445pools corresposding to store_pool are reset.
059ec3d9
PH
446
447Arguments:
f3ebb786
JH
448 r place to back up to
449 func function from which called
059ec3d9
PH
450 linenumber line number in source file
451
452Returns: nothing
453*/
454
f3ebb786
JH
455static void
456internal_store_reset(void * ptr, int pool, const char *func, int linenumber)
059ec3d9 457{
cf0812d5 458storeblock * bb;
f3ebb786 459storeblock * b = current_block[pool];
cf0812d5 460char * bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
f3ebb786
JH
461int newlength, count;
462#ifndef COMPILE_UTILITY
463int oldmalloc = pool_malloc;
464#endif
059ec3d9
PH
465
466/* Last store operation was not a get */
467
f3ebb786 468store_last_get[pool] = NULL;
059ec3d9
PH
469
470/* See if the place is in the current block - as it often will be. Otherwise,
471search for the block in which it lies. */
472
cf0812d5 473if (CS ptr < bc || CS ptr > bc + b->length)
059ec3d9 474 {
f3ebb786 475 for (b = chainbase[pool]; b; b = b->next)
059ec3d9 476 {
cf0812d5
JH
477 bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
478 if (CS ptr >= bc && CS ptr <= bc + b->length) break;
059ec3d9 479 }
cf0812d5 480 if (!b)
438257ba 481 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "internal error: store_reset(%p) "
f3ebb786 482 "failed: pool=%d %-14s %4d", ptr, pool, func, linenumber);
059ec3d9
PH
483 }
484
485/* Back up, rounding to the alignment if necessary. When testing, flatten
486the released memory. */
487
cf0812d5 488newlength = bc + b->length - CS ptr;
059ec3d9 489#ifndef COMPILE_UTILITY
65a32f85 490if (debug_store)
2c9f7ff8 491 {
f3ebb786 492 assert_no_variables(ptr, newlength, func, linenumber);
8768d548 493 if (f.running_in_test_harness)
64073d9c
JH
494 {
495 (void) VALGRIND_MAKE_MEM_DEFINED(ptr, newlength);
496 memset(ptr, 0xF0, newlength);
497 }
2c9f7ff8 498 }
059ec3d9 499#endif
4d8bb202 500(void) VALGRIND_MAKE_MEM_NOACCESS(ptr, newlength);
f3ebb786
JH
501next_yield[pool] = CS ptr + (newlength % alignment);
502count = yield_length[pool];
503count = (yield_length[pool] = newlength - (newlength % alignment)) - count;
504current_block[pool] = b;
505
506/* Free any subsequent block. Do NOT free the first
507successor, if our current block has less than 256 bytes left. This should
508prevent us from flapping memory. However, keep this block only when it has
509the default size. */
510
511if ( yield_length[pool] < STOREPOOL_MIN_SIZE
512 && b->next
513 && b->next->length == STORE_BLOCK_SIZE)
7f36d675 514 {
059ec3d9 515 b = b->next;
cf0812d5 516#ifndef COMPILE_UTILITY
65a32f85 517 if (debug_store)
cf0812d5 518 assert_no_variables(b, b->length + ALIGNED_SIZEOF_STOREBLOCK,
f3ebb786 519 func, linenumber);
cf0812d5
JH
520#endif
521 (void) VALGRIND_MAKE_MEM_NOACCESS(CS b + ALIGNED_SIZEOF_STOREBLOCK,
4d8bb202 522 b->length - ALIGNED_SIZEOF_STOREBLOCK);
7f36d675 523 }
059ec3d9
PH
524
525bb = b->next;
526b->next = NULL;
527
cf0812d5 528while ((b = bb))
059ec3d9 529 {
f3ebb786 530 int siz = b->length + ALIGNED_SIZEOF_STOREBLOCK;
cf0812d5 531#ifndef COMPILE_UTILITY
65a32f85 532 if (debug_store)
cf0812d5 533 assert_no_variables(b, b->length + ALIGNED_SIZEOF_STOREBLOCK,
f3ebb786 534 func, linenumber);
cf0812d5 535#endif
059ec3d9 536 bb = bb->next;
f3ebb786
JH
537 nbytes[pool] -= siz;
538 pool_malloc -= siz;
539 nblocks[pool]--;
540 if (pool < POOL_TAINT_BASE)
65766f1b 541 internal_untainted_free(b, func, linenumber);
f3ebb786 542 else
65766f1b 543 internal_tainted_free(b, func, linenumber);
059ec3d9
PH
544 }
545
546/* Cut out the debugging stuff for utilities, but stop picky compilers from
547giving warnings. */
548
549#ifdef COMPILE_UTILITY
f3ebb786 550func = func;
059ec3d9
PH
551linenumber = linenumber;
552#else
553DEBUG(D_memory)
f3ebb786
JH
554 debug_printf("---%d Rst %6p %5d %-14s %4d %d\n", pool, ptr,
555 count + oldmalloc - pool_malloc,
556 func, linenumber, pool_malloc);
557#endif /* COMPILE_UTILITY */
558}
559
560
561rmark
562store_reset_3(rmark r, int pool, const char *func, int linenumber)
563{
564void ** ptr = r;
565
566if (pool >= POOL_TAINT_BASE)
567 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
568 "store_reset called for pool %d: %s %d\n", pool, func, linenumber);
569if (!r)
570 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
571 "store_reset called with bad mark: %s %d\n", func, linenumber);
572
573internal_store_reset(*ptr, pool + POOL_TAINT_BASE, func, linenumber);
574internal_store_reset(ptr, pool, func, linenumber);
575return NULL;
576}
577
578
579
580/* Free tail-end unused allocation. This lets us allocate a big chunk
581early, for cases when we only discover later how much was really needed.
582
583Can be called with a value from store_get(), or an offset after such. Only
584the tainted or untainted pool that serviced the store_get() will be affected.
585
586This is mostly a cut-down version of internal_store_reset().
587XXX needs rationalising
588*/
589
590void
591store_release_above_3(void *ptr, const char *func, int linenumber)
592{
593/* Search all pools' "current" blocks. If it isn't one of those,
594ignore it (it usually will be). */
595
596for (int pool = 0; pool < nelem(current_block); pool++)
059ec3d9 597 {
f3ebb786
JH
598 storeblock * b = current_block[pool];
599 char * bc;
600 int count, newlength;
601
602 if (!b)
603 continue;
604
605 bc = CS b + ALIGNED_SIZEOF_STOREBLOCK;
606 if (CS ptr < bc || CS ptr > bc + b->length)
607 continue;
608
609 /* Last store operation was not a get */
610
611 store_last_get[pool] = NULL;
612
613 /* Back up, rounding to the alignment if necessary. When testing, flatten
614 the released memory. */
615
616 newlength = bc + b->length - CS ptr;
617#ifndef COMPILE_UTILITY
618 if (debug_store)
619 {
620 assert_no_variables(ptr, newlength, func, linenumber);
621 if (f.running_in_test_harness)
622 {
623 (void) VALGRIND_MAKE_MEM_DEFINED(ptr, newlength);
624 memset(ptr, 0xF0, newlength);
625 }
626 }
627#endif
628 (void) VALGRIND_MAKE_MEM_NOACCESS(ptr, newlength);
629 next_yield[pool] = CS ptr + (newlength % alignment);
630 count = yield_length[pool];
631 count = (yield_length[pool] = newlength - (newlength % alignment)) - count;
632
633 /* Cut out the debugging stuff for utilities, but stop picky compilers from
634 giving warnings. */
635
636#ifdef COMPILE_UTILITY
637 func = func;
638 linenumber = linenumber;
639#else
640 DEBUG(D_memory)
641 debug_printf("---%d Rel %6p %5d %-14s %4d %d\n", pool, ptr, count,
642 func, linenumber, pool_malloc);
643#endif
644 return;
059ec3d9 645 }
f3ebb786
JH
646#ifndef COMPILE_UTILITY
647DEBUG(D_memory)
648 debug_printf("non-last memory release try: %s %d\n", func, linenumber);
649#endif
059ec3d9
PH
650}
651
652
653
f3ebb786
JH
654rmark
655store_mark_3(const char *func, int linenumber)
656{
657void ** p;
658
659if (store_pool >= POOL_TAINT_BASE)
660 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
661 "store_mark called for pool %d: %s %d\n", store_pool, func, linenumber);
662
663/* Stash a mark for the tainted-twin release, in the untainted twin. Return
664a cookie (actually the address in the untainted pool) to the caller.
665Reset uses the cookie to recover the t-mark, winds back the tainted pool with it
666and winds back the untainted pool with the cookie. */
667
668p = store_get_3(sizeof(void *), FALSE, func, linenumber);
669*p = store_get_3(0, TRUE, func, linenumber);
670return p;
671}
672
673
059ec3d9
PH
674
675
676/************************************************
677* Release store *
678************************************************/
679
459fca58
JH
680/* This function checks that the pointer it is given is the first thing in a
681block, and if so, releases that block.
059ec3d9
PH
682
683Arguments:
684 block block of store to consider
f3ebb786 685 func function from which called
059ec3d9
PH
686 linenumber line number in source file
687
688Returns: nothing
689*/
690
459fca58 691static void
f3ebb786 692store_release_3(void * block, int pool, const char * func, int linenumber)
059ec3d9 693{
059ec3d9
PH
694/* It will never be the first block, so no need to check that. */
695
f3ebb786 696for (storeblock * b = chainbase[pool]; b; b = b->next)
059ec3d9 697 {
459fca58
JH
698 storeblock * bb = b->next;
699 if (bb && CS block == CS bb + ALIGNED_SIZEOF_STOREBLOCK)
059ec3d9 700 {
f3ebb786 701 int siz = bb->length + ALIGNED_SIZEOF_STOREBLOCK;
059ec3d9 702 b->next = bb->next;
f3ebb786
JH
703 nbytes[pool] -= siz;
704 pool_malloc -= siz;
705 nblocks[pool]--;
059ec3d9
PH
706
707 /* Cut out the debugging stuff for utilities, but stop picky compilers
708 from giving warnings. */
709
459fca58 710#ifdef COMPILE_UTILITY
f3ebb786 711 func = func;
059ec3d9 712 linenumber = linenumber;
459fca58 713#else
059ec3d9 714 DEBUG(D_memory)
f3ebb786
JH
715 debug_printf("-Release %6p %-20s %4d %d\n", (void *)bb, func,
716 linenumber, pool_malloc);
459fca58 717
8768d548 718 if (f.running_in_test_harness)
059ec3d9 719 memset(bb, 0xF0, bb->length+ALIGNED_SIZEOF_STOREBLOCK);
459fca58 720#endif /* COMPILE_UTILITY */
059ec3d9
PH
721
722 free(bb);
723 return;
724 }
725 }
726}
727
728
459fca58
JH
729/************************************************
730* Move store *
731************************************************/
732
733/* Allocate a new block big enough to expend to the given size and
734copy the current data into it. Free the old one if possible.
735
736This function is specifically provided for use when reading very
737long strings, e.g. header lines. When the string gets longer than a
738complete block, it gets copied to a new block. It is helpful to free
739the old block iff the previous copy of the string is at its start,
740and therefore the only thing in it. Otherwise, for very long strings,
741dead store can pile up somewhat disastrously. This function checks that
742the pointer it is given is the first thing in a block, and that nothing
743has been allocated since. If so, releases that block.
744
745Arguments:
746 block
747 newsize
748 len
749
750Returns: new location of data
751*/
752
753void *
f3ebb786
JH
754store_newblock_3(void * block, BOOL tainted, int newsize, int len,
755 const char * func, int linenumber)
459fca58 756{
f3ebb786
JH
757int pool = tainted ? store_pool + POOL_TAINT_BASE : store_pool;
758BOOL release_ok = !tainted && store_last_get[pool] == block;
759uschar * newtext;
760
aaabfafe 761#ifndef MACRO_PREDEF
f3ebb786
JH
762if (is_tainted(block) != tainted)
763 die_tainted(US"store_newblock", CUS func, linenumber);
aaabfafe 764#endif
459fca58 765
f3ebb786 766newtext = store_get(newsize, tainted);
459fca58 767memcpy(newtext, block, len);
f3ebb786 768if (release_ok) store_release_3(block, pool, func, linenumber);
459fca58
JH
769return (void *)newtext;
770}
771
772
059ec3d9
PH
773
774
f3ebb786
JH
775/******************************************************************************/
776static void *
777store_alloc_tail(void * yield, int size, const char * func, int line,
778 const uschar * type)
779{
780if ((nonpool_malloc += size) > max_nonpool_malloc)
781 max_nonpool_malloc = nonpool_malloc;
782
783/* Cut out the debugging stuff for utilities, but stop picky compilers from
784giving warnings. */
785
786#ifdef COMPILE_UTILITY
787func = func; line = line; type = type;
788#else
789
790/* If running in test harness, spend time making sure all the new store
791is not filled with zeros so as to catch problems. */
792
793if (f.running_in_test_harness)
794 memset(yield, 0xF0, (size_t)size);
795DEBUG(D_memory) debug_printf("--%6s %6p %5d bytes\t%-14s %4d\tpool %5d nonpool %5d\n",
796 type, yield, size, func, line, pool_malloc, nonpool_malloc);
797#endif /* COMPILE_UTILITY */
798
799return yield;
800}
801
802/*************************************************
803* Mmap store *
804*************************************************/
805
806static void *
807store_mmap(int size, const char * func, int line)
808{
809void * yield, * top;
810
811if (size < 16) size = 16;
812
813if (!(yield = mmap(NULL, (size_t)size,
814 PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)))
815 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to mmap %d bytes of memory: "
816 "called from line %d of %s", size, line, func);
817
818if (yield < tainted_base) tainted_base = yield;
6d95688d 819if ((top = US yield + size) > tainted_top) tainted_top = top;
f3ebb786
JH
820
821return store_alloc_tail(yield, size, func, line, US"Mmap");
822}
823
059ec3d9
PH
824/*************************************************
825* Malloc store *
826*************************************************/
827
828/* Running out of store is a total disaster for exim. Some malloc functions
829do not run happily on very small sizes, nor do they document this fact. This
830function is called via the macro store_malloc().
831
832Arguments:
833 size amount of store wanted
f3ebb786 834 func function from which called
059ec3d9
PH
835 linenumber line number in source file
836
837Returns: pointer to gotten store (panic on failure)
838*/
839
f3ebb786
JH
840static void *
841internal_store_malloc(int size, const char *func, int linenumber)
059ec3d9 842{
f3ebb786 843void * yield;
059ec3d9
PH
844
845if (size < 16) size = 16;
059ec3d9 846
40c90bca 847if (!(yield = malloc((size_t)size)))
059ec3d9 848 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to malloc %d bytes of memory: "
f3ebb786 849 "called from line %d in %s", size, linenumber, func);
059ec3d9 850
f3ebb786
JH
851return store_alloc_tail(yield, size, func, linenumber, US"Malloc");
852}
059ec3d9 853
f3ebb786
JH
854void *
855store_malloc_3(int size, const char *func, int linenumber)
856{
857if (n_nonpool_blocks++ > max_nonpool_blocks)
858 max_nonpool_blocks = n_nonpool_blocks;
859return internal_store_malloc(size, func, linenumber);
059ec3d9
PH
860}
861
862
863/************************************************
864* Free store *
865************************************************/
866
867/* This function is called by the macro store_free().
868
869Arguments:
870 block block of store to free
f3ebb786 871 func function from which called
059ec3d9
PH
872 linenumber line number in source file
873
874Returns: nothing
875*/
876
f3ebb786 877static void
65766f1b 878internal_untainted_free(void * block, const char * func, int linenumber)
059ec3d9
PH
879{
880#ifdef COMPILE_UTILITY
f3ebb786 881func = func;
059ec3d9
PH
882linenumber = linenumber;
883#else
884DEBUG(D_memory)
f3ebb786 885 debug_printf("----Free %6p %-20s %4d\n", block, func, linenumber);
059ec3d9
PH
886#endif /* COMPILE_UTILITY */
887free(block);
888}
889
f3ebb786 890void
65766f1b 891store_free_3(void * block, const char * func, int linenumber)
f3ebb786
JH
892{
893n_nonpool_blocks--;
65766f1b
JH
894internal_untainted_free(block, func, linenumber);
895}
896
897/******************************************************************************/
898static void
899internal_tainted_free(storeblock * block, const char * func, int linenumber)
900{
901#ifdef COMPILE_UTILITY
902func = func;
903linenumber = linenumber;
904#else
905DEBUG(D_memory)
906 debug_printf("---Unmap %6p %-20s %4d\n", block, func, linenumber);
907#endif
908munmap((void *)block, block->length + ALIGNED_SIZEOF_STOREBLOCK);
f3ebb786
JH
909}
910
911/******************************************************************************/
912/* Stats output on process exit */
913void
914store_exit(void)
915{
916#ifndef COMPILE_UTILITY
917DEBUG(D_memory)
918 {
919 debug_printf("----Exit nonpool max: %3d kB in %d blocks\n",
920 (max_nonpool_malloc+1023)/1024, max_nonpool_blocks);
921 debug_printf("----Exit npools max: %3d kB\n", max_pool_malloc/1024);
922 for (int i = 0; i < NPOOLS; i++)
923 debug_printf("----Exit pool %d max: %3d kB in %d blocks\t%s %s\n",
924 i, maxbytes[i]/1024, maxblocks[i], poolclass[i], pooluse[i]);
925 }
926#endif
927}
928
059ec3d9 929/* End of store.c */