1 #include "jemalloc/internal/jemalloc_preamble.h"
2 #include "jemalloc/internal/jemalloc_internal_includes.h"
3
4 #include "jemalloc/internal/malloc_io.h"
5 #include "jemalloc/internal/util.h"
6
7 #ifdef assert
8 # undef assert
9 #endif
10 #ifdef not_reached
11 # undef not_reached
12 #endif
13 #ifdef not_implemented
14 # undef not_implemented
15 #endif
16 #ifdef assert_not_implemented
17 # undef assert_not_implemented
18 #endif
19
20 /*
21 * Define simple versions of assertion macros that won't recurse in case
22 * of assertion failures in malloc_*printf().
23 */
24 #define assert(e) do { \
25 if (config_debug && !(e)) { \
26 malloc_write("<jemalloc>: Failed assertion\n"); \
27 abort(); \
28 } \
29 } while (0)
30
31 #define not_reached() do { \
32 if (config_debug) { \
33 malloc_write("<jemalloc>: Unreachable code reached\n"); \
34 abort(); \
35 } \
36 unreachable(); \
37 } while (0)
38
39 #define not_implemented() do { \
40 if (config_debug) { \
41 malloc_write("<jemalloc>: Not implemented\n"); \
42 abort(); \
43 } \
44 } while (0)
45
46 #define assert_not_implemented(e) do { \
47 if (unlikely(config_debug && !(e))) { \
48 not_implemented(); \
49 } \
50 } while (0)
51
52 /******************************************************************************/
53 /* Function prototypes for non-inline static functions. */
54
55 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
56 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
57 size_t *slen_p);
58 #define D2S_BUFSIZE (1 + U2S_BUFSIZE)
59 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p);
60 #define O2S_BUFSIZE (1 + U2S_BUFSIZE)
61 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
62 #define X2S_BUFSIZE (2 + U2S_BUFSIZE)
63 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
64 size_t *slen_p);
65
66 /******************************************************************************/
67
68 /* malloc_message() setup. */
69 void
wrtmessage(void * cbopaque,const char * s)70 wrtmessage(void *cbopaque, const char *s) {
71 malloc_write_fd(STDERR_FILENO, s, strlen(s));
72 }
73
74 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
75
76 JEMALLOC_ATTR(visibility("hidden"))
77 void
wrtmessage_1_0(const char * s1,const char * s2,const char * s3,const char * s4)78 wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
79
80 wrtmessage(NULL, s1);
81 wrtmessage(NULL, s2);
82 wrtmessage(NULL, s3);
83 wrtmessage(NULL, s4);
84 }
85
86 void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
87 const char *s4) = wrtmessage_1_0;
88 __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
89
90 /*
91 * Wrapper around malloc_message() that avoids the need for
92 * je_malloc_message(...) throughout the code.
93 */
94 void
malloc_write(const char * s)95 malloc_write(const char *s) {
96 if (je_malloc_message != NULL) {
97 je_malloc_message(NULL, s);
98 } else {
99 wrtmessage(NULL, s);
100 }
101 }
102
103 /*
104 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
105 * provide a wrapper.
106 */
107 int
buferror(int err,char * buf,size_t buflen)108 buferror(int err, char *buf, size_t buflen) {
109 #ifdef _WIN32
110 FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
111 (LPSTR)buf, (DWORD)buflen, NULL);
112 return 0;
113 #elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE)
114 char *b = strerror_r(err, buf, buflen);
115 if (b != buf) {
116 strncpy(buf, b, buflen);
117 buf[buflen-1] = '\0';
118 }
119 return 0;
120 #else
121 return strerror_r(err, buf, buflen);
122 #endif
123 }
124
125 uintmax_t
malloc_strtoumax(const char * restrict nptr,char ** restrict endptr,int base)126 malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
127 uintmax_t ret, digit;
128 unsigned b;
129 bool neg;
130 const char *p, *ns;
131
132 p = nptr;
133 if (base < 0 || base == 1 || base > 36) {
134 ns = p;
135 set_errno(EINVAL);
136 ret = UINTMAX_MAX;
137 goto label_return;
138 }
139 b = base;
140
141 /* Swallow leading whitespace and get sign, if any. */
142 neg = false;
143 while (true) {
144 switch (*p) {
145 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
146 p++;
147 break;
148 case '-':
149 neg = true;
150 JEMALLOC_FALLTHROUGH;
151 case '+':
152 p++;
153 JEMALLOC_FALLTHROUGH;
154 default:
155 goto label_prefix;
156 }
157 }
158
159 /* Get prefix, if any. */
160 label_prefix:
161 /*
162 * Note where the first non-whitespace/sign character is so that it is
163 * possible to tell whether any digits are consumed (e.g., " 0" vs.
164 * " -x").
165 */
166 ns = p;
167 if (*p == '0') {
168 switch (p[1]) {
169 case '0': case '1': case '2': case '3': case '4': case '5':
170 case '6': case '7':
171 if (b == 0) {
172 b = 8;
173 }
174 if (b == 8) {
175 p++;
176 }
177 break;
178 case 'X': case 'x':
179 switch (p[2]) {
180 case '0': case '1': case '2': case '3': case '4':
181 case '5': case '6': case '7': case '8': case '9':
182 case 'A': case 'B': case 'C': case 'D': case 'E':
183 case 'F':
184 case 'a': case 'b': case 'c': case 'd': case 'e':
185 case 'f':
186 if (b == 0) {
187 b = 16;
188 }
189 if (b == 16) {
190 p += 2;
191 }
192 break;
193 default:
194 break;
195 }
196 break;
197 default:
198 p++;
199 ret = 0;
200 goto label_return;
201 }
202 }
203 if (b == 0) {
204 b = 10;
205 }
206
207 /* Convert. */
208 ret = 0;
209 while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
210 || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
211 || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
212 uintmax_t pret = ret;
213 ret *= b;
214 ret += digit;
215 if (ret < pret) {
216 /* Overflow. */
217 set_errno(ERANGE);
218 ret = UINTMAX_MAX;
219 goto label_return;
220 }
221 p++;
222 }
223 if (neg) {
224 ret = (uintmax_t)(-((intmax_t)ret));
225 }
226
227 if (p == ns) {
228 /* No conversion performed. */
229 set_errno(EINVAL);
230 ret = UINTMAX_MAX;
231 goto label_return;
232 }
233
234 label_return:
235 if (endptr != NULL) {
236 if (p == ns) {
237 /* No characters were converted. */
238 *endptr = (char *)nptr;
239 } else {
240 *endptr = (char *)p;
241 }
242 }
243 return ret;
244 }
245
246 static char *
u2s(uintmax_t x,unsigned base,bool uppercase,char * s,size_t * slen_p)247 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
248 unsigned i;
249
250 i = U2S_BUFSIZE - 1;
251 s[i] = '\0';
252 switch (base) {
253 case 10:
254 do {
255 i--;
256 s[i] = "0123456789"[x % (uint64_t)10];
257 x /= (uint64_t)10;
258 } while (x > 0);
259 break;
260 case 16: {
261 const char *digits = (uppercase)
262 ? "0123456789ABCDEF"
263 : "0123456789abcdef";
264
265 do {
266 i--;
267 s[i] = digits[x & 0xf];
268 x >>= 4;
269 } while (x > 0);
270 break;
271 } default: {
272 const char *digits = (uppercase)
273 ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
274 : "0123456789abcdefghijklmnopqrstuvwxyz";
275
276 assert(base >= 2 && base <= 36);
277 do {
278 i--;
279 s[i] = digits[x % (uint64_t)base];
280 x /= (uint64_t)base;
281 } while (x > 0);
282 }}
283
284 *slen_p = U2S_BUFSIZE - 1 - i;
285 return &s[i];
286 }
287
288 static char *
d2s(intmax_t x,char sign,char * s,size_t * slen_p)289 d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
290 bool neg;
291
292 if ((neg = (x < 0))) {
293 x = -x;
294 }
295 s = u2s(x, 10, false, s, slen_p);
296 if (neg) {
297 sign = '-';
298 }
299 switch (sign) {
300 case '-':
301 if (!neg) {
302 break;
303 }
304 JEMALLOC_FALLTHROUGH;
305 case ' ':
306 case '+':
307 s--;
308 (*slen_p)++;
309 *s = sign;
310 break;
311 default: not_reached();
312 }
313 return s;
314 }
315
316 static char *
o2s(uintmax_t x,bool alt_form,char * s,size_t * slen_p)317 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
318 s = u2s(x, 8, false, s, slen_p);
319 if (alt_form && *s != '0') {
320 s--;
321 (*slen_p)++;
322 *s = '0';
323 }
324 return s;
325 }
326
327 static char *
x2s(uintmax_t x,bool alt_form,bool uppercase,char * s,size_t * slen_p)328 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
329 s = u2s(x, 16, uppercase, s, slen_p);
330 if (alt_form) {
331 s -= 2;
332 (*slen_p) += 2;
333 memcpy(s, uppercase ? "0X" : "0x", 2);
334 }
335 return s;
336 }
337
338 JEMALLOC_COLD
339 size_t
malloc_vsnprintf(char * str,size_t size,const char * format,va_list ap)340 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
341 size_t i;
342 const char *f;
343
344 #define APPEND_C(c) do { \
345 if (i < size) { \
346 str[i] = (c); \
347 } \
348 i++; \
349 } while (0)
350 #define APPEND_S(s, slen) do { \
351 if (i < size) { \
352 size_t cpylen = (slen <= size - i) ? slen : size - i; \
353 memcpy(&str[i], s, cpylen); \
354 } \
355 i += slen; \
356 } while (0)
357 #define APPEND_PADDED_S(s, slen, width, left_justify) do { \
358 /* Left padding. */ \
359 size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \
360 (size_t)width - slen : 0); \
361 if (!left_justify && pad_len != 0) { \
362 size_t j; \
363 for (j = 0; j < pad_len; j++) { \
364 if (pad_zero) { \
365 APPEND_C('0'); \
366 } else { \
367 APPEND_C(' '); \
368 } \
369 } \
370 } \
371 /* Value. */ \
372 APPEND_S(s, slen); \
373 /* Right padding. */ \
374 if (left_justify && pad_len != 0) { \
375 size_t j; \
376 for (j = 0; j < pad_len; j++) { \
377 APPEND_C(' '); \
378 } \
379 } \
380 } while (0)
381 #define GET_ARG_NUMERIC(val, len) do { \
382 switch ((unsigned char)len) { \
383 case '?': \
384 val = va_arg(ap, int); \
385 break; \
386 case '?' | 0x80: \
387 val = va_arg(ap, unsigned int); \
388 break; \
389 case 'l': \
390 val = va_arg(ap, long); \
391 break; \
392 case 'l' | 0x80: \
393 val = va_arg(ap, unsigned long); \
394 break; \
395 case 'q': \
396 val = va_arg(ap, long long); \
397 break; \
398 case 'q' | 0x80: \
399 val = va_arg(ap, unsigned long long); \
400 break; \
401 case 'j': \
402 val = va_arg(ap, intmax_t); \
403 break; \
404 case 'j' | 0x80: \
405 val = va_arg(ap, uintmax_t); \
406 break; \
407 case 't': \
408 val = va_arg(ap, ptrdiff_t); \
409 break; \
410 case 'z': \
411 val = va_arg(ap, ssize_t); \
412 break; \
413 case 'z' | 0x80: \
414 val = va_arg(ap, size_t); \
415 break; \
416 case 'p': /* Synthetic; used for %p. */ \
417 val = va_arg(ap, uintptr_t); \
418 break; \
419 default: \
420 not_reached(); \
421 val = 0; \
422 } \
423 } while (0)
424
425 i = 0;
426 f = format;
427 while (true) {
428 switch (*f) {
429 case '\0': goto label_out;
430 case '%': {
431 bool alt_form = false;
432 bool left_justify = false;
433 bool plus_space = false;
434 bool plus_plus = false;
435 int prec = -1;
436 int width = -1;
437 unsigned char len = '?';
438 char *s;
439 size_t slen;
440 bool first_width_digit = true;
441 bool pad_zero = false;
442
443 f++;
444 /* Flags. */
445 while (true) {
446 switch (*f) {
447 case '#':
448 assert(!alt_form);
449 alt_form = true;
450 break;
451 case '-':
452 assert(!left_justify);
453 left_justify = true;
454 break;
455 case ' ':
456 assert(!plus_space);
457 plus_space = true;
458 break;
459 case '+':
460 assert(!plus_plus);
461 plus_plus = true;
462 break;
463 default: goto label_width;
464 }
465 f++;
466 }
467 /* Width. */
468 label_width:
469 switch (*f) {
470 case '*':
471 width = va_arg(ap, int);
472 f++;
473 if (width < 0) {
474 left_justify = true;
475 width = -width;
476 }
477 break;
478 case '0':
479 if (first_width_digit) {
480 pad_zero = true;
481 }
482 JEMALLOC_FALLTHROUGH;
483 case '1': case '2': case '3': case '4':
484 case '5': case '6': case '7': case '8': case '9': {
485 uintmax_t uwidth;
486 set_errno(0);
487 uwidth = malloc_strtoumax(f, (char **)&f, 10);
488 assert(uwidth != UINTMAX_MAX || get_errno() !=
489 ERANGE);
490 width = (int)uwidth;
491 first_width_digit = false;
492 break;
493 } default:
494 break;
495 }
496 /* Width/precision separator. */
497 if (*f == '.') {
498 f++;
499 } else {
500 goto label_length;
501 }
502 /* Precision. */
503 switch (*f) {
504 case '*':
505 prec = va_arg(ap, int);
506 f++;
507 break;
508 case '0': case '1': case '2': case '3': case '4':
509 case '5': case '6': case '7': case '8': case '9': {
510 uintmax_t uprec;
511 set_errno(0);
512 uprec = malloc_strtoumax(f, (char **)&f, 10);
513 assert(uprec != UINTMAX_MAX || get_errno() !=
514 ERANGE);
515 prec = (int)uprec;
516 break;
517 }
518 default: break;
519 }
520 /* Length. */
521 label_length:
522 switch (*f) {
523 case 'l':
524 f++;
525 if (*f == 'l') {
526 len = 'q';
527 f++;
528 } else {
529 len = 'l';
530 }
531 break;
532 case 'q': case 'j': case 't': case 'z':
533 len = *f;
534 f++;
535 break;
536 default: break;
537 }
538 /* Conversion specifier. */
539 switch (*f) {
540 case '%':
541 /* %% */
542 APPEND_C(*f);
543 f++;
544 break;
545 case 'd': case 'i': {
546 intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
547 char buf[D2S_BUFSIZE];
548
549 /*
550 * Outputting negative, zero-padded numbers
551 * would require a nontrivial rework of the
552 * interaction between the width and padding
553 * (since 0 padding goes between the '-' and the
554 * number, while ' ' padding goes either before
555 * the - or after the number. Since we
556 * currently don't ever need 0-padded negative
557 * numbers, just don't bother supporting it.
558 */
559 assert(!pad_zero);
560
561 GET_ARG_NUMERIC(val, len);
562 s = d2s(val, (plus_plus ? '+' : (plus_space ?
563 ' ' : '-')), buf, &slen);
564 APPEND_PADDED_S(s, slen, width, left_justify);
565 f++;
566 break;
567 } case 'o': {
568 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
569 char buf[O2S_BUFSIZE];
570
571 GET_ARG_NUMERIC(val, len | 0x80);
572 s = o2s(val, alt_form, buf, &slen);
573 APPEND_PADDED_S(s, slen, width, left_justify);
574 f++;
575 break;
576 } case 'u': {
577 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
578 char buf[U2S_BUFSIZE];
579
580 GET_ARG_NUMERIC(val, len | 0x80);
581 s = u2s(val, 10, false, buf, &slen);
582 APPEND_PADDED_S(s, slen, width, left_justify);
583 f++;
584 break;
585 } case 'x': case 'X': {
586 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
587 char buf[X2S_BUFSIZE];
588
589 GET_ARG_NUMERIC(val, len | 0x80);
590 s = x2s(val, alt_form, *f == 'X', buf, &slen);
591 APPEND_PADDED_S(s, slen, width, left_justify);
592 f++;
593 break;
594 } case 'c': {
595 unsigned char val;
596 char buf[2];
597
598 assert(len == '?' || len == 'l');
599 assert_not_implemented(len != 'l');
600 val = va_arg(ap, int);
601 buf[0] = val;
602 buf[1] = '\0';
603 APPEND_PADDED_S(buf, 1, width, left_justify);
604 f++;
605 break;
606 } case 's':
607 assert(len == '?' || len == 'l');
608 assert_not_implemented(len != 'l');
609 s = va_arg(ap, char *);
610 slen = (prec < 0) ? strlen(s) : (size_t)prec;
611 APPEND_PADDED_S(s, slen, width, left_justify);
612 f++;
613 break;
614 case 'p': {
615 uintmax_t val;
616 char buf[X2S_BUFSIZE];
617
618 GET_ARG_NUMERIC(val, 'p');
619 s = x2s(val, true, false, buf, &slen);
620 APPEND_PADDED_S(s, slen, width, left_justify);
621 f++;
622 break;
623 } default: not_reached();
624 }
625 break;
626 } default: {
627 APPEND_C(*f);
628 f++;
629 break;
630 }}
631 }
632 label_out:
633 if (i < size) {
634 str[i] = '\0';
635 } else {
636 str[size - 1] = '\0';
637 }
638
639 #undef APPEND_C
640 #undef APPEND_S
641 #undef APPEND_PADDED_S
642 #undef GET_ARG_NUMERIC
643 return i;
644 }
645
646 JEMALLOC_FORMAT_PRINTF(3, 4)
647 size_t
malloc_snprintf(char * str,size_t size,const char * format,...)648 malloc_snprintf(char *str, size_t size, const char *format, ...) {
649 size_t ret;
650 va_list ap;
651
652 va_start(ap, format);
653 ret = malloc_vsnprintf(str, size, format, ap);
654 va_end(ap);
655
656 return ret;
657 }
658
659 void
malloc_vcprintf(write_cb_t * write_cb,void * cbopaque,const char * format,va_list ap)660 malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format,
661 va_list ap) {
662 char buf[MALLOC_PRINTF_BUFSIZE];
663
664 if (write_cb == NULL) {
665 /*
666 * The caller did not provide an alternate write_cb callback
667 * function, so use the default one. malloc_write() is an
668 * inline function, so use malloc_message() directly here.
669 */
670 write_cb = (je_malloc_message != NULL) ? je_malloc_message :
671 wrtmessage;
672 }
673
674 malloc_vsnprintf(buf, sizeof(buf), format, ap);
675 write_cb(cbopaque, buf);
676 }
677
678 /*
679 * Print to a callback function in such a way as to (hopefully) avoid memory
680 * allocation.
681 */
682 JEMALLOC_FORMAT_PRINTF(3, 4)
683 void
malloc_cprintf(write_cb_t * write_cb,void * cbopaque,const char * format,...)684 malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) {
685 va_list ap;
686
687 va_start(ap, format);
688 malloc_vcprintf(write_cb, cbopaque, format, ap);
689 va_end(ap);
690 }
691
692 /* Print to stderr in such a way as to avoid memory allocation. */
693 JEMALLOC_FORMAT_PRINTF(1, 2)
694 void
malloc_printf(const char * format,...)695 malloc_printf(const char *format, ...) {
696 va_list ap;
697
698 va_start(ap, format);
699 malloc_vcprintf(NULL, NULL, format, ap);
700 va_end(ap);
701 }
702
703 /*
704 * Restore normal assertion macros, in order to make it possible to compile all
705 * C files as a single concatenation.
706 */
707 #undef assert
708 #undef not_reached
709 #undef not_implemented
710 #undef assert_not_implemented
711 #include "jemalloc/internal/assert.h"
712