From 43a11a7ed10a36eac6ab45a877fda9770de3ed40 Mon Sep 17 00:00:00 2001 From: Kirill Smelkov Date: Sat, 1 Dec 2012 01:00:23 +0400 Subject: [PATCH 1/4] Make tcc work after self-compiling with bounds-check enabled For vstack Fabrice used the trick to initialize vtop to &vstack[-1], so that on first push, vtop becomes &vstack[0] and a value is also stored there - everything works. Except that when tcc is compiled with bounds-checking enabled, vstack - 1 returns INVALID_POINTER and oops... Let's workaround it with artificial 1 vstack slot which will not be used, but only serve as an indicator that pointing to &vstack[-1] is ok. Now, tcc, after being self-compiled with -b works: $ ./tcc -B. -o tccb -DONE_SOURCE -DCONFIG_MULTIARCHDIR=\"i386-linux-gnu\" tcc.c -ldl $ cd tests $ ../tcc -B.. -run tcctest.c >1 $ ../tccb -B.. -run tcctest.c >2 $ diff -u 1 2 and note, tcc's compilation speed is not affected: $ ./tcc -B. -bench -DONE_SOURCE -DCONFIG_MULTIARCHDIR=\"i386-linux-gnu\" -c tcc.c before: 8270 idents, 47221 lines, 1527730 bytes, 0.152 s, 309800 lines/s, 10.0 MB/s after: 8271 idents, 47221 lines, 1527733 bytes, 0.152 s, 310107 lines/s, 10.0 MB/s But note, that `tcc -b -run tcc` is still broken - for example it crashes on $ cat x.c double get100 () { return 100.0; } $ ./tcc -B. -b -DTCC_TARGET_I386 -DCONFIG_MULTIARCHDIR=\"i386-linux-gnu\" -run \ -DONE_SOURCE ./tcc.c -B. -c x.c Runtime error: dereferencing invalid pointer ./tccpp.c:1953: at 0xa7beebdf parse_number() (included from ./libtcc.c, ./tcc.c) ./tccpp.c:3003: by 0xa7bf0708 next() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:4465: by 0xa7bfe348 block() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:4440: by 0xa7bfe212 block() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:5529: by 0xa7c01929 gen_function() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:5767: by 0xa7c02602 decl0() (included from ./libtcc.c, ./tcc.c) that's because lib/bcheck.c runtime needs more fixes -- see next patches. --- tcc.h | 3 ++- tccgen.c | 2 +- tccpp.c | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tcc.h b/tcc.h index 9e7c574..859c85f 100644 --- a/tcc.h +++ b/tcc.h @@ -1135,7 +1135,8 @@ ST_DATA Sym *local_label_stack; ST_DATA Sym *global_label_stack; ST_DATA Sym *define_stack; ST_DATA CType char_pointer_type, func_old_type, int_type, size_type; -ST_DATA SValue vstack[VSTACK_SIZE], *vtop; +ST_DATA SValue __vstack[1+/*to make bcheck happy*/ VSTACK_SIZE], *vtop; +#define vstack (__vstack + 1) ST_DATA int rsym, anon_sym, ind, loc; ST_DATA int const_wanted; /* true if constant wanted */ diff --git a/tccgen.c b/tccgen.c index f79da36..b9142d2 100644 --- a/tccgen.c +++ b/tccgen.c @@ -55,7 +55,7 @@ ST_DATA Sym *define_stack; ST_DATA Sym *global_label_stack; ST_DATA Sym *local_label_stack; -ST_DATA SValue vstack[VSTACK_SIZE], *vtop; +ST_DATA SValue __vstack[1+VSTACK_SIZE], *vtop; ST_DATA int const_wanted; /* true if constant wanted */ ST_DATA int nocode_wanted; /* true if no code generation wanted for an expression */ diff --git a/tccpp.c b/tccpp.c index aff5a53..b80c986 100644 --- a/tccpp.c +++ b/tccpp.c @@ -3041,7 +3041,6 @@ ST_FUNC void preprocess_init(TCCState *s1) s1->ifdef_stack_ptr = s1->ifdef_stack; file->ifdef_stack_ptr = s1->ifdef_stack_ptr; - /* XXX: not ANSI compliant: bound checking says error */ vtop = vstack - 1; s1->pack_stack[0] = 0; s1->pack_stack_ptr = s1->pack_stack; From efd9d92b7c5d88387709a891c54848f266f7fada Mon Sep 17 00:00:00 2001 From: Kirill Smelkov Date: Sun, 9 Dec 2012 18:48:48 +0400 Subject: [PATCH 2/4] lib/bcheck: Don't assume heap goes right after bss At startup __bound_init() wants to mark malloc zone as invalid memory, so that any access to memory on heap, not allocated through malloc be invalid. Other pages are initialized as empty regions, access to which is not treated as invalid by bounds-checking. The problem is code incorrectly assumed that heap goes right after bss, and that is not correct for two cases: 1) if we are running from `tcc -b -run`, program text data and bss will be already in malloced memory, possibly in mmaped region insead of heap, and marking memory as invalid from _end will not cover heap and probably wrongly mark correct regions. 2) if address space randomization is turned on, again heap does not start from _end, and we'll mark as invalid something else instead of malloc area. For example with the following diagnostic patch ... diff --git a/tcc.c b/tcc.c index 5dd5725..31c46e8 100644 --- a/tcc.c +++ b/tcc.c @@ -479,6 +479,8 @@ static int parse_args(TCCState *s, int argc, char **argv) return optind; } +extern int _etext, _edata, _end; + int main(int argc, char **argv) { int i; @@ -487,6 +489,18 @@ int main(int argc, char **argv) int64_t start_time = 0; const char *default_file = NULL; + void *brk; + + brk = sbrk(0); + + fprintf(stderr, "\n>>> TCC\n\n"); + fprintf(stderr, "etext:\t%10p\n", &_etext); + fprintf(stderr, "edata:\t%10p\n", &_edata); + fprintf(stderr, "end:\t%10p\n", &_end); + fprintf(stderr, "brk:\t%10p\n", brk); + fprintf(stderr, "stack:\t%10p\n", &brk); + + fprintf(stderr, "&errno: %p\n", &errno); s = tcc_new(); output_type = TCC_OUTPUT_EXE; diff --git a/tccrun.c b/tccrun.c index 531f46a..25ed30a 100644 --- a/tccrun.c +++ b/tccrun.c @@ -91,6 +91,8 @@ LIBTCCAPI int tcc_run(TCCState *s1, int argc, char **argv) int (*prog_main)(int, char **); int ret; + fprintf(stderr, "\n\ntcc_run() ...\n\n"); + if (tcc_relocate(s1, TCC_RELOCATE_AUTO) < 0) return -1; diff --git a/lib/bcheck.c b/lib/bcheck.c index ea5b233..8b26a5f 100644 --- a/lib/bcheck.c +++ b/lib/bcheck.c @@ -296,6 +326,8 @@ static void mark_invalid(unsigned long addr, unsigned long size) start = addr; end = addr + size; + fprintf(stderr, "mark_invalid %10p - %10p\n", (void *)addr, (void *)end); + t2_start = (start + BOUND_T3_SIZE - 1) >> BOUND_T3_BITS; if (end != 0) t2_end = end >> BOUND_T3_BITS; ... Look how memory is laid out for `tcc -b -run ...`: $ ./tcc -B. -b -DTCC_TARGET_I386 -DCONFIG_MULTIARCHDIR=\"i386-linux-gnu\" -run \ -DONE_SOURCE ./tcc.c -B. -c x.c >>> TCC etext: 0x8065477 edata: 0x8070220 end: 0x807a95c brk: 0x807b000 stack: 0xaffff0f0 &errno: 0xa7e25688 tcc_run() ... mark_invalid 0xfff80000 - (nil) mark_invalid 0xa7c31d98 - 0xafc31d98 >>> TCC etext: 0xa7c22767 edata: 0xa7c2759c end: 0xa7c31d98 brk: 0x8211000 stack: 0xafffeff0 &errno: 0xa7e25688 Runtime error: dereferencing invalid pointer ./tccpp.c:1953: at 0xa7beebdf parse_number() (included from ./libtcc.c, ./tcc.c) ./tccpp.c:3003: by 0xa7bf0708 next() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:4465: by 0xa7bfe348 block() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:4440: by 0xa7bfe212 block() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:5529: by 0xa7c01929 gen_function() (included from ./libtcc.c, ./tcc.c) ./tccgen.c:5767: by 0xa7c02602 decl0() (included from ./libtcc.c, ./tcc.c) The second mark_invalid goes right after in-memory-compiled program's _end, and oops, that's not where malloc zone is (starts from brk), and oops again, mark_invalid covers e.g. errno. Then compiled tcc is crasshing by bcheck on errno access: 1776 static void parse_number(const char *p) 1777 { 1778 int b, t, shift, frac_bits, s, exp_val, ch; ... 1951 *q = '\0'; 1952 t = toup(ch); 1953 errno = 0; The solution here is to use sbrk(0) as approximation for the program break start instead of &_end: - if we are a separately compiled program, __bound_init() runs early, and sbrk(0) should be equal or very near to start_brk (in case other constructors malloc something), or - if we are running from under `tcc -b -run`, sbrk(0) will return start of heap portion which is under this program control, and not mark as invalid earlier allocated memory. With this patch `tcc -b -run tcc.c ...` succeeds compiling above small-test program (diagnostic patch is still applied too): $ ./tcc -B. -b -DTCC_TARGET_I386 -DCONFIG_MULTIARCHDIR=\"i386-linux-gnu\" -run \ -DONE_SOURCE ./tcc.c -B. -c x.c >>> TCC etext: 0x8065477 edata: 0x8070220 end: 0x807a95c brk: 0x807b000 stack: 0xaffff0f0 &errno: 0xa7e25688 tcc_run() ... mark_invalid 0xfff80000 - (nil) mark_invalid 0x8211000 - 0x10211000 >>> TCC etext: 0xa7c22777 edata: 0xa7c275ac end: 0xa7c31da8 brk: 0x8211000 stack: 0xafffeff0 &errno: 0xa7e25688 (completes ok) but running `tcc -b -run tcc.c -run tests/tcctest.c` sigsegv's - that's the plot for the next patch. --- lib/bcheck.c | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/lib/bcheck.c b/lib/bcheck.c index 5ea2d0b..ea5b233 100644 --- a/lib/bcheck.c +++ b/lib/bcheck.c @@ -25,6 +25,7 @@ && !defined(__DragonFly__) && !defined(__OpenBSD__) #include #endif +#include //#define BOUND_DEBUG @@ -94,9 +95,6 @@ static void *saved_realloc_hook; static void *saved_memalign_hook; #endif -/* linker definitions */ -extern char _end; - /* TCC definitions */ extern char __bounds_start; /* start of static bounds table */ /* error message, just for TCC */ @@ -379,9 +377,32 @@ void __bound_init(void) #if !defined(__TINYC__) && defined(CONFIG_TCC_MALLOC_HOOKS) /* malloc zone is also marked invalid. can only use that with - hooks because all libs should use the same malloc. The solution - would be to build a new malloc for tcc. */ - start = (unsigned long)&_end; + * hooks because all libs should use the same malloc. The solution + * would be to build a new malloc for tcc. + * + * usually heap (= malloc zone) comes right after bss, i.e. after _end, but + * not always - either if we are running from under `tcc -b -run`, or if + * address space randomization is turned on(a), heap start will be separated + * from bss end. + * + * So sbrk(0) will be a good approximation for start_brk: + * + * - if we are a separately compiled program, __bound_init() runs early, + * and sbrk(0) should be equal or very near to start_brk(b) (in case other + * constructors malloc something), or + * + * - if we are running from under `tcc -b -run`, sbrk(0) will return + * start of heap portion which is under this program control, and not + * mark as invalid earlier allocated memory. + * + * + * (a) /proc/sys/kernel/randomize_va_space = 2, on Linux; + * usually turned on by default. + * + * (b) on Linux >= v3.3, the alternative is to read + * start_brk from /proc/self/stat + */ + start = (unsigned long)sbrk(0); size = 128 * 0x100000; mark_invalid(start, size); #endif From dbeb4faf21597914534c791f2ca63d3ed7427810 Mon Sep 17 00:00:00 2001 From: Kirill Smelkov Date: Sun, 9 Dec 2012 19:30:28 +0400 Subject: [PATCH 3/4] lib/bcheck: Fix code typo in __bound_delete_region() We were calling get_page() with t2 index which is not correct, since get_page() operate on t1 indices. The bug is here from day-1, from 60f781c4 (first version of bounds checker) and show as a crash in __bound_delete_region() at program exit: $ ./tcc -B. -DTCC_TARGET_I386 -DCONFIG_MULTIARCHDIR=\"i386-linux-gnu\" -b -run -DONE_SOURCE \ ./tcc.c -B. -DTCC_TARGET_I386 -DCONFIG_MULTIARCHDIR=\"i386-linux-gnu\" -run -DONE_SOURCE \ ./tcc.c -B. -run tests/tcctest.c (lot's of correct output from tcctest) Runtime error: dereferencing invalid pointer at 0xa7c21cc4 __bound_delete_region() by (nil) ??? Segmentation fault The fix is simple - last page should be get through t1_end, like it is done in __bound_new_region(). After this patch, tcc is being able to compile itself with -b, then compile itself again and run tcctest with correct output. Tests follow. --- lib/bcheck.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bcheck.c b/lib/bcheck.c index ea5b233..00b90fe 100644 --- a/lib/bcheck.c +++ b/lib/bcheck.c @@ -613,7 +613,7 @@ int __bound_delete_region(void *p) } } /* last page */ - page = get_page(t2_end); + page = get_page(t1_end); e2 = (BoundEntry *)((char *)page + t2_end); for(e=page;estart = 0; From 031ff872be7be76c2c088981a827cbb2a29dfc13 Mon Sep 17 00:00:00 2001 From: Kirill Smelkov Date: Sun, 9 Dec 2012 19:43:40 +0400 Subject: [PATCH 4/4] tests: Add tests for compile/run tcc.c with `tcc -b` then compile tcc.c again, then run tcctest.c Just like with test[123] add their test[123]b variants. After previous 3 patchs all test pass here on Debian GNU/Linux on i385 with gcc-4.7 with or without memory randomization turned on. --- .gitignore | 3 +++ tests/Makefile | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ae4965c..376ea52 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,11 @@ tc1.c error.c i386-gen1.c test.out1 +test.out1b test.out2 +test.out2b test.out3 +test.out3b web.sh memdebug.c bench diff --git a/tests/Makefile b/tests/Makefile index da7c3f9..116178f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -12,6 +12,9 @@ TESTS = libtest \ test3 \ speedtest \ btest \ + test1b\ + test2b\ + test3b\ weaktest # test4 # this test does not seem to work on any platform @@ -20,6 +23,9 @@ TESTS = libtest \ # bounds-checking is supported only on i386 ifneq ($(ARCH),i386) TESTS := $(filter-out btest,$(TESTS)) + TESTS := $(filter-out test1b,$(TESTS)) + TESTS := $(filter-out test2b,$(TESTS)) + TESTS := $(filter-out test3b,$(TESTS)) endif # these should work too @@ -87,18 +93,33 @@ test1: test.ref $(TCC) -run tcctest.c > test.out1 @if diff -u test.ref test.out1 ; then echo "Auto Test OK"; fi +test1b: test.ref + @echo ------------ $@ ------------ + $(TCC) -b -run tcctest.c > test.out1b + @if diff -u test.ref test.out1b ; then echo "Auto Test OK"; fi + # iterated test2 (compile tcc then compile tcctest.c !) test2: test.ref @echo ------------ $@ ------------ $(TCC) $(RUN_TCC) $(RUN_TCC) -run tcctest.c > test.out2 @if diff -u test.ref test.out2 ; then echo "Auto Test2 OK"; fi +test2b: test.ref ../bcheck.o + @echo ------------ $@ ------------ + $(TCC) -b $(RUN_TCC) $(RUN_TCC) -run tcctest.c > test.out2b + @if diff -u test.ref test.out2b ; then echo "Auto Test2b OK"; fi + # iterated test3 (compile tcc then compile tcc then compile tcctest.c !) test3: test.ref @echo ------------ $@ ------------ $(TCC) $(RUN_TCC) $(RUN_TCC) $(RUN_TCC) -run tcctest.c > test.out3 @if diff -u test.ref test.out3 ; then echo "Auto Test3 OK"; fi +test3b: test.ref + @echo ------------ $@ ------------ + $(TCC) -b $(RUN_TCC) $(RUN_TCC) $(RUN_TCC) -run tcctest.c > test.out3b + @if diff -u test.ref test.out3b ; then echo "Auto Test3 OK"; fi + # binary output test test4: test.ref @echo ------------ $@ ------------ @@ -190,5 +211,5 @@ cache: tcc_g # clean clean: - rm -vf *~ *.o *.a *.bin *.i *.ref *.out *.out? *.gcc \ + rm -vf *~ *.o *.a *.bin *.i *.ref *.out *.out? *.out?b *.gcc \ tcctest[1234] ex? libtcc_test$(EXESUF) tcc_g tcclib.h