diff --git a/.github/workflows/ci-drk.yml b/.github/workflows/ci-drk.yml index 15aa21da8..78f97901d 100644 --- a/.github/workflows/ci-drk.yml +++ b/.github/workflows/ci-drk.yml @@ -17,6 +17,13 @@ jobs: sudo apt install -y doxygen vera++ libelf-dev scons cmake g++-multilib echo "MAKE_CMD=make -j$(nproc)" >> $GITHUB_ENV + - name: Generate DynamoRIO headers via CMake + run: | + mkdir -p build + cd build + cmake .. + $MAKE_CMD + - name: Download and extract kernel source run: | VERSION="6.6.130" @@ -30,12 +37,6 @@ jobs: make defconfig $MAKE_CMD modules_prepare - - name: Generate DynamoRIO headers via CMake - run: | - mkdir -p build - cd build - cmake .. - - name: Build drk # TODO i#10: Convert to cmake. run: cd core && $MAKE_CMD -f drk.mk KERNELDIR=$GITHUB_WORKSPACE/$KERNEL_DIR diff --git a/core/annotations.c b/core/annotations.c index 516355192..32608c199 100644 --- a/core/annotations.c +++ b/core/annotations.c @@ -258,7 +258,7 @@ free_annotation_handler(void *p); */ void -annotation_init() +annotation_init(void) { handlers = strhash_hash_create( GLOBAL_DCONTEXT, 8, 80, /* favor a small table */ @@ -302,7 +302,7 @@ annotation_init() } void -annotation_exit() +annotation_exit(void) { # if !(defined(WINDOWS) && defined(X64)) uint i; diff --git a/core/annotations.h b/core/annotations.h index e4e927eea..680306072 100644 --- a/core/annotations.h +++ b/core/annotations.h @@ -124,10 +124,10 @@ typedef struct _dr_annotation_handler_t { } dr_annotation_handler_t; void -annotation_init(); +annotation_init(void); void -annotation_exit(); +annotation_exit(void); static inline bool is_annotation_label(instr_t *instr) diff --git a/core/arch/arch.c b/core/arch/arch.c index 55a5f68d9..acd971656 100644 --- a/core/arch/arch.c +++ b/core/arch/arch.c @@ -327,6 +327,7 @@ shared_gencode_emit(generated_code_t *gencode _IF_X86_64(bool x86_mode)) pc = check_size_and_cache_line(isa_mode, gencode, pc); gencode->fcache_enter = pc; pc = emit_fcache_enter_shared(GLOBAL_DCONTEXT, gencode, pc); + gencode->fcache_enter_end = pc; pc = check_size_and_cache_line(isa_mode, gencode, pc); gencode->fcache_return = pc; pc = emit_fcache_return_shared(GLOBAL_DCONTEXT, gencode, pc); @@ -989,15 +990,40 @@ emit_ibl_routine_and_template(dcontext_t *dcontext, generated_code_t *code, byte ibl_code->thread_shared_routine = thread_shared; ibl_code->branch_type = branch_type; ibl_code->source_fragment_type = source_type; +#ifdef LINUX_KERNEL + ibl_code->num_ibl_found_exits = 0; +#endif pc = emit_indirect_branch_lookup(dcontext, code, pc, fcache_return_pc, target_trace_table, inline_ibl_head, ibl_code); +#ifdef LINUX_KERNEL + ibl_code->indirect_branch_lookup_routine_end = pc; +#endif if (inline_ibl_head) { /* create the inlined ibl template */ pc = check_size_and_cache_line(isa_mode, code, pc); pc = emit_inline_ibl_stub(dcontext, pc, ibl_code, target_trace_table); } +#ifdef LINUX_KERNEL + pc = check_size_and_cache_line(isa_mode, code, pc); + ibl_code->found_unlinked = pc; + pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, false, + false); + pc = check_size_and_cache_line(isa_mode, code, pc); + ibl_code->found_unlinked_prefix = pc; + pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, false, + true); + pc = check_size_and_cache_line(isa_mode, code, pc); + ibl_code->found_unlinked_eflags = pc; + pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, true, + false); + pc = check_size_and_cache_line(isa_mode, code, pc); + ibl_code->found_unlinked_eflags_prefix = pc; + pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, true, + true); +#endif + ibl_code->far_ibl = pc; pc = emit_far_ibl( dcontext, pc, ibl_code, @@ -1148,6 +1174,25 @@ emit_syscall_routines(dcontext_t *dcontext, generated_code_t *code, byte *pc, code->do_syscall = pc; pc = emit_do_syscall(dcontext, code, pc, code->fcache_return, thread_shared, 0, &code->do_syscall_offs); +#elif defined(LINUX_KERNEL) + ASSERT(thread_shared); + pc = check_size_and_cache_line(isa_mode, code, pc); + code->syscall_entry = pc; + pc = emit_syscall_entry(dcontext, code->fcache_return, + os_get_native_syscall_entry(dcontext), pc); + pc = check_size_and_cache_line(isa_mode, code, pc); + code->common_vector_entry = pc; + pc = emit_common_vector_entry(dcontext, os_get_tls_base(dcontext), + os_get_interrupt_handler(0), pc); + pc = check_size_and_cache_line(isa_mode, code, pc); + for (int vector = VECTOR_START; vector < VECTOR_END; vector++) { + /* Pack as many as we can on a cache line. */ + if (CROSSES_ALIGNMENT(pc, VECTOR_ENTRY_CODE_SIZE, proc_get_cache_line_size())) { + pc = check_size_and_cache_line(isa_mode, code, pc); + } + code->vector_entry[vector - VECTOR_START] = pc; + pc = emit_vector_entry(dcontext, code->common_vector_entry, vector, pc); + } #else /* UNIX */ pc = check_size_and_cache_line(isa_mode, code, pc); code->do_syscall = pc; @@ -1180,6 +1225,19 @@ emit_syscall_routines(dcontext_t *dcontext, generated_code_t *code, byte *pc, return pc; } +#ifdef LINUX_KERNEL +void +optimize_syscall_code(dcontext_t *dcontext, fragment_t *f) +{ + generated_code_t *code = THREAD_GENCODE(dcontext); + byte *pc = code->syscall_entry; + /* Assert pc points to swapgs. */ + ASSERT(pc[0] == 0x0f && pc[1] == 0x01 && pc[2] == 0xf8); + pc += 3; + insert_relative_jump(pc, f->start_pc, false); +} +#endif + void arch_thread_init(dcontext_t *dcontext) { @@ -1285,6 +1343,7 @@ arch_thread_init(dcontext_t *dcontext) pc = check_size_and_cache_line(isa_mode, code, pc); code->fcache_enter = pc; pc = emit_fcache_enter(dcontext, code, pc); + code->fcache_enter_end = pc; pc = check_size_and_cache_line(isa_mode, code, pc); code->fcache_return = pc; pc = emit_fcache_return(dcontext, code, pc); @@ -1783,6 +1842,32 @@ in_generated_routine(dcontext_t *dcontext, cache_pc pc) /* XXX: what about inlined IBL stubs */ } +static bool +in_fcache_enter_for_gencode(generated_code_t *code, cache_pc pc) +{ + return pc != NULL && pc >= code->fcache_enter && pc < code->fcache_enter_end; +} + +bool +in_fcache_enter(dcontext_t *dcontext, cache_pc pc) +{ + generated_code_t *code = THREAD_GENCODE(dcontext); + if (in_fcache_enter_for_gencode(code, pc)) + return true; + if (USE_SHARED_GENCODE()) { + if (in_fcache_enter_for_gencode(shared_code, pc)) + return true; +#if defined(X86) && defined(X64) + if (shared_code_x86 != NULL && in_fcache_enter_for_gencode(shared_code_x86, pc)) + return true; + if (shared_code_x86_to_x64 != NULL && + in_fcache_enter_for_gencode(shared_code_x86_to_x64, pc)) + return true; +#endif + } + return false; +} + static bool in_fcache_return_for_gencode(generated_code_t *code, cache_pc pc) { @@ -1978,6 +2063,52 @@ get_do_vmkuw_syscall_entry(dcontext_t *dcontext) # endif #endif +#ifdef LINUX_KERNEL +cache_pc +get_syscall_entry(dcontext_t *dcontext) +{ + generated_code_t *code = THREAD_GENCODE(dcontext); + return (cache_pc)code->syscall_entry; +} + +cache_pc +get_vector_entry(dcontext_t *dcontext, interrupt_vector_t vector) +{ + generated_code_t *code = THREAD_GENCODE(dcontext); + ASSERT(vector >= VECTOR_START && vector < VECTOR_END); + return (cache_pc)code->vector_entry[vector - VECTOR_START]; +} + +bool +vector_has_error_code(interrupt_vector_t vector) +{ + ASSERT(vector >= VECTOR_START && vector < VECTOR_END); + switch (vector) { + case VECTOR_DOUBLE_FAULT: + case VECTOR_INVALID_TSS: + case VECTOR_SEGMENT_NOT_PRESENT: + case VECTOR_STACK_FAULT: + case VECTOR_GENERAL_PROTECTION: + case VECTOR_PAGE_FAULT: + case VECTOR_ALIGNMENT_CHECK: return true; + default: return false; + } +} + +bool +vector_is_synchronous(interrupt_vector_t vector) +{ + ASSERT(vector >= VECTOR_START && vector < VECTOR_END); + return vector >= VECTOR_EXCEPTION_START && vector < VECTOR_EXCEPTION_END; +} + +bool +was_kernel_interrupted(interrupt_stack_frame_t *frame) +{ + return (frame->cs & 3) == 0; +} +#endif + cache_pc fcache_return_routine(dcontext_t *dcontext) { @@ -2540,6 +2671,39 @@ get_ibl_routine_code(dcontext_t *dcontext, ibl_branch_type_t branch_type, : GENCODE_FROM_DCONTEXT)); } +#ifdef LINUX_KERNEL +ibl_code_t * +get_ibl_code_from_routine_pc(dcontext_t *dcontext, cache_pc pc) +{ + ibl_source_fragment_type_t source_fragment_type; + ibl_branch_type_t branch_type; +# if defined(X86) && defined(X64) + gencode_mode_t mode; +# endif + + for (source_fragment_type = IBL_SOURCE_TYPE_START; + source_fragment_type < IBL_SOURCE_TYPE_END; source_fragment_type++) { + for (branch_type = IBL_BRANCH_TYPE_START; branch_type < IBL_BRANCH_TYPE_END; + branch_type++) { +# if defined(X86) && defined(X64) + for (mode = GENCODE_X64; mode <= GENCODE_X86_TO_X64; mode++) { +# endif + ibl_code_t *code = get_ibl_routine_code_internal( + dcontext, source_fragment_type, branch_type _IF_X86_64(mode)); + if (code != NULL && code->initialized && + pc >= code->indirect_branch_lookup_routine && + pc < code->indirect_branch_lookup_routine_end) { + return code; + } +# if defined(X86) && defined(X64) + } +# endif + } + } + return NULL; +} +#endif + #ifdef WINDOWS /* XXX We support a private and shared fragments simultaneously targeting * shared syscall -- -shared_fragment_shared_syscalls must be on and both @@ -2818,6 +2982,10 @@ get_fcache_target(dcontext_t *dcontext) void set_fcache_target(dcontext_t *dcontext, cache_pc value) { +#ifdef LINUX_KERNEL + /* Save the last app tag. We need this to handle interrupts in fcache_enter. */ + dcontext->next_app_tag = dcontext->next_tag; +#endif /* we used to use mcontext.pc, but that's in the writable * portion of the dcontext, and so for self-protection we use the * next_tag slot, which is protected @@ -2881,7 +3049,7 @@ get_global_do_syscall_entry() /* used only by cleanup_and_terminate to avoid the sysenter * sygate hack version */ byte * -get_cleanup_and_terminate_global_do_syscall_entry() +get_cleanup_and_terminate_global_do_syscall_entry(void) { /* see note above: for 32-bit linux apps we use int. * xref PR 332427 as well where sysenter causes a crash @@ -4026,17 +4194,17 @@ unit_test_atomic_ops(void) ATOMIC_8BYTE_WRITE(&count3, value64, false); EXPECT(count3, -1); # endif - EXPECT(atomic_inc_and_test(&count1), true); /* result is 0 */ - EXPECT(atomic_inc_and_test(&count1), false); /* result is 1 */ - EXPECT(atomic_dec_and_test(&count1), false); /* init value is 1, result is 0 */ - EXPECT(atomic_dec_and_test(&count1), true); /* init value is 0, result is -1 */ - EXPECT(atomic_dec_becomes_zero(&count1), false); /* result is -2 */ + EXPECT(d_r_atomic_inc_and_test(&count1), true); /* result is 0 */ + EXPECT(d_r_atomic_inc_and_test(&count1), false); /* result is 1 */ + EXPECT(d_r_atomic_dec_and_test(&count1), false); /* init value is 1, result is 0 */ + EXPECT(d_r_atomic_dec_and_test(&count1), true); /* init value is 0, result is -1 */ + EXPECT(d_r_atomic_dec_becomes_zero(&count1), false); /* result is -2 */ EXPECT(atomic_compare_exchange_int(&count1, -3, 1), false); /* no exchange */ EXPECT(count1, -2); EXPECT(atomic_compare_exchange_int(&count1, -2, 1), true); /* exchange */ - EXPECT(atomic_dec_becomes_zero(&count1), true); /* result is 0 */ - EXPECT(atomic_dec_and_test(&count1), true); /* init value is 0, result is -1 */ - EXPECT(atomic_dec_and_test(&count1), true); /* init value is -1, result is -2 */ + EXPECT(d_r_atomic_dec_becomes_zero(&count1), true); /* result is 0 */ + EXPECT(d_r_atomic_dec_and_test(&count1), true); /* init value is 0, result is -1 */ + EXPECT(d_r_atomic_dec_and_test(&count1), true); /* init value is -1, result is -2 */ ATOMIC_4BYTE_WRITE(&count1, 0, false); do_parallel_updates(); EXPECT(count1, MAX_NUM_THREADS); diff --git a/core/arch/arch.h b/core/arch/arch.h index 4e170d97c..20417ea5b 100644 --- a/core/arch/arch.h +++ b/core/arch/arch.h @@ -44,11 +44,11 @@ #ifndef ARCH_H #define ARCH_H -#include /* for offsetof */ -#include "instr.h" /* for reg_id_t */ -#include "decode.h" /* for X64_CACHE_MODE_DC */ -#include "arch_exports.h" /* for FRAG_IS_32 and FRAG_IS_X86_TO_X64 */ -#include "../fragment.h" /* IS_IBL_TARGET */ +#include "../stddef_wrapper.h" /* for offsetof */ +#include "instr.h" /* for reg_id_t */ +#include "decode.h" /* for X64_CACHE_MODE_DC */ +#include "arch_exports.h" /* for FRAG_IS_32 and FRAG_IS_X86_TO_X64 */ +#include "../fragment.h" /* IS_IBL_TARGET */ #include "ir_utils.h" #if defined(X86) && defined(X64) @@ -293,7 +293,7 @@ extern bool d_r_client_avx512_code_in_use; /* This routine determines whether zmm registers should be saved. */ static inline bool -d_r_is_avx512_code_in_use() +d_r_is_avx512_code_in_use(void) { return *d_r_avx512_code_in_use; } @@ -325,13 +325,13 @@ d_r_set_avx512_code_in_use(bool in_use, app_pc pc) } static inline bool -d_r_is_client_avx512_code_in_use() +d_r_is_client_avx512_code_in_use(void) { return d_r_client_avx512_code_in_use; } static inline void -d_r_set_client_avx512_code_in_use() +d_r_set_client_avx512_code_in_use(void) { SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); ATOMIC_1BYTE_WRITE(&d_r_client_avx512_code_in_use, (bool)true, false); @@ -648,7 +648,7 @@ mangle_insert_clone_code(dcontext_t *dcontext, instrlist_t *ilist, /* Returns the number of bytes the stack pointer has to be aligned to. */ static inline uint -get_ABI_stack_alignment() +get_ABI_stack_alignment(void) { return ABI_STACK_ALIGNMENT; } @@ -834,12 +834,31 @@ typedef struct patch_entry_t { negative offsets are from end of instruction */ } patch_entry_t; +#ifdef LINUX_KERNEL +# define MAX_IBL_FOUND_EXITS 2 +#else +# define MAX_IBL_FOUND_EXITS 0 +#endif + +#ifdef LINUX_KERNEL +typedef struct ibl_found_exit_t { + byte *jmp_pc; + int original_jmp_length; + byte original_jmp_code[MAX_INSTR_LENGTH]; + /* True iff this exit targets fragment prefixes. */ + bool targets_prefix; + /* True iff this exit restores eflags. */ + bool restored_eflags; +} ibl_found_exit_t; +#endif + enum { MAX_PATCH_ENTRIES = #ifdef HASHTABLE_STATISTICS 6 + /* will need more only for statistics */ #endif - 7, /* we use 5 normally, 7 w/ -atomic_inlined_linking and inlining */ + 7 + /* we use 5 normally, 7 w/ -atomic_inlined_linking and inlining */ + MAX_IBL_FOUND_EXITS, /* Patch entry flags */ /* Patch offset entries for dynamic updates from input variables */ /* use computed address if set, value at address otherwise */ @@ -910,6 +929,15 @@ typedef struct ibl_code_t { far_ref_t far_jmp_unlinked_opnd; #endif byte *unlinked_ibl_entry; +#ifdef LINUX_KERNEL + byte *found_unlinked; + byte *found_unlinked_eflags; + byte *found_unlinked_prefix; + byte *found_unlinked_eflags_prefix; + int num_ibl_found_exits; + ibl_found_exit_t ibl_found_exits[MAX_IBL_FOUND_EXITS]; + byte *indirect_branch_lookup_routine_end; +#endif byte *target_delete_entry; uint ibl_routine_length; /* offsets into ibl routine */ @@ -957,6 +985,7 @@ typedef struct ibl_code_t { */ typedef struct _generated_code_t { byte *fcache_enter; + byte *fcache_enter_end; byte *fcache_return; byte *fcache_return_end; #ifdef WINDOWS_PC_SAMPLE @@ -1047,6 +1076,12 @@ typedef struct _generated_code_t { byte *clean_call_restore; byte *clean_call_restore_end; +#ifdef LINUX_KERNEL + byte *syscall_entry; + byte *common_vector_entry; + byte *vector_entry[VECTOR_END - VECTOR_START]; +#endif + bool thread_shared; bool writable; #if defined(X86) && defined(X64) @@ -1094,7 +1129,7 @@ cache_pc trace_head_return_coarse_routine(IF_X86_64(gencode_mode_t mode)); /* shared clean call context switch */ bool -client_clean_call_is_thread_private(); +client_clean_call_is_thread_private(void); cache_pc get_clean_call_save(dcontext_t *dcontext _IF_X86_64(gencode_mode_t mode)); cache_pc @@ -1222,6 +1257,12 @@ byte * emit_indirect_branch_lookup(dcontext_t *dcontext, generated_code_t *code, byte *pc, byte *fcache_return_pc, bool target_trace_table, bool inline_ibl_head, ibl_code_t *ibl_code); +#ifdef LINUX_KERNEL +byte * +emit_ibl_found_unlinked_code(dcontext_t *dcontext, byte *pc, byte *fcache_return_pc, + ibl_code_t *ibl_code, bool restore_eflags, + bool include_prefix); +#endif void update_indirect_branch_lookup(dcontext_t *dcontext); bool @@ -1265,6 +1306,19 @@ emit_do_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc, byte *fcache_return_pc, bool thread_shared, int interrupt, uint *syscall_offs /*OUT*/); +#ifdef LINUX_KERNEL +# define VECTOR_ENTRY_CODE_SIZE (2 * PUSH_IMM32_LENGTH + JMP_LONG_LENGTH) +byte * +emit_syscall_entry(dcontext_t *dcontext, cache_pc fcache_return, app_pc target, + cache_pc pc); +byte * +emit_common_vector_entry(dcontext_t *dcontext, byte *tls_base, + interrupt_handler_t handler, cache_pc pc); +byte * +emit_vector_entry(dcontext_t *dcontext, byte *common_vector_entry_pc, + interrupt_vector_t vector, cache_pc pc); +#endif + #ifdef AARCH64 /* Generate move (immediate) of a 64-bit value using at most 4 instructions. * pc must be a writable (vmcode) pc. diff --git a/core/arch/arch_exports.h b/core/arch/arch_exports.h index 270942fcd..fbaf0ff30 100644 --- a/core/arch/arch_exports.h +++ b/core/arch/arch_exports.h @@ -99,6 +99,7 @@ typedef enum { struct _fragment_entry_t; /* in fragment.h */ struct _ibl_table_t; /* in fragment.h */ +struct ibl_code_t; /* in arch.h */ /* Scratch space and state required to be easily accessible from * in-cache indirect branch lookup routines, store in thread-local storage. @@ -422,6 +423,10 @@ get_time(void); void d_r_arch_init(void); void d_r_arch_exit(IF_WINDOWS_ELSE_NP(bool detach_stacked_callbacks, void)); +#ifdef LINUX_KERNEL +void +optimize_syscall_code(dcontext_t *dcontext, fragment_t *f); +#endif void arch_thread_init(dcontext_t *dcontext); void @@ -445,6 +450,8 @@ is_indirect_branch_lookup_routine(dcontext_t *dcontext, cache_pc pc); bool in_generated_routine(dcontext_t *dcontext, cache_pc pc); bool +in_fcache_enter(dcontext_t *dcontext, cache_pc pc); +bool in_fcache_return(dcontext_t *dcontext, cache_pc pc); bool in_clean_call_save(dcontext_t *dcontext, cache_pc pc); @@ -1232,6 +1239,14 @@ void link_indirect_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, bool hot_patch); void unlink_indirect_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l); +#ifdef LINUX_KERNEL +void +unlink_ibl_routine(dcontext_t *dcontext, cache_pc interrupted_ibl_pc); +void +link_ibl_routine(dcontext_t *dcontext, cache_pc interrupted_ibl_pc); +struct ibl_code_t * +get_ibl_code_from_routine_pc(dcontext_t *dcontext, cache_pc pc); +#endif void insert_fragment_prefix(dcontext_t *dcontext, fragment_t *f); int @@ -1879,4 +1894,84 @@ typedef struct _rseq_entry_state_t { # define ARCH_SUPPORTS_HW_CACHE_CONSISTENCY #endif +#ifdef LINUX_KERNEL +typedef enum { + VECTOR_DIVIDE_ERROR = 0, + VECTOR_START = VECTOR_DIVIDE_ERROR, + VECTOR_EXCEPTION_START = VECTOR_DIVIDE_ERROR, + VECTOR_DEBUG = 1, + VECTOR_NMI = 2, + VECTOR_BREAKPOINT = 3, + VECTOR_OVERFLOW = 4, + VECTOR_BOUND_RANGE_EXCEEDED = 5, + VECTOR_INVALID_OPCODE = 6, + VECTOR_DEVICE_NOT_AVAILABLE = 7, + VECTOR_DOUBLE_FAULT = 8, + VECTOR_COPROCESSOR_SEGMENT_OVERRUN = 9, + VECTOR_INVALID_TSS = 10, + VECTOR_SEGMENT_NOT_PRESENT = 11, + VECTOR_STACK_FAULT = 12, + VECTOR_GENERAL_PROTECTION = 13, + VECTOR_PAGE_FAULT = 14, + /* no 15 */ + VECTOR_X87_FPU_FLOATING_POINT_ERROR = 16, + VECTOR_ALIGNMENT_CHECK = 17, + VECTOR_MACHINE_CHECK = 18, + VECTOR_SIMD_FLOATING_POINT = 19, + VECTOR_EXCEPTION_END = 20, + VECTOR_INTERRUPT_START = 20, + VECTOR_INTERRUPT_END = 256, + VECTOR_END = VECTOR_INTERRUPT_END +} interrupt_vector_t; + +# define MAGIC_FAKE_ERROR 0xfffffffffbadbeef + +void +patch_interrupt(dcontext_t *dcontext, cache_pc patch_pc, interrupt_vector_t vector, + byte save_buffer[INT_LENGTH]); + +void +unpatch_interrupt(dcontext_t *dcontext, cache_pc patch_pc, byte save_buffer[INT_LENGTH]); + +/* Returns true if the hardware normally pushes an error code on this vector's + * interrupt stack frame. + */ +bool +vector_has_error_code(interrupt_vector_t vector); + +/* Returns true if the interrupt vector corresponds to a synchronous CPU exception. */ +bool +vector_is_synchronous(interrupt_vector_t vector); + +/* The layout of the interrupt stack frame. The stack pointer will point to + * error_code. */ +typedef struct { + reg_t error_code; + byte *xip; + reg_t cs; + reg_t xflags; + reg_t xsp; + reg_t ss; +} interrupt_stack_frame_t; + +# define INTERRUPT_STACK_FRAME_ALIGNMENT 0x10 + +bool +was_kernel_interrupted(interrupt_stack_frame_t *frame); + +typedef void (*interrupt_handler_t)(interrupt_stack_frame_t *, priv_mcontext_t *, + interrupt_vector_t); + +/* Returns the gencode for syscall entry. */ +cache_pc +get_syscall_entry(dcontext_t *dcontext); +/* Returns the gencode for vector entry. */ +cache_pc +get_vector_entry(dcontext_t *dcontext, interrupt_vector_t vector); + +void +clean_call_clear_saved_interrupt_flag(dcontext_t *dcontext, byte *sp); + +#endif /* LINUX_KERNEL */ + #endif /* _ARCH_EXPORTS_H_ */ diff --git a/core/arch/atomic_exports.h b/core/arch/atomic_exports.h index ad5cc7ee2..725a2cce7 100644 --- a/core/arch/atomic_exports.h +++ b/core/arch/atomic_exports.h @@ -195,19 +195,19 @@ /* returns true if result value is zero */ static inline bool -atomic_inc_and_test(volatile int *var) +d_r_atomic_inc_and_test(volatile int *var) { return (ATOMIC_INC(int, *(var)) == 0); } /* Returns true if result value is negative. */ static inline bool -atomic_dec_and_test(volatile int *var) +d_r_atomic_dec_and_test(volatile int *var) { return (ATOMIC_DEC(int, *(var)) < 0); } /* returns true if result value is zero */ static inline bool -atomic_dec_becomes_zero(volatile int *var) +d_r_atomic_dec_becomes_zero(volatile int *var) { return (ATOMIC_DEC(int, *(var)) == 0); } @@ -576,19 +576,19 @@ DEF_ATOMIC_incdec(ATOMIC_INC_int, int, "w", "add") DEF_ATOMIC_incdec(ATOMIC_INC_ } while (0) /* wait for interrupt */ static inline bool -atomic_inc_and_test(volatile int *var) +d_r_atomic_inc_and_test(volatile int *var) { return atomic_add_exchange_int(var, 1) == 0; } static inline bool -atomic_dec_and_test(volatile int *var) +d_r_atomic_dec_and_test(volatile int *var) { return atomic_add_exchange_int(var, -1) < 0; } static inline bool -atomic_dec_becomes_zero(volatile int *var) +d_r_atomic_dec_becomes_zero(volatile int *var) { return atomic_add_exchange_int(var, -1) == 0; } @@ -1050,19 +1050,19 @@ atomic_exchange_int(volatile int *var, int newval) # define SPINLOCK_PAUSE() __asm__ __volatile__(".int 0x0100000F"); static inline bool -atomic_inc_and_test(volatile int *var) +d_r_atomic_inc_and_test(volatile int *var) { return atomic_add_exchange_int(var, 1) == 0; } static inline bool -atomic_dec_and_test(volatile int *var) +d_r_atomic_dec_and_test(volatile int *var) { return atomic_add_exchange_int(var, -1) < 0; } static inline bool -atomic_dec_becomes_zero(volatile int *var) +d_r_atomic_dec_becomes_zero(volatile int *var) { return atomic_add_exchange_int(var, -1) == 0; } @@ -1085,7 +1085,7 @@ atomic_dec_becomes_zero(volatile int *var) * Returns true if the resulting value is zero, otherwise returns false */ static inline bool -atomic_inc_and_test(volatile int *var) +d_r_atomic_inc_and_test(volatile int *var) { bool is_zero; ATOMIC_INC_int_nowzero(*var, is_zero); @@ -1096,7 +1096,7 @@ atomic_inc_and_test(volatile int *var) * Returns true if the result is negative, otherwise returns false. */ static inline bool -atomic_dec_and_test(volatile int *var) +d_r_atomic_dec_and_test(volatile int *var) { bool is_negative; ATOMIC_DEC_int_nowneg(*var, is_negative); @@ -1107,7 +1107,7 @@ atomic_dec_and_test(volatile int *var) * Returns true if the resulting value is zero, otherwise returns false. */ static inline bool -atomic_dec_becomes_zero(volatile int *var) +d_r_atomic_dec_becomes_zero(volatile int *var) { bool is_zero; ATOMIC_DEC_int_nowzero(*var, is_zero); diff --git a/core/arch/emit_utils_shared.c b/core/arch/emit_utils_shared.c index 0b8adbf45..fe0b1bc9d 100644 --- a/core/arch/emit_utils_shared.c +++ b/core/arch/emit_utils_shared.c @@ -3163,6 +3163,7 @@ append_ibl_found(dcontext_t *dcontext, instrlist_t *ilist, ibl_code_t *ibl_code, /*>>> RESTORE_FROM_UPCONTEXT xbx_OFFSET,%xbx */ /*>>> jmp *FRAGMENT_START_PC_OFFS(%xcx) */ instr_t *inst = NULL; + instr_t *final_jmp = NULL; IF_X86_64(bool x86_to_x64_ibl_opt = (ibl_code->x86_to_x64_mode && DYNAMO_OPTION(x86_to_x64_ibl_opt));) @@ -3257,9 +3258,9 @@ append_ibl_found(dcontext_t *dcontext, instrlist_t *ilist, ibl_code_t *ibl_code, #if defined(AARCH64) || defined(RISCV64) ASSERT_NOT_IMPLEMENTED(false); /* TODO i#1569 i#3544 */ #else - APP(ilist, - XINST_CREATE_jump_mem(dcontext, - OPND_CREATE_MEMPTR(SCRATCH_REG2, start_pc_offset))); + final_jmp = XINST_CREATE_jump_mem( + dcontext, OPND_CREATE_MEMPTR(SCRATCH_REG2, start_pc_offset)); + APP(ilist, final_jmp); #endif } else { /* There is no prefix so we must restore all and jmp through memory: @@ -3281,10 +3282,9 @@ append_ibl_found(dcontext_t *dcontext, instrlist_t *ilist, ibl_code_t *ibl_code, RESTORE_FROM_TLS(dcontext, SCRATCH_REG2, MANGLE_XCX_SPILL_SLOT)); } else APP(ilist, RESTORE_FROM_DC(dcontext, SCRATCH_REG2, SCRATCH_REG2_OFFS)); - APP(ilist, - XINST_CREATE_jump_mem( - dcontext, - OPND_DC_FIELD(absolute, dcontext, OPSZ_PTR, SCRATCH_REG2_OFFS))); + final_jmp = XINST_CREATE_jump_mem( + dcontext, OPND_DC_FIELD(absolute, dcontext, OPSZ_PTR, SCRATCH_REG2_OFFS)); + APP(ilist, final_jmp); #elif defined(AARCH64) ASSERT_NOT_IMPLEMENTED(false); /* TODO i#1569: NYI on AArch64 */ #elif defined(ARM) @@ -3295,22 +3295,48 @@ append_ibl_found(dcontext_t *dcontext, instrlist_t *ilist, ibl_code_t *ibl_code, } else { APP(ilist, SAVE_TO_TLS(dcontext, SCRATCH_REG2, INDIRECT_STUB_SPILL_SLOT)); #if defined(X86) && defined(X64) - if (x86_to_x64_ibl_opt) + if (x86_to_x64_ibl_opt) { APP(ilist, RESTORE_FROM_REG(dcontext, SCRATCH_REG2, REG_R9)); - else -#endif + } else { APP(ilist, RESTORE_FROM_TLS(dcontext, SCRATCH_REG2, MANGLE_XCX_SPILL_SLOT)); + } +#else + APP(ilist, RESTORE_FROM_TLS(dcontext, SCRATCH_REG2, MANGLE_XCX_SPILL_SLOT)); +#endif #if defined(AARCH64) || defined(RISCV64) ASSERT_NOT_IMPLEMENTED(false); /* TODO i#1569 i#3544 */ #else - APP(ilist, - XINST_CREATE_jump_mem(dcontext, - OPND_TLS_FIELD(INDIRECT_STUB_SPILL_SLOT))); + final_jmp = + XINST_CREATE_jump_mem(dcontext, OPND_TLS_FIELD(INDIRECT_STUB_SPILL_SLOT)); + APP(ilist, final_jmp); #endif } } + inst = final_jmp; +#if defined(X86) && defined(LINUX_KERNEL) + if (final_jmp != NULL) { + ibl_found_exit_t *exit; + ASSERT(ibl_code->num_ibl_found_exits < MAX_IBL_FOUND_EXITS); + exit = &ibl_code->ibl_found_exits[ibl_code->num_ibl_found_exits]; + ibl_code->num_ibl_found_exits += 1; + add_patch_marker(patch, final_jmp, PATCH_ASSEMBLE_ABSOLUTE, 0, + (ptr_uint_t *)&exit->jmp_pc); + exit->targets_prefix = target_prefix; + exit->restored_eflags = restore_eflags; + exit->original_jmp_length = instr_length(dcontext, final_jmp); + ASSERT(exit->original_jmp_length <= MAX_INSTR_LENGTH); + if (exit->original_jmp_length < JMP_LONG_LENGTH) { + APP(ilist, + INSTR_CREATE_nopNbyte(dcontext, + JMP_LONG_LENGTH - exit->original_jmp_length)); + } + memset(exit->original_jmp_code, 0x90, sizeof(exit->original_jmp_code)); + instr_encode(dcontext, final_jmp, exit->original_jmp_code); + } +#endif + if (fragment_found != NULL) *fragment_found = inst; } diff --git a/core/arch/interp.c b/core/arch/interp.c index a18b9efa0..652998f54 100644 --- a/core/arch/interp.c +++ b/core/arch/interp.c @@ -129,7 +129,7 @@ DECLARE_NEVERPROT_VAR(uint debug_bb_count, 0); /* initialization */ void -interp_init() +interp_init(void) { if (INTERNAL_OPTION(bbdump_tags)) { bbdump_file = open_log_file("bbs", NULL, 0); @@ -146,7 +146,7 @@ static int num_rets_removed; /* cleanup */ void -interp_exit() +interp_exit(void) { if (INTERNAL_OPTION(bbdump_tags)) { close_log_file(bbdump_file); @@ -1647,6 +1647,7 @@ bb_process_mov_seg(dcontext_t *dcontext, build_bb_t *bb) } #endif /* UNIX && X86 */ +#ifndef LINUX_KERNEL /* Returns true to indicate that ignorable syscall processing is completed * with *continue_bb indicating if the bb should be continued or not. * When returning false, continue_bb isn't pertinent. @@ -1657,7 +1658,7 @@ bb_process_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb, int sysnum, { STATS_INC(ignorable_syscalls); BBPRINT(bb, 3, "found ignorable system call 0x%04x\n", sysnum); -#ifdef WINDOWS +# ifdef WINDOWS if (get_syscall_method() != SYSCALL_METHOD_SYSENTER) { DOCHECK(1, { if (get_syscall_method() == SYSCALL_METHOD_WOW64) @@ -1711,7 +1712,7 @@ bb_process_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb, int sysnum, STATS_DEC(ignorable_syscalls); return false; } -#elif defined(MACOS) && defined(X86) +# elif defined(MACOS) && defined(X86) if (instr_get_opcode(bb->instr) == OP_sysenter) { /* To continue after the sysenter we need to go to the ret ibl, as user-mode * sysenter wrappers put the retaddr into edx as the post-kernel continuation. @@ -1726,12 +1727,13 @@ bb_process_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb, int sysnum, } else if (continue_bb != NULL) *continue_bb = true; return true; -#else +# else if (continue_bb != NULL) *continue_bb = true; return true; -#endif +# endif } +#endif // LINUX_KERNEL #ifdef WINDOWS /* Process a syscall that is executed via shared syscall. */ @@ -1849,6 +1851,7 @@ adjust_it_instr_for_split(dcontext_t *dcontext, instr_t *it, uint pos) } #endif /* ARM */ +#ifndef LINUX_KERNEL static bool bb_process_non_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb, int sysnum) { @@ -1859,26 +1862,26 @@ bb_process_non_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb, int sysnu LOG(THREAD, LOG_INTERP, 3, "ending bb at syscall & removing the interrupt itself\n"); /* Indicate that this is a non-ignorable syscall so mangle will remove */ /* XXX i#1551: maybe we should union int80 and svc as both are inline syscall? */ -#ifdef UNIX +# ifdef UNIX if (instr_get_opcode(bb->instr) == IF_X86_ELSE(OP_int, IF_RISCV64_ELSE(OP_ecall, OP_svc))) { -# if defined(MACOS) && defined(X86) +# if defined(MACOS) && defined(X86) int num = instr_get_interrupt_number(bb->instr); if (num == 0x81 || num == 0x82) { bb->exit_type |= LINK_SPECIAL_EXIT; bb->instr->flags |= INSTR_BRANCH_SPECIAL_EXIT; } else { ASSERT(num == 0x80); -# endif /* MACOS && X86 */ +# endif /* MACOS && X86 */ bb->exit_type |= LINK_NI_SYSCALL_INT; bb->instr->flags |= INSTR_NI_SYSCALL_INT; -# if defined(MACOS) && defined(X86) +# if defined(MACOS) && defined(X86) } -# endif +# endif } else -#endif +# endif bb->instr->flags |= INSTR_NI_SYSCALL; -#ifdef ARM +# ifdef ARM /* we assume all conditional syscalls are treated as non-ignorable */ if (instr_is_predicated(bb->instr)) { instr_t *it; @@ -1894,18 +1897,24 @@ bb_process_non_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb, int sysnu adjust_it_instr_for_split(dcontext, it, pos); } } -#endif +# endif /* Set instr to NULL in order to get translation of exit cti correct. */ bb->instr = NULL; /* this block must be the last one in a trace */ bb->flags |= FRAG_MUST_END_TRACE; return false; /* end bb now */ } +#endif // LINUX_KERNEL /* returns true to indicate "continue bb" and false to indicate "end bb now" */ static inline bool bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb) { +#ifdef LINUX_KERNEL + ASSERT_MESSAGE(CHKLVL_ASSERTS, "There should not be system calls in the kernel.", + false); + return true; +#else int sysnum; /* PR 307284: for simplicity do syscall/int processing post-client. * We give up on inlining but we can still use ignorable/shared syscalls @@ -1913,12 +1922,12 @@ bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb) */ if (bb->pass_to_client && !bb->post_client) return false; -#ifdef DGC_DIAGNOSTICS +# ifdef DGC_DIAGNOSTICS if (TEST(FRAG_DYNGEN, bb->flags) && !is_dyngen_vsyscall(bb->instr_start)) { LOG(THREAD, LOG_INTERP, 1, "WARNING: syscall @ " PFX " in dyngen code!\n", bb->instr_start); } -#endif +# endif BBPRINT(bb, 4, "interp: syscall @ " PFX "\n", bb->instr_start); check_syscall_method(dcontext, bb->instr); bb->flags |= FRAG_HAS_SYSCALL; @@ -1926,7 +1935,7 @@ bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb) * we let bb keep going, else we end bb and flag it */ sysnum = find_syscall_num(dcontext, bb->ilist, bb->instr); -#ifdef VMX86_SERVER +# ifdef VMX86_SERVER DOSTATS({ if (instr_get_opcode(bb->instr) == OP_int && instr_get_interrupt_number(bb->instr) == VMKUW_SYSCALL_GATEWAY) { @@ -1934,14 +1943,14 @@ bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb) LOG(THREAD, LOG_SYSCALLS, 2, "vmkuw system call site: #=%d\n", sysnum); } }); -#endif +# endif BBPRINT(bb, 3, "syscall # is %d\n", sysnum); if (sysnum != -1 && instrument_filter_syscall(dcontext, sysnum)) { BBPRINT(bb, 3, "client asking to intercept => pretending syscall # %d is -1\n", sysnum); sysnum = -1; } -#ifdef ARM +# ifdef ARM if (sysnum != -1 && instr_is_predicated(bb->instr)) { BBPRINT(bb, 3, "conditional system calls cannot be inlined => " @@ -1949,17 +1958,17 @@ bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb) sysnum); sysnum = -1; } -#endif +# endif if (sysnum != -1 && DYNAMO_OPTION(ignore_syscalls) && ignorable_system_call(sysnum, bb->instr, NULL) -#ifdef X86 +# ifdef X86 /* PR 288101: On Linux we do not yet support inlined sysenter instrs as we * do not have in-cache support for the post-sysenter continuation: we rely * for now on very simple sysenter handling where d_r_dispatch uses asynch_target * to know where to go next. */ IF_LINUX(&&instr_get_opcode(bb->instr) != OP_sysenter) -#endif /* X86 */ +# endif /* X86 */ ) { bool continue_bb; @@ -1970,16 +1979,17 @@ bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb) return continue_bb; } } -#ifdef WINDOWS +# ifdef WINDOWS if (sysnum != -1 && DYNAMO_OPTION(shared_syscalls) && optimizable_system_call(sysnum)) { bb_process_shared_syscall(dcontext, bb, sysnum); return false; } -#endif +# endif /* Fall thru and handle as a non-ignorable syscall. */ return bb_process_non_ignorable_syscall(dcontext, bb, sysnum); +#endif } /* Case 3922: for wow64 we treat "call *fs:0xc0" as a system call. @@ -2750,7 +2760,7 @@ client_process_bb(dcontext_t *dcontext, build_bb_t *bb) /* DrMem#1735: pass app pc, not selfmod copy pc */ app_pc tag = bb->pretend_pc == NULL ? bb->start_pc : bb->pretend_pc; -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) if (TEST(FRAG_STARTS_RSEQ_REGION, bb->flags)) { rseq_insert_start_label(dcontext, tag, bb->ilist); /* This is a temporary flag, as it overlaps with another used later. */ @@ -3286,17 +3296,19 @@ build_bb_ilist(dcontext_t *dcontext, build_bb_t *bb) /* to split riprel, need to decode every instr */ /* in x86_to_x64, need to translate every x86 instr */ IF_X64(|| DYNAMO_OPTION(coarse_split_riprel) || DYNAMO_OPTION(x86_to_x64)) || - INTERNAL_OPTION(full_decode) - /* We separate rseq regions into their own blocks to make this check easier. */ - IF_LINUX(|| - (!vmvector_empty(d_r_rseq_areas) && - vmvector_overlap(d_r_rseq_areas, bb->start_pc, bb->start_pc + 1)))) + INTERNAL_OPTION(full_decode)) { bb->full_decode = true; - else { -#ifdef CHECK_RETURNS_SSE2 + } +#if defined(LINUX) && !defined(LINUX_KERNEL) + /* We separate rseq regions into their own blocks to make this check easier. */ + if (!vmvector_empty(d_r_rseq_areas) && + vmvector_overlap(d_r_rseq_areas, bb->start_pc, bb->start_pc + 1)) { bb->full_decode = true; -#endif } +#endif +#ifdef CHECK_RETURNS_SSE2 + bb->full_decode = true; +#endif LOG(THREAD, LOG_INTERP, 3, "\ninterp%s: ", IF_X86_64_ELSE(X64_MODE_DC(dcontext) ? "" : " (x86 mode)", "")); @@ -4034,7 +4046,7 @@ build_bb_ilist(dcontext_t *dcontext, build_bb_t *bb) return; } } -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) /* XXX: i#1247: After a call to a native module throught plt, DR * loses control of the app b/c of _dl_runtime_resolve */ @@ -6906,7 +6918,7 @@ mangle_trace(dcontext_t *dcontext, instrlist_t *ilist, monitor_data_t *md) { instr_t *inst, *next_inst, *start_instr, *jmp; uint blk, num_exits_deleted; - app_pc fallthrough = NULL; + app_pc fallthrough_pc = NULL; bool found_syscall = false, found_int = false; /* We don't assert that mangle_trace_at_end() is true b/c the client @@ -6974,7 +6986,7 @@ mangle_trace(dcontext_t *dcontext, instrlist_t *ilist, monitor_data_t *md) /* Do not call instr_length() on this inst: use length * of translation! (i#509) */ - fallthrough = decode_next_pc(dcontext, xl8); + fallthrough_pc = decode_next_pc(dcontext, xl8); } /* PR 299808: identify bb boundaries. We can't go by translations alone, as @@ -7056,7 +7068,7 @@ mangle_trace(dcontext_t *dcontext, instrlist_t *ilist, monitor_data_t *md) return false; } /* must have been no final exit cti: add final fall-through jmp */ - jmp = create_exit_jmp(dcontext, fallthrough, fallthrough, 0); + jmp = create_exit_jmp(dcontext, fallthrough_pc, fallthrough_pc, 0); /* XXX PR 307284: support client modifying, replacing, or adding * syscalls and ints: need to re-analyze. Then we wouldn't * need the md->final_exit_flags field anymore. diff --git a/core/arch/mangle_shared.c b/core/arch/mangle_shared.c index e10731482..6a635fd6e 100644 --- a/core/arch/mangle_shared.c +++ b/core/arch/mangle_shared.c @@ -935,7 +935,7 @@ get_call_return_address(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr return retaddr; } -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) /* find the system call number in instrlist for an inlined system call * by simpling walking the ilist backward and finding "mov immed => %eax" * without checking cti or expanding instr @@ -960,7 +960,7 @@ static void mangle_syscall(dcontext_t *dcontext, instrlist_t *ilist, uint flags, instr_t *instr, instr_t *next_instr) { -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) if (get_syscall_method() != SYSCALL_METHOD_INT && get_syscall_method() != SYSCALL_METHOD_SYSCALL && get_syscall_method() != SYSCALL_METHOD_SYSENTER && @@ -1019,12 +1019,12 @@ mangle_syscall(dcontext_t *dcontext, instrlist_t *ilist, uint flags, instr_t *in instr_set_app(instr); instrlist_postinsert(ilist, instr, nop); } -#endif /* UNIX */ +#endif /* UNIX && !LINUX_KERNEL */ mangle_syscall_arch(dcontext, ilist, flags, instr, next_instr); } -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) /* If skip is false: * changes the jmp right before the next syscall (after pc) to target the * exit cti immediately following it; @@ -1136,9 +1136,9 @@ mangle_syscall_code(dcontext_t *dcontext, fragment_t *f, byte *pc, bool skip) instr_free(dcontext, &instr); return true; } -#endif /* UNIX */ +#endif /* UNIX && !LINUX_KERNEL */ -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) /*************************************************************************** * Rseq (restartable sequence) mangling. */ @@ -1826,7 +1826,7 @@ mangle_rseq_finalize(dcontext_t *dcontext, instrlist_t *ilist, fragment_t *f) /* We should have found at least one set of labels. */ ASSERT(label_sets_found > 0); } -#endif /* LINUX */ +#endif /* LINUX && !LINUX_KERNEL */ /* TOP-LEVEL MANGLE * This routine is responsible for mangling a fragment into the form @@ -1914,7 +1914,7 @@ d_r_mangle(dcontext_t *dcontext, instrlist_t *ilist, uint *flags DR_PARAM_INOUT, translate_x86_to_x64(dcontext, ilist, &instr); #endif -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) /* Mangle stores inside restartable sequences ("rseq"). We could avoid the * per-instr check if we disallowed rseq blocks in traces and prevented * fall-through in a bb, but that would lead to more problems than it would @@ -2211,6 +2211,7 @@ cti_is_normal_elision(instr_t *instr) * (although -1 is invalid), so be sure to test for -1 and not just <0 as a failure * code. */ +#ifndef LINUX_KERNEL int find_syscall_num(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr) { @@ -2330,6 +2331,7 @@ find_syscall_num(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr) IF_X64(ASSERT_TRUNCATE(int, int, syscall)); return (int)syscall; } +#endif /* !LINUX_KERNEL */ void mangle_finalize(dcontext_t *dcontext, instrlist_t *ilist, fragment_t *f) @@ -2339,7 +2341,7 @@ mangle_finalize(dcontext_t *dcontext, instrlist_t *ilist, fragment_t *f) finalize_selfmod_sandbox(dcontext, f); } #endif -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) if (TEST(INSTR_RSEQ_ENDPOINT, ilist->flags)) mangle_rseq_finalize(dcontext, ilist, f); #endif diff --git a/core/arch/x86/emit_utils.c b/core/arch/x86/emit_utils.c index 3fc21b15c..8d4a33359 100644 --- a/core/arch/x86/emit_utils.c +++ b/core/arch/x86/emit_utils.c @@ -51,6 +51,10 @@ #include "decode_private.h" +#ifdef LINUX_KERNEL +# include "msr.h" +#endif + #define PRE instrlist_meta_preinsert #define APP instrlist_meta_append @@ -517,10 +521,9 @@ insert_inlined_ibl(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, byte *pc, } else { insert_relative_target(start_pc + ibl_code->inline_linkedjmp_offs, linked_exit_target, NOT_HOT_PATCHABLE); - insert_relative_target( - start_pc + ibl_code->inline_unlink_offs + - 1 /* skip jmp opcode: see emit_inline_ibl_stub XXX */, - unlinked_exit_target, NOT_HOT_PATCHABLE); + /* Skip 1-byte jmp opcode: see emit_inline_ibl_stub XXX. */ + byte *unlink_disp = start_pc + ibl_code->inline_unlink_offs + 1; + insert_relative_target(unlink_disp, unlinked_exit_target, NOT_HOT_PATCHABLE); } return start_pc + ibl_code->inline_stub_length; @@ -931,6 +934,61 @@ unlink_indirect_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l) #endif } +#ifdef LINUX_KERNEL +static byte * +get_ibl_found_unlinked_target(ibl_code_t *ibl_code, ibl_found_exit_t *exit) +{ + if (exit->restored_eflags && exit->targets_prefix) { + return ibl_code->found_unlinked_prefix; + } else if (exit->restored_eflags && !exit->targets_prefix) { + return ibl_code->found_unlinked; + } else if (!exit->restored_eflags && exit->targets_prefix) { + return ibl_code->found_unlinked_eflags_prefix; + } else { /* !exit->restored_eflags && !exit->targets_prefix */ + /* In append_ibl_found, !target_prefix => restore_eflags. */ + ASSERT_NOT_REACHED(); + return ibl_code->found_unlinked_eflags; + } +} + +static void +ibl_linking_common(dcontext_t *dcontext, cache_pc interrupted_ibl_pc, bool link) +{ + int i; + generated_code_t *gencode = THREAD_GENCODE(dcontext); + ibl_code_t *code = get_ibl_code_from_routine_pc(dcontext, interrupted_ibl_pc); + ASSERT(code != NULL); + ASSERT(code->indirect_branch_lookup_routine >= gencode->gen_start_pc && + code->indirect_branch_lookup_routine < gencode->gen_end_pc); + protect_generated_code(gencode, WRITABLE); + for (i = 0; i < code->num_ibl_found_exits; i++) { + ibl_found_exit_t *exit = &code->ibl_found_exits[i]; + if (link) { + memcpy(vmcode_get_writable_addr(exit->jmp_pc), exit->original_jmp_code, + JMP_LONG_LENGTH); + } else { + IF_DEBUG(byte *pc =) + insert_relative_jump(exit->jmp_pc, get_ibl_found_unlinked_target(code, exit), + false); + ASSERT(pc - exit->jmp_pc == JMP_LONG_LENGTH); + } + } + protect_generated_code(gencode, READONLY); +} + +void +unlink_ibl_routine(dcontext_t *dcontext, cache_pc interrupted_ibl_pc) +{ + ibl_linking_common(dcontext, interrupted_ibl_pc, false /* unlink */); +} + +void +link_ibl_routine(dcontext_t *dcontext, cache_pc interrupted_ibl_pc) +{ + ibl_linking_common(dcontext, interrupted_ibl_pc, true /* link */); +} +#endif + /******************************************************************************* * COARSE-GRAIN FRAGMENT SUPPORT */ @@ -3089,6 +3147,54 @@ emit_indirect_branch_lookup(dcontext_t *dcontext, generated_code_t *code, byte * return pc + ibl_code->ibl_routine_length; } +#ifdef LINUX_KERNEL +byte * +emit_ibl_found_unlinked_code(dcontext_t *dcontext, byte *pc, byte *fcache_return_pc, + ibl_code_t *ibl_code, bool restore_eflags, + bool include_prefix) +{ + instrlist_t *ilist = instrlist_create(dcontext); + bool absolute = !ibl_code->thread_shared_routine; + IF_X86_64(bool x86_to_x64_ibl_opt = + (ibl_code->x86_to_x64_mode && DYNAMO_OPTION(x86_to_x64_ibl_opt));) + if (absolute) { + ASSERT_NOT_IMPLEMENTED(false); + return NULL; + } + + if (restore_eflags || include_prefix) { + insert_restore_eflags(dcontext, ilist, NULL /* append */, 0, IBL_EFLAGS_IN_TLS(), + !absolute _IF_X64(x86_to_x64_ibl_opt)); + } + + APP(ilist, SAVE_TO_TLS(dcontext, REG_XAX, DIRECT_STUB_SPILL_SLOT)); + + if (include_prefix) { + APP(ilist, + INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX), + OPND_CREATE_MEMPTR(REG_XCX, HASHLOOKUP_START_PC_OFFS))); + APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XCX, MANGLE_XCX_SPILL_SLOT)); + } else { + APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XAX, INDIRECT_STUB_SPILL_SLOT)); + } + + insert_shared_get_dcontext(dcontext, ilist, NULL, true /* save xdi to scratch */); + + APP(ilist, SAVE_TO_DC(dcontext, REG_XAX, NEXT_TAG_OFFSET)); + APP(ilist, + INSTR_CREATE_mov_imm( + dcontext, opnd_create_reg(REG_XAX), + OPND_CREATE_INTPTR((ptr_uint_t)get_ibl_unlinked_found_linkstub()))); + + insert_shared_restore_dcontext_reg(dcontext, ilist, NULL); + APP(ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(fcache_return_pc))); + + pc = instrlist_encode(dcontext, ilist, pc, false /* no instr targets */); + instrlist_clear_and_destroy(dcontext, ilist); + return pc; +} +#endif + void relink_special_ibl_xfer(dcontext_t *dcontext, int index, ibl_entry_point_type_t entry_type, ibl_branch_type_t ibl_type) @@ -3143,6 +3249,400 @@ fill_with_nops(dr_isa_mode_t isa_mode, byte *addr, size_t size) return true; } +#ifdef LINUX_KERNEL +/* + # It's important to not use the native stack in this routine. x86 does not + # automatically switch stacks on system calls. So, if we pushed things here, + # then we would overwrite anything on the user's stack past %rsp. + swapgs + mov %xdi, %gs:DCONTEXT_BASE_SPILL_SLOT + mov %gs:TLS_DCONTEXT_SLOT, %xdi + mov %xax, %gs:TLS_XAX_SLOT + # dcontext->next_tag = target + mov target, %xax + mov %xax, NEXT_TAG_OFFSET(%xdi) + # xax = get_syscall_entry_linkstub(). fcache_return sets + # dcontext->last_exit = xax. + mov get_syscall_entry_linkstub(), %xax + # restore %xdi + mov %gs:DCONTEXT_BASE_SPILL_SLOT, %xdi + jmp fcache_return + jmp unexpected_return + */ +byte * +emit_syscall_entry(dcontext_t *dcontext, cache_pc fcache_return, app_pc target, + cache_pc pc) +{ + instr_t *swapgs = instr_create(dcontext); + + /* TODO i#31: If Kernel Indirect Branch Tracking (IBT) is enabled, the first + * instruction at target (entry_SYSCALL_64) is endbr64 (0xf3 0x0f 0x1e 0xfa). Skip it + * since endbr64 is not yet implemented in DynamoRIO's opcode tables. + */ + if (target[0] == 0xf3 && target[1] == 0x0f && target[2] == 0x1e && + target[3] == 0xfa) { + target += 4; + } + + decode(dcontext, target, swapgs); + ASSERT(instr_get_opcode(swapgs) == OP_swapgs); + + instrlist_t *ilist = instrlist_create(dcontext); + APP(ilist, swapgs); + /* Save XAX and XDI. Get dcontext into XDI. */ + insert_shared_get_dcontext(dcontext, ilist, NULL, true /* save_xdi */); + APP(ilist, SAVE_TO_TLS(dcontext, REG_XAX, TLS_XAX_SLOT)); + /* dcontext->next_tag = target */ + APP(ilist, + INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX), + OPND_CREATE_INTPTR(target))); + APP(ilist, SAVE_TO_DC_VIA_REG(false, dcontext, REG_XDI, REG_XAX, NEXT_TAG_OFFSET)); + /* XAX = get_syscall_entry_linkstub() (fcache_return sets + * dcontext->last_exit = XAX) */ + APP(ilist, + INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX), + OPND_CREATE_INTPTR(get_syscall_entry_linkstub()))); + /* Restore XDI. Don't need to restore XAX because fcache_return does that + * for us. */ + insert_shared_restore_dcontext_reg(dcontext, ilist, NULL); + /* Do fcache_return. */ + APP(ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(fcache_return))); + APP(ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc((app_pc)unexpected_return))); + pc = instrlist_encode(dcontext, ilist, pc, false); + instrlist_clear_and_destroy(dcontext, ilist); + return pc; +} + +/* Pushes mcontext onto the stack with pc = 0 and xmm registers with garbage + values. Clobbers xax. + + # Push an mcontext onto the stack. For now, we push 0 in place of %pc to + # indicate an invalid pc. The dispatcher will have to load the native + # entry point from the OS module. + push 0 + pushf + lea [%rsp - XMM_SLOTS_SIZE], %rsp + push %r15 + ... + push %r8 + push %xax + push %xcx + push %xdx + push %xbx + push %xsp + push %xbp + push %xsi + push %xdi + # Save the correct %xsp by offsetting the pushed mcontext and anything + # else that was pushed on the stack (i.e., extra_stack bytes). + # mcontext.xsp = %xsp + sizeof(dr_mcontext_t) + extra_stack + lea [%xsp + sizeof(dr_mcontext_t) + sizeof(reg_t)], %xax + mov %xax, [%xsp + offsetof(dr_mcontext_t, xsp)] +*/ + +static void +append_push_mcontext(dcontext_t *dcontext, int extra_stack, instrlist_t *ilist) +{ + int simd_slots_size = + MCXT_TOTAL_SIMD_SLOTS_SIZE + MCXT_TOTAL_OPMASK_SLOTS_SIZE + PRE_XMM_PADDING; + /* push %pc - use 0 to indicate that it's invalid */ + APP(ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0))); + APP(ilist, INSTR_CREATE_RAW_pushf(dcontext)); + ASSERT(!preserve_xmm_caller_saved()); + APP(ilist, + INSTR_CREATE_lea( + dcontext, opnd_create_reg(REG_XSP), + opnd_create_base_disp(REG_XSP, REG_NULL, 0, -simd_slots_size, OPSZ_lea))); +# ifdef X64 + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R15))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R14))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R13))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R12))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R11))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R10))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R9))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R8))); +# endif + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XCX))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XDX))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XBX))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XSP))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XBP))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XSI))); + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XDI))); + /* We pushed the wrong value for rsp. Fix that. */ + APP(ilist, + INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XAX), + opnd_create_base_disp(REG_XSP, REG_NULL, 0, + sizeof(priv_mcontext_t) + extra_stack, + OPSZ_lea))); + APP(ilist, + INSTR_CREATE_mov_st(dcontext, + opnd_create_base_disp(REG_XSP, REG_NULL, 0, + offsetof(priv_mcontext_t, xsp), + OPSZ_PTR), + opnd_create_reg(REG_XAX))); +} + +/* + Expects an interrupt stack frame with the vector number and error pushed + onto it. The stack is created like this where rsp = mcontext->rsp = native + rsp at the beginning of the interrupt handler. + + ss + xsp + xflags + cs + xip + error_code rsp + vector rsp - 8 + mcontext->pc rsp - 16 + ... the rest of mcontext + + + + # Restore the Kernel's GS base if necessary. + mov $MSR_GS_BASE, %rcx + rdmsr + shl $32, %rdx + or %rax, %rdx + mov $tls_base, %rcx + cmp %rcx, %rdx + jeq no_swapgs: + swapgs + no_swapgs: + + # Save handler arguments into registers. + lea [%rsp + sizeof(dr_mcontext_t)], %rax + mov %rsp, %rbx + mov [%rsp + sizeof(dr_mcontext_t) + sizeof(reg_t)], %rcx + + old_rsp = rsp + if (!on_dstack(rsp)) { + swap_stacks() + } + push(old_rsp) + handle_interrupt() + pop(rsp), so rsp = old_rsp + + # Swap stacks if we aren't already on the dstack. + mov %rsp, %r8 + # if rsp > dcontext->dstack, then goto swap_stacks + mov dcontext->dstack, %rdx + cmp %rdx, %rsp + jg swap_stacks + # if rsp < (smallest addr in dstack), + # then goto swapped_stacks + mov dcontext->dstack - DYNAMORIO_STACK_SIZE, %rdx + cmp %rdx, %rsp + jg swapped_stacks + swap_stacks: + mov dcontext->dstack, %rsp + swapped_stacks: + push %r8 + + # Call the handler. + handler(isf=%rax, mcontext=%rbx, vector=%rcx) + + # Restore native stack pointer. + pop %rsp + + # pop off mcontext + pop %xdi + pop %xsi + pop %xbp + pop %xsp + pop %xbx + pop %xdx + pop %xcx + pop %xax + pop %r8 + ... + pop %r15 + # Pop off unused XMM registers and dead xflags, xip, vector, and the + # error code. + lea [%rsp + simd_slots_size + sizeof(reg_t) * 4], %rsp + + iret +*/ +byte * +emit_common_vector_entry(dcontext_t *dcontext, byte *tls_base, + interrupt_handler_t handler, cache_pc pc) +{ + instrlist_t *ilist = instrlist_create(dcontext); + instr_t *no_swapgs = INSTR_CREATE_label(dcontext); + instr_t *swap_stacks = INSTR_CREATE_label(dcontext); + instr_t *swapped_stacks = INSTR_CREATE_label(dcontext); + int simd_slots_size = + MCXT_TOTAL_SIMD_SLOTS_SIZE + MCXT_TOTAL_OPMASK_SLOTS_SIZE + PRE_XMM_PADDING; + /* After we push mcontext, the stack contains + * mcontext + * vector + * interrupt stack frame (isf) + * if the isf does not have an error code, then we point the first byte of + * isf (i.e., the error code) to vector. + */ + int vector_offset = sizeof(priv_mcontext_t); + int isf_offset = vector_offset + sizeof(reg_t); + + append_push_mcontext(dcontext, sizeof(reg_t), ilist); + + /* Check if TLS is setup. Perform a swapgs if necessary. + * TODO(peter): We'll make this much more efficient when we implement FS + * stealing. In particular, we'll know which instruction addresses can fault + * with the native FS value. Knowing that, we won't have to check the MSR. + */ + APP(ilist, + INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX), + OPND_CREATE_INT64(MSR_GS_BASE))); + APP(ilist, INSTR_CREATE_rdmsr(dcontext)); + APP(ilist, + INSTR_CREATE_shl(dcontext, opnd_create_reg(REG_XDX), OPND_CREATE_INT8(32))); + APP(ilist, + INSTR_CREATE_or(dcontext, opnd_create_reg(REG_XDX), opnd_create_reg(REG_XAX))); + APP(ilist, + INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX), + OPND_CREATE_INT64(tls_base))); + APP(ilist, + INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XCX), opnd_create_reg(REG_XDX))); + APP(ilist, INSTR_CREATE_jcc(dcontext, OP_je, opnd_create_instr(no_swapgs))); + APP(ilist, INSTR_CREATE_swapgs(dcontext)); + APP(ilist, no_swapgs); + + /* Set rax = isf, rbx = mcontext, and rcx = vector then call + * handler(isf, mcontext, vector). + */ + APP(ilist, + INSTR_CREATE_lea( + dcontext, opnd_create_reg(REG_RAX), + opnd_create_base_disp(REG_XSP, REG_NULL, 0, isf_offset, OPSZ_lea))); + APP(ilist, + INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_RBX), + opnd_create_reg(REG_XSP))); + APP(ilist, + INSTR_CREATE_mov_ld( + dcontext, opnd_create_reg(REG_RCX), + opnd_create_base_disp(REG_XSP, REG_NULL, 0, vector_offset, OPSZ_PTR))); + + /* Use r8 to backup rsp for after the handler. */ + APP(ilist, + INSTR_CREATE_mov_st(dcontext, opnd_create_reg(REG_R8), opnd_create_reg(REG_RSP))); + /* rdx = dcontext->dstack */ + /* TODO(peter): We could probably perform this check with an immediate in + * cmp as long as the dstack address can be represented with a 32-bit sign + * extended value. */ + APP(ilist, + INSTR_CREATE_mov_imm( + dcontext, opnd_create_reg(REG_RDX), + opnd_create_immed_int((ptr_int_t)dcontext->dstack, OPSZ_PTR))); + /* TODO(peter): Kernel stacks are normally at lower addresses than the start + * of the dstack (i.e., this branch normally falls through). We could + * optimize this code by having the common check first + * (if rsp <= dstack - DYNAMORIO_STACK_SIZE). + */ + APP(ilist, + INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_RSP), opnd_create_reg(REG_RDX))); + APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jg, opnd_create_instr(swap_stacks))); + APP(ilist, + INSTR_CREATE_mov_imm( + dcontext, opnd_create_reg(REG_RDX), + opnd_create_immed_int((ptr_int_t)(dcontext->dstack - DYNAMORIO_STACK_SIZE), + OPSZ_PTR))); + APP(ilist, + INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_RSP), opnd_create_reg(REG_RDX))); + APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jg, opnd_create_instr(swapped_stacks))); + APP(ilist, swap_stacks); + APP(ilist, + INSTR_CREATE_mov_imm( + dcontext, opnd_create_reg(REG_RSP), + opnd_create_immed_int((ptr_int_t)dcontext->dstack, OPSZ_PTR))); + APP(ilist, swapped_stacks); + /* Push the old stack pointer value. */ + APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R8))); + + dr_insert_call(dcontext, ilist, NULL /*append*/, handler, 3, opnd_create_reg(REG_RAX), + opnd_create_reg(REG_RBX), opnd_create_reg(REG_RCX)); + + /* Restore the old stack pointer. Note that we could use callee-saved R12 + * instead of R8 and avoid the push/pop. I'm using push/pop just in case ;-) + */ + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RSP))); + + /* Restore the registers stored in mcontext. */ + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RDI))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RSI))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RBP))); + /* pop RSP into dead RBX */ + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RBX))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RBX))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RDX))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RCX))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RAX))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R8))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R9))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R10))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R11))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R12))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R13))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R14))); + APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R15))); + /* Fake pop xmm (which we don't use), xflags (which are dead), xip (which is + * dead), the vector, and the error code. */ + APP(ilist, + INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP), + opnd_create_base_disp(REG_XSP, REG_NULL, 0, + simd_slots_size + 4 * sizeof(reg_t), + OPSZ_lea))); + + APP(ilist, INSTR_CREATE_iret(dcontext)); + + pc = instrlist_encode(dcontext, ilist, pc, true); + instrlist_clear_and_destroy(dcontext, ilist); + return pc; +} + +/* + push $vector + jmp common_vector_entry_pc +*/ +byte * +emit_vector_entry(dcontext_t *dcontext, byte *common_vector_entry_pc, + interrupt_vector_t vector, byte *pc) +{ + instrlist_t *ilist = instrlist_create(dcontext); + if (!vector_has_error_code(vector)) { + APP(ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(MAGIC_FAKE_ERROR))); + } + APP(ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(vector))); + APP(ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(common_vector_entry_pc))); + pc = instrlist_encode(dcontext, ilist, pc, false); + instrlist_clear_and_destroy(dcontext, ilist); + return pc; +} + +void +patch_interrupt(dcontext_t *dcontext, cache_pc patch_pc, interrupt_vector_t vector, + byte save_buffer[INT_LENGTH]) +{ + memcpy(save_buffer, patch_pc, INT_LENGTH); + byte *write_pc = vmcode_get_writable_addr(patch_pc); + write_pc[0] = INTN_OPCODE; + ASSERT(!vector_has_error_code(vector)); + ASSERT_TRUNCATE(write_pc[1], byte, vector); + write_pc[1] = (byte)vector; +} + +void +unpatch_interrupt(dcontext_t *dcontext, cache_pc patch_pc, byte save_buffer[INT_LENGTH]) +{ + ASSERT(patch_pc[0] == INTN_OPCODE); + ASSERT(!vector_has_error_code(patch_pc[1])); + byte *write_pc = vmcode_get_writable_addr(patch_pc); + memcpy(write_pc, save_buffer, INT_LENGTH); +} +#endif /* LINUX_KERNEL */ + /* If code_buf points to a jmp rel32 returns true and returns the target of * the jmp in jmp_target as if was located at app_loc. */ bool diff --git a/core/arch/x86/mangle.c b/core/arch/x86/mangle.c index 5032c877b..399b98ed7 100644 --- a/core/arch/x86/mangle.c +++ b/core/arch/x86/mangle.c @@ -58,7 +58,7 @@ # include "../rct.h" /* rct_add_rip_rel_addr */ #endif -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) # include #endif @@ -863,6 +863,19 @@ insert_clean_call_with_arg_jmp_if_ret_true(dcontext_t *dcontext, instrlist_t *il instr_set_target(jcc, opnd_create_instr(false_popa)); } +#ifdef LINUX_KERNEL +void +clean_call_clear_saved_interrupt_flag(dcontext_t *dcontext, byte *sp) +{ + priv_mcontext_t *mc = get_priv_mcontext_from_dstack(dcontext); + if (sp <= (byte *)&mc->xflags) { + /* Clean call pushes 0 before it pushes xflags. */ + ASSERT(mc->pc == 0); + mc->xflags &= ~EFLAGS_IF; + } +} +#endif + /* If !precise, encode_pc is treated as +- a page (meant for clients * writing an instrlist to gencode so not sure of exact placement but * within a page). diff --git a/core/barrier.c b/core/barrier.c index 44a76362b..16149f291 100644 --- a/core/barrier.c +++ b/core/barrier.c @@ -17,18 +17,18 @@ bool barrier_wait(barrier_t *barrier) { int count; - mutex_lock(&barrier->lock); + d_r_mutex_lock(&barrier->lock); ASSERT(barrier->count > 0); barrier->count -= 1; count = barrier->count; - mutex_unlock(&barrier->lock); + d_r_mutex_unlock(&barrier->lock); if (count == 0) { return true; } for (;;) { - mutex_lock(&barrier->lock); + d_r_mutex_lock(&barrier->lock); count = barrier->count; - mutex_unlock(&barrier->lock); + d_r_mutex_unlock(&barrier->lock); if (count == 0) { return false; } diff --git a/core/buildmark.c b/core/buildmark.c index 270341557..6c1d58858 100644 --- a/core/buildmark.c +++ b/core/buildmark.c @@ -34,7 +34,10 @@ #include "globals.h" -#if defined(__DATE__) && defined(__TIME__) +/* Avoid __DATE__ and __TIME__ in kernel builds to avoid errors from + * -Werror=date-time and to support reproducible builds. + */ +#if defined(__DATE__) && defined(__TIME__) && !defined(LINUX_KERNEL) const char dynamorio_buildmark[] = __DATE__ " " __TIME__; #else const char dynamorio_buildmark[] = "unknown"; diff --git a/core/config.c b/core/config.c index 385435c72..5ac666367 100644 --- a/core/config.c +++ b/core/config.c @@ -45,6 +45,10 @@ # include "ntdll.h" # endif +# ifdef LINUX_KERNEL +# include "kernel_interface.h" +# endif + /* DYNAMORIO_VAR_CONFIGDIR is searched first, and then these: */ # ifdef UNIX # define GLOBAL_CONFIG_DIR "/etc/dynamorio" @@ -192,7 +196,9 @@ static config_vals_t *config_reread_vals; const char * my_getenv(IF_WINDOWS_ELSE_NP(const wchar_t *, const char *) var, char *buf, size_t bufsz) { -# ifdef UNIX +# ifdef LINUX_KERNEL + return kernel_getenv(var); +# elif defined(UNIX) return getenv(var); # else wchar_t wbuf[MAX_CONFIG_VALUE]; diff --git a/core/dispatch.c b/core/dispatch.c index 9c32a5eca..e0fa9af6c 100644 --- a/core/dispatch.c +++ b/core/dispatch.c @@ -89,8 +89,10 @@ dispatch_exit_fcache(dcontext_t *dcontext); static void dispatch_exit_fcache_stats(dcontext_t *dcontext); +#ifndef LINUX_KERNEL static void handle_post_system_call(dcontext_t *dcontext); +#endif static void handle_special_tag(dcontext_t *dcontext); @@ -135,7 +137,7 @@ d_r_dispatch(dcontext_t *dcontext) fragment_t coarse_f; #ifdef HAVE_TLS -# if defined(UNIX) && defined(X86) +# if defined(UNIX) && defined(X86) && !defined(LINUX_KERNEL) /* i#2089: the parent of a new thread has TLS in an unstable state * and needs to restore it prior to invoking get_thread_private_dcontext(). */ @@ -470,7 +472,7 @@ dispatch_enter_fcache(dcontext_t *dcontext, fragment_t *targetf) check_filter("linux.thread;linux.clone", get_short_name(get_application_name()))); #endif -#if defined(UNIX) && !defined(DGC_DIAGNOSTICS) && defined(X86) +#if defined(UNIX) && !defined(DGC_DIAGNOSTICS) && defined(X86) && !defined(LINUX_KERNEL) /* i#107: handle segment register usage conflicts between app and dr: * if the target fragment has an instr that updates the segment selector, * update the corresponding information maintained by DR. @@ -502,7 +504,7 @@ dispatch_enter_fcache(dcontext_t *dcontext, fragment_t *targetf) PC_AS_JMP_TGT(FRAG_ISA_MODE(targetf->flags), FCACHE_ENTRY_PC(targetf)) #endif ); -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) if (dcontext->signals_pending > 0) { /* i#2019: the fcache_enter generated code starts with a check for pending * signals, allowing the signal handling code to simply queue signals that @@ -731,6 +733,7 @@ dispatch_enter_native(dcontext_t *dcontext) ASSERT_NOT_REACHED(); } +#ifndef LINUX_KERNEL static void set_next_tag_to_prior_syscall(dcontext_t *dcontext) { @@ -748,6 +751,7 @@ set_next_tag_to_prior_syscall(dcontext_t *dcontext) : FRAG_ISA_MODE(dcontext->last_fragment->flags)); ASSERT(is_syscall_at_pc(dcontext, dcontext->next_tag)); } +#endif static void dispatch_enter_dynamorio(dcontext_t *dcontext) @@ -820,6 +824,10 @@ dispatch_enter_dynamorio(dcontext_t *dcontext) */ if (wherewasi == DR_WHERE_APP) { /* first entrance */ +#ifdef LINUX_KERNEL + ASSERT(dcontext->last_exit == get_starting_linkstub() || + IS_KERNEL_ENTRY_LINKSTUB(dcontext->last_exit)); +#else if (dcontext->last_exit == get_syscall_linkstub()) { /* i#813: the app hit our post-sysenter hook while native. * XXX: should we try to process ni syscalls here? But we're only @@ -837,6 +845,7 @@ dispatch_enter_dynamorio(dcontext_t *dcontext) /* new thread */ IF_WINDOWS_ELSE_0(dcontext->last_exit == get_asynch_linkstub())); } +#endif } else { /* MUST be set, if only to a fake linkstub_t */ ASSERT(dcontext->last_exit != NULL); @@ -899,8 +908,10 @@ dispatch_enter_dynamorio(dcontext_t *dcontext) KSTART_DC(dcontext, dispatch_num_exits); if (wherewasi != DR_WHERE_APP) { /* if not first entrance */ +#ifndef LINUX_KERNEL if (get_at_syscall(dcontext)) handle_post_system_call(dcontext); +#endif #ifdef X86 /* If the next basic block starts at a debug register value, @@ -913,6 +924,7 @@ dispatch_enter_dynamorio(dcontext_t *dcontext) } #endif +#ifndef LINUX_KERNEL /* A non-ignorable syscall or cb return ending a bb must be acted on * We do it here to avoid becoming couldbelinking twice. * @@ -931,6 +943,7 @@ dispatch_enter_dynamorio(dcontext_t *dcontext) */ } } +#endif #ifdef WINDOWS else if (TEST(LINK_CALLBACK_RETURN, dcontext->last_exit->flags)) { handle_callback_return(dcontext); @@ -992,7 +1005,7 @@ dispatch_enter_dynamorio(dcontext_t *dcontext) ASSERT_NOT_REACHED(); } else if (dcontext->upcontext.upcontext.exit_reason == EXIT_REASON_RSEQ_ABORT) { -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) rseq_process_native_abort(dcontext); #else ASSERT_NOT_REACHED(); @@ -1190,6 +1203,12 @@ dispatch_exit_fcache(dcontext_t *dcontext) * we'll keep coming back here on hits in the outdated table. */ fragment_update_ibl_tables(dcontext); + } else if (dcontext->last_exit == get_ibl_unlinked_found_linkstub()) { + fragment_t *ibl_target, wrapper; + ASSERT(in_fcache(dcontext->next_tag)); + ibl_target = fragment_pclookup(dcontext, dcontext->next_tag, &wrapper); + ASSERT(ibl_target != NULL); + dcontext->next_tag = ibl_target->tag; } /* ref bug 2323, we need monitor to restore last fragment now, @@ -1225,7 +1244,7 @@ dispatch_exit_fcache(dcontext_t *dcontext) } #endif -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) if (dcontext->signals_pending != 0) { /* XXX: We can overflow the app stack if we stack up too many signals * by interrupting prev handlers -- exacerbated by RAC lack of @@ -1390,6 +1409,11 @@ dispatch_exit_fcache_stats(dcontext_t *dcontext) STATS_INC(num_exits_ibl_deleted); KSWITCH_STOP_NOT_PROPAGATED(fcache_default); return; + } else if (dcontext->last_exit == get_ibl_unlinked_found_linkstub()) { + LOG(THREAD, LOG_DISPATCH, 2, "Exit from interrupt in ibl hit in ibl\n"); + STATS_INC(num_exits_ibl_unlinked_found); + KSWITCH_STOP_NOT_PROPAGATED(fcache_default); + return; } else if (dcontext->last_exit == get_asynch_linkstub()) { LOG(THREAD, LOG_DISPATCH, 2, "Exit from asynch event\n"); STATS_INC(num_exits_asynch); @@ -1793,7 +1817,7 @@ dispatch_exit_fcache_stats(dcontext_t *dcontext) * SYSTEM CALLS */ -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) static void adjust_syscall_continuation(dcontext_t *dcontext) { @@ -1853,6 +1877,7 @@ adjust_syscall_continuation(dcontext_t *dcontext) } #endif +#ifndef LINUX_KERNEL /* used to execute a system call instruction in the code cache * dcontext->next_tag is store elsewhere and restored after the system call * for resumption of execution post-syscall @@ -1867,23 +1892,23 @@ handle_system_call(dcontext_t *dcontext) int sysnum = os_normalized_sysnum((int)MCXT_SYSNUM_REG(mc), NULL, dcontext); app_pc saved_next_tag = dcontext->next_tag; bool repeat = false; -#ifdef WINDOWS +# ifdef WINDOWS /* make sure to ask about syscall before pre_syscall, which will swap new mc in! */ bool use_prev_dcontext = is_cb_return_syscall(dcontext); -#elif defined(X86) +# elif defined(X86) if (TEST(LINK_NI_SYSCALL_INT, dcontext->last_exit->flags)) { LOG(THREAD, LOG_SYSCALLS, 2, "Using do_int_syscall\n"); do_syscall = (app_pc)get_do_int_syscall_entry(dcontext); /* last_exit will be for the syscall so set a flag (could alternatively * set up a separate exit stub but this is simpler) */ dcontext->sys_was_int = true; -# ifdef VMX86_SERVER +# ifdef VMX86_SERVER if (is_vmkuw_sysnum(mc->xax)) { /* Even w/ syscall # shift int80 => ENOSYS */ do_syscall = get_do_vmkuw_syscall_entry(dcontext); LOG(THREAD, LOG_SYSCALLS, 2, "Using do_vmkuw_syscall\n"); } -# endif +# endif } else if (TEST(LINK_SPECIAL_EXIT, dcontext->last_exit->flags)) { if (dcontext->upcontext.upcontext.exit_reason == EXIT_REASON_NI_SYSCALL_INT_0x81) do_syscall = (app_pc)get_do_int81_syscall_entry(dcontext); @@ -1897,7 +1922,7 @@ handle_system_call(dcontext_t *dcontext) dcontext->sys_was_int = false; IF_NOT_X64(IF_VMX86(ASSERT(!is_vmkuw_sysnum(mc->xax)))); } -#endif +# endif /* We invoke here rather than inside pre_syscall() primarily so we can * set use_prev_dcontext(), but also b/c the windows and linux uses @@ -1920,10 +1945,10 @@ handle_system_call(dcontext_t *dcontext) LOG(THREAD, LOG_SYSCALLS, 2, "skipping syscall %d on client request\n", MCXT_SYSNUM_REG(mc)); } -#ifdef WINDOWS +# ifdef WINDOWS /* re-set in case client changed the number */ use_prev_dcontext = is_cb_return_syscall(dcontext); -#endif +# endif /* some syscalls require modifying local memory * XXX: move this unprot down to those syscalls to avoid unprot-prot-unprot-prot @@ -1936,10 +1961,10 @@ handle_system_call(dcontext_t *dcontext) LOG(THREAD, LOG_SYSCALLS, 2, "Entry into do_syscall to execute a non-ignorable system call\n"); -#ifdef SIDELINE +# ifdef SIDELINE /* clear cur-trace field so we don't think cur trace is still running */ sideline_trace = NULL; -#endif +# endif /* our flushing design assumes our syscall handlers are nolinking, * to avoid multiple-flusher deadlocks @@ -1957,14 +1982,14 @@ handle_system_call(dcontext_t *dcontext) */ dcontext->asynch_target = get_fcache_target(dcontext); -#ifdef WINDOWS +# ifdef WINDOWS if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) { /* kernel sends control directly to 0x7ffe0304 so we need * to mangle the return address */ /* Ref case 5461 - edx will become top of stack post-syscall */ ASSERT(get_mcontext(dcontext)->xsp == get_mcontext(dcontext)->xdx); -# ifdef HOT_PATCHING_INTERFACE +# ifdef HOT_PATCHING_INTERFACE /* For hotp_only, vsyscall_syscall_end_pc can be NULL as dr will never * interp a system call. Also, for hotp_only, control can came here * from native only to do a syscall that was hooked. @@ -1972,9 +1997,9 @@ handle_system_call(dcontext_t *dcontext) ASSERT(!DYNAMO_OPTION(hotp_only) || (DYNAMO_OPTION(hotp_only) && dcontext->next_tag == BACK_TO_NATIVE_AFTER_SYSCALL)); -# else +# else ASSERT(vsyscall_syscall_end_pc != NULL || get_os_version() >= WINDOWS_VERSION_8); -# endif +# endif /* NOTE - the stack mangling must match that of intercept_nt_continue() * and shared_syscall as not all routines looking at the stack * differentiate. */ @@ -1982,10 +2007,10 @@ handle_system_call(dcontext_t *dcontext) /* win8 x86 syscalls have inlined sysenter routines */ (get_os_version() >= WINDOWS_VERSION_8 && dcontext->thread_record->under_dynamo_control)) { -# ifdef HOT_PATCHING_INTERFACE +# ifdef HOT_PATCHING_INTERFACE /* Don't expect to be here for -hotp_only */ ASSERT_CURIOSITY(!DYNAMO_OPTION(hotp_only)); -# endif +# endif ASSERT(dcontext->next_tag != BACK_TO_NATIVE_AFTER_SYSCALL); /* currently pc is the ret after sysenter, we need it to be the return point * (the ret after the call to the vsyscall sysenter) @@ -2031,9 +2056,9 @@ handle_system_call(dcontext_t *dcontext) *((app_pc *)get_mcontext(dcontext)->xsp) = after_do_syscall_code(dcontext); } } -#endif +# endif -#if defined(MACOS) && defined(X86) +# if defined(MACOS) && defined(X86) if (get_syscall_method() == SYSCALL_METHOD_SYSENTER && !dcontext->sys_was_int) { /* The kernel returns control to whatever user-mode places in edx. * We want to put this in even if we skip the syscall as we'll still call @@ -2044,12 +2069,12 @@ handle_system_call(dcontext_t *dcontext) dcontext->app_xdx = mc->xdx; mc->xdx = (reg_t)post_sysenter; } -#endif +# endif /* first do the pre-system-call */ if (execute_syscall && pre_system_call(dcontext)) { /* now do the actual syscall instruction */ -#ifdef UNIX +# ifdef UNIX /* XXX: move into some routine inside unix/? * if so, move #include of sys/syscall.h too */ @@ -2078,7 +2103,7 @@ handle_system_call(dcontext_t *dcontext) LOG(THREAD, LOG_SYSCALLS, 3, "for sigreturn, set sys_param1 to " PFX "\n", dcontext->sys_param1); } -#else +# else if (use_prev_dcontext) { /* get the current, but now swapped out, dcontext */ dcontext_t *tmp_dcontext = dcontext; @@ -2095,7 +2120,7 @@ handle_system_call(dcontext_t *dcontext) */ tmp_dcontext->whereami = DR_WHERE_FCACHE; } -#endif +# endif SELF_PROTECT_LOCAL(dcontext, READONLY); @@ -2103,7 +2128,7 @@ handle_system_call(dcontext_t *dcontext) KSTART_DC(dcontext, syscall_fcache); /* stopped in dispatch_exit_fcache_stats */ bool is_ignorable = ignorable_system_call(sysnum, NULL, dcontext); do { -#ifdef UNIX +# ifdef UNIX /* It is difficult to undo some pre-syscall handling, especially for * sigreturn's signal mask and clone syscalls. We go ahead and run the * syscall before we deliver the signal for all non-ignorable syscalls. @@ -2120,21 +2145,21 @@ handle_system_call(dcontext_t *dcontext) !is_ignorable); if (!is_ignorable && dcontext->signals_pending > 0) dcontext->signals_pending = -1; -#endif +# endif enter_fcache(dcontext, (fcache_enter_func_t) /* DEFAULT_ISA_MODE as we want the ISA mode of our gencode */ convert_data_to_function( PC_AS_JMP_TGT(DEFAULT_ISA_MODE, (app_pc)fcache_enter)), PC_AS_JMP_TGT(DEFAULT_ISA_MODE, do_syscall)); -#ifdef UNIX +# ifdef UNIX if (!is_ignorable && dcontext->signals_pending > 0) repeat = true; else break; -#endif +# endif } while (repeat); -#ifdef UNIX +# if defined(UNIX) && !defined(LINUX_KERNEL) if (dcontext->signals_pending != 0) { /* i#2019: see comments in dispatch_enter_fcache() */ KSTOP(syscall_fcache); @@ -2154,7 +2179,7 @@ handle_system_call(dcontext_t *dcontext) trace_abort(dcontext); receive_pending_signal(dcontext); } else -#endif +# endif /* will handle post processing in handle_post_system_call */ ASSERT_NOT_REACHED(); } else { @@ -2165,7 +2190,7 @@ handle_system_call(dcontext_t *dcontext) if (execute_syscall) { instrument_post_syscall(dcontext, dcontext->sys_num); } -#ifdef WINDOWS +# ifdef WINDOWS if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) { /* decided to skip syscall -- pop retaddr, restore sysenter storage * (if applicable) and set next target */ @@ -2181,14 +2206,16 @@ handle_system_call(dcontext_t *dcontext) */ get_mcontext(dcontext)->xsp -= XSP_SZ; } -#else +# else adjust_syscall_continuation(dcontext); set_fcache_target(dcontext, dcontext->asynch_target); -#endif +# endif } SELF_PROTECT_LOCAL(dcontext, READONLY); } +#endif +#ifndef LINUX_KERNEL static void handle_post_system_call(dcontext_t *dcontext) { @@ -2203,7 +2230,7 @@ handle_post_system_call(dcontext_t *dcontext) /* some syscalls require modifying local memory */ SELF_PROTECT_LOCAL(dcontext, WRITABLE); -#ifdef UNIX +# ifdef UNIX /* restore mcontext values prior to invoking instrument_post_syscall() */ if (was_sigreturn_syscall(dcontext)) { /* restore app xax/r0 */ @@ -2212,16 +2239,16 @@ handle_post_system_call(dcontext_t *dcontext) dcontext->sys_param1, dcontext->asynch_target); /* XXX i#3544: Check if this is a proper register to use */ mc->IF_X86_ELSE(xax, IF_RISCV64_ELSE(a0, r0)) = dcontext->sys_param1; -# ifdef MACOS +# ifdef MACOS /* We need to skip the use app_xdx, as we've changed the context. * We can't just set app_xdx from handle_sigreturn() as the * pre-sysenter code clobbers app_xdx, and we want to handle * a failed SYS_sigreturn. */ skip_adjust = true; -# endif +# endif } -#endif +# endif /* i#1661: ensure we set the right pc for dr_get_mcontext() */ get_mcontext(dcontext)->pc = dcontext->asynch_target; @@ -2231,7 +2258,7 @@ handle_post_system_call(dcontext_t *dcontext) /* XXX: need to handle syscall failure -- those that clobbered asynch_target * need to restore it to its previous value, which has to be stored somewhere! */ -#ifdef WINDOWS +# ifdef WINDOWS if (DYNAMO_OPTION(sygate_sysenter) && get_syscall_method() == SYSCALL_METHOD_SYSENTER) { /* restore sysenter_storage, note stack was popped twice for @@ -2239,25 +2266,26 @@ handle_post_system_call(dcontext_t *dcontext) get_mcontext(dcontext)->xsp -= XSP_SZ; *((app_pc *)get_mcontext(dcontext)->xsp) = dcontext->sysenter_storage; } -#else +# else if (!skip_adjust) adjust_syscall_continuation(dcontext); -#endif +# endif set_fcache_target(dcontext, dcontext->asynch_target); -#ifdef WINDOWS +# ifdef WINDOWS /* We no longer need asynch_target so zero it out. Other pieces of DR * -- callback & APC handling, detach -- test asynch_target to determine * where the next app pc to execute is stored. If asynch_target != 0, * it holds the value, else it's in the esi slot. */ dcontext->asynch_target = 0; -#endif +# endif LOG(THREAD, LOG_SYSCALLS, 3, "finished handling system call\n"); SELF_PROTECT_LOCAL(dcontext, READONLY); /* caller will go back to couldbelinking status */ } +#endif /* !LINUX_KERNEL */ #ifdef WINDOWS /* in callback.c */ diff --git a/core/drk.mk b/core/drk.mk index fc14357bc..0143d2c45 100644 --- a/core/drk.mk +++ b/core/drk.mk @@ -10,7 +10,7 @@ API_INCLUDE_DIR ?= lib/include .PHONY: test DR_CORE_DIR := $(shell pwd) -DR_INCLUDE_FLAGS := -DDR_REG_ENUM_COMPATIBILITY -DLINUX_KERNEL\ +DR_INCLUDE_FLAGS := -DDR_REG_ENUM_COMPATIBILITY -DLINUX_KERNEL -DDYNAMORIO_INTERNAL -DDR_DO_NOT_DEFINE_MAX_MIN\ -I$(DR_CORE_DIR)/../build\ -I$(DR_CORE_DIR)/$(API_INCLUDE_DIR)\ -I$(DR_CORE_DIR)\ diff --git a/core/drlibc/drlibc.h b/core/drlibc/drlibc.h index 51e46df98..1f24d716e 100644 --- a/core/drlibc/drlibc.h +++ b/core/drlibc/drlibc.h @@ -41,12 +41,14 @@ #define _DR_LIBC_H_ #ifdef UNIX +# ifndef LINUX_KERNEL /* _LARGEFILE64_SOURCE should make libc struct match kernel. */ -# ifndef _LARGEFILE64_SOURCE -# define _LARGEFILE64_SOURCE +# ifndef _LARGEFILE64_SOURCE +# define _LARGEFILE64_SOURCE +# endif +# include +# include # endif -# include -# include #endif #if defined(MACOS) && defined(AARCH64) @@ -147,8 +149,10 @@ find_script_interpreter(DR_PARAM_OUT script_interpreter_t *result, DR_PARAM_IN const char *fname, ssize_t (*reader)(const char *pathname, void *buf, size_t count)); +# ifndef LINUX_KERNEL ptr_int_t dr_stat_syscall(const char *fname, struct stat64 *st); +# endif #endif /* UNIX */ #if defined(WINDOWS) && !defined(X64) diff --git a/core/dynamo.c b/core/dynamo.c index 5e67f08a3..5a4eb3351 100644 --- a/core/dynamo.c +++ b/core/dynamo.c @@ -67,6 +67,13 @@ # include "annotations.h" #endif +#ifdef LINUX_KERNEL +# include "barrier.h" +# include "dynamorio_module_interface.h" +# include "kernel_interface.h" +# include "msr.h" +#endif + #ifdef WINDOWS /* for close handle, duplicate handle, free memory and constants associated with them */ @@ -386,6 +393,99 @@ get_dr_stats(void) return d_r_stats; } +#ifdef LINUX_KERNEL +/* Init barriers */ +static barrier_t before_dynamo_app_init; +static barrier_t after_dynamo_app_init; +static barrier_t before_dynamorio_app_take_over; + +/* Exit barriers */ +static barrier_t main_thread_exit; +static barrier_t other_threads_exit; + +void +dr_pre_smp_init(dr_exports_t *exports, const char *options) +{ + exports->stats_data = &nonshared_stats; + exports->stats_size = sizeof(dr_statistics_t); + kernel_setenv(DYNAMORIO_VAR_OPTIONS, options); + barrier_init(&before_dynamo_app_init, get_num_processors()); + barrier_init(&after_dynamo_app_init, get_num_processors()); + barrier_init(&before_dynamorio_app_take_over, get_num_processors()); + barrier_init(&main_thread_exit, get_num_processors()); + barrier_init(&other_threads_exit, get_num_processors()); +} + +void +cpu_exports_init(dcontext_t *dcontext, dr_cpu_exports_t *exports) +{ +# ifdef KSTATS + if (DYNAMO_OPTION(kstats)) { + exports->kstats_data = &dcontext->thread_kstats->vars_kstats; + exports->kstats_size = sizeof(dcontext->thread_kstats->vars_kstats); + } else { +# endif + exports->kstats_data = NULL; + exports->kstats_size = 0; +# ifdef KSTATS + } +# endif +} + +void +dr_smp_init(dr_cpu_exports_t *exports) +{ + /* Interrupts are disabled at this point. */ + if (barrier_wait(&before_dynamo_app_init)) { + /* barrier_wait only returns true for one thread. We designate this as + * the main thread for initilization. + */ + dynamorio_app_init(); + barrier_wait(&after_dynamo_app_init); + } else { + barrier_wait(&after_dynamo_app_init); + /* The following three lines are called for the main thread by + * dynamorio_app_init. + */ + dynamo_thread_init(NULL, NULL, NULL, false); + ENTERING_DR(); + // instrument_thread_init(get_thread_private_dcontext(), false, false); + } + os_warm_fcache(get_thread_private_dcontext()); + cpu_exports_init(get_thread_private_dcontext(), exports); + barrier_wait(&before_dynamorio_app_take_over); + /* dynamorio_app_take_over needs to be called at this point from another + * module. Otherwise, we would try to cache some DR instructions (i.e., this + * function's implicit return), which DR isn't designed to handle. See + * dynamorio_app_take_over() in dynamorio_module_interface.h. */ +} + +void +dr_smp_exit(void) +{ + DEBUG_DECLARE(int res;) + static bool main_thread_exited = false; + if (barrier_wait(&main_thread_exit)) { + barrier_wait(&other_threads_exit); + barrier_destroy(&before_dynamo_app_init); + barrier_destroy(&after_dynamo_app_init); + barrier_destroy(&before_dynamorio_app_take_over); + barrier_destroy(&main_thread_exit); + barrier_destroy(&other_threads_exit); + IF_DEBUG(res =) dr_app_cleanup(); + ASSERT(res == SUCCESS); + asm volatile("mfence"); + main_thread_exited = true; + } else { + IF_DEBUG(res =) dynamo_thread_exit(); + ASSERT(res == SUCCESS); + barrier_wait(&other_threads_exit); + } + while (!main_thread_exited) { + } +} +#endif + /* initialize per-process dynamo state; this must be called before any * threads are created and before any other API calls are made; * returns zero on success, non-zero on failure @@ -407,7 +507,7 @@ dynamorio_app_init_part_one_options(void) } } else /* we do enter if nullcalls is on */ { -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) os_page_size_init((const char **)our_environ, is_our_environ_followed_by_auxv()); #endif #ifdef WINDOWS @@ -417,10 +517,11 @@ dynamorio_app_init_part_one_options(void) /* avoid time() for libc independence */ DODEBUG(starttime = query_time_seconds();); -#ifdef UNIX +#ifndef LINUX_KERNEL +# ifdef UNIX if (getenv(DYNAMORIO_VAR_EXECVE) != NULL) { post_execve = true; -# ifdef VMX86_SERVER +# ifdef VMX86_SERVER /* PR 458917: our gdt slot was not cleared on exec so we need to * clear it now to ensure we don't leak it and eventually run out of * slots. We could alternatively call os_tls_exit() prior to @@ -429,7 +530,7 @@ dynamorio_app_init_part_one_options(void) * TLS index we had. */ os_tls_pre_init(atoi(getenv(DYNAMORIO_VAR_EXECVE))); -# endif +# endif /* important to remove it, don't want to propagate to forked children, etc. */ /* i#909: unsetenv is unsafe as it messes up auxv access, so we disable */ disable_env(DYNAMORIO_VAR_EXECVE); @@ -437,6 +538,7 @@ dynamorio_app_init_part_one_options(void) ASSERT(getenv(DYNAMORIO_VAR_EXECVE) == NULL); } else post_execve = false; +# endif #endif /* default non-zero dynamo settings (options structure is @@ -789,7 +891,7 @@ dynamorio_app_init_part_two_finalize(void) return SUCCESS; } -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) void dynamorio_fork_init(dcontext_t *dcontext) { @@ -896,7 +998,7 @@ dynamorio_fork_init(dcontext_t *dcontext) instrument_fork_init(dcontext); } } -#endif /* UNIX */ +#endif /* UNIX && !LINUX_KERNEL */ /* To make DynamoRIO useful as a library for a standalone client * application (as opposed to a client library that works with @@ -920,7 +1022,7 @@ standalone_init(void) /* avoid issues w/ GLOBAL_DCONTEXT instead of thread dcontext */ dynamo_options.deadlock_avoidance = false; #endif -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) os_page_size_init((const char **)our_environ, is_our_environ_followed_by_auxv()); #endif #ifdef WINDOWS @@ -1777,6 +1879,9 @@ initialize_dynamo_context(dcontext_t *dcontext) dcontext->initialized = true; dcontext->whereami = DR_WHERE_APP; dcontext->next_tag = NULL; +#ifdef LINUX_KERNEL + dcontext->next_app_tag = NULL; +#endif dcontext->native_exec_postsyscall = NULL; memset(dcontext->native_retstack, 0, sizeof(dcontext->native_retstack)); dcontext->native_retstack_cur = 0; diff --git a/core/exports.py b/core/exports.py index cf3d3c2d3..63d8dd382 100755 --- a/core/exports.py +++ b/core/exports.py @@ -17,29 +17,461 @@ def check_open(args): def excluded_functions(): return { + "INSTR_CREATE_nop2byte_reg", + "INSTR_CREATE_nop3byte_reg", "__attribute__", - "dr_init", - "dr_smp_exit", + "decode_memory_reference_size", + "disassemble", + "disassemble_from_copy", + "disassemble_set_syntax", + "disassemble_to_buffer", + "disassemble_with_info", + "dr_add_prefixes_to_basic_blocks", + "dr_annotation_pass_pc", + "dr_annotation_register_call", + "dr_annotation_register_return", + "dr_annotation_register_valgrind", + "dr_annotation_set_return_value", + "dr_annotation_unregister_call", + "dr_annotation_unregister_return", + "dr_annotation_unregister_valgrind", + "dr_app_handle_mbr_target", + "dr_client_iterator_hasnext", + "dr_client_iterator_next", + "dr_client_iterator_next_ex", + "dr_client_iterator_start", + "dr_client_iterator_stop", + "dr_client_main", + "dr_client_thread_set_suspendable", + "dr_config_status_code_to_string", + "dr_copy_module_data", + "dr_create_client_thread", + "dr_free_module_data", "dr_get_app_PEB", - "instr_exit_stub_code", - "dr_syscall_get_result", + "dr_get_client_info", + "dr_get_client_info_ex", + "dr_get_config_dir", + "dr_get_isa_mode", + "dr_get_main_module", + "dr_get_proc_address", + "dr_get_proc_address_ex", + "dr_get_stderr_file", + "dr_get_stdin_file", + "dr_get_stdout_file", + "dr_get_vector_length", + "dr_init", + "dr_inject_get_image_name", + "dr_inject_get_process_id", + "dr_inject_prepare_new_process_group", + "dr_inject_prepare_to_attach", + "dr_inject_prepare_to_exec", + "dr_inject_prepare_to_ptrace", + "dr_inject_print_stats", + "dr_inject_process_create", + "dr_inject_process_exit", + "dr_inject_process_inject", + "dr_inject_process_run", + "dr_inject_using_debug_key", + "dr_inject_wait_for_child", + "dr_invoke_syscall_as_app", "dr_is_wow64", - "dr_create_client_thread", + "dr_lookup_module", + "dr_lookup_module_by_name", + "dr_module_contains_addr", + "dr_module_import_iterator_hasnext", + "dr_module_import_iterator_next", + "dr_module_import_iterator_start", + "dr_module_import_iterator_stop", + "dr_module_iterator_hasnext", + "dr_module_iterator_next", + "dr_module_iterator_start", + "dr_module_iterator_stop", + "dr_module_preferred_name", + "dr_module_set_should_instrument", + "dr_module_should_instrument", + "dr_nudge_pid", + "dr_num_registered_clients", + "dr_process_is_registered", + "dr_raw_brk", + "dr_raw_mremap", + "dr_register_client", + "dr_register_client_ex", + "dr_register_inject_paths", + "dr_register_kernel_xfer_event", + "dr_register_process", + "dr_set_isa_mode", + "dr_set_vector_length", + "dr_smp_exit", + "dr_symbol_export_iterator_hasnext", + "dr_symbol_export_iterator_next", + "dr_symbol_export_iterator_start", + "dr_symbol_export_iterator_stop", + "dr_symbol_import_iterator_hasnext", + "dr_symbol_import_iterator_next", + "dr_symbol_import_iterator_start", + "dr_symbol_import_iterator_stop", "dr_syscall_get_param", - "instr_branch_targets_prefix", - "INSTR_CREATE_nop2byte_reg", - "dr_get_stdin_file", - "dr_add_prefixes_to_basic_blocks", + "dr_syscall_get_result", + "dr_syscall_get_result_ex", + "dr_syscall_invoke_another", + "dr_syscall_set_param", + "dr_syscall_set_result", + "dr_syscall_set_result_ex", + "dr_syscall_set_sysnum", + "dr_unregister_client", + "dr_unregister_kernel_xfer_event", + "dr_unregister_process", + "drfront_access", + "drfront_appdata_logdir", + "drfront_bufprint", + "drfront_char_to_tchar", + "drfront_cleanup_args", + "drfront_convert_args", + "drfront_create_dir", + "drfront_dir_exists", + "drfront_dir_try_writable", + "drfront_fetch_module_symbols", + "drfront_get_absolute_path", + "drfront_get_app_full_path", + "drfront_get_env_var", + "drfront_is_64bit_app", + "drfront_is_graphical_app", + "drfront_remove_dir", + "drfront_searchenv", + "drfront_set_client_symbol_search_path", + "drfront_set_symbol_search_path", + "drfront_set_verbose", + "drfront_string_replace_character", + "drfront_string_replace_character_wide", + "drfront_sym_exit", + "drfront_sym_init", + "drfront_tchar_to_char", + "drfront_tchar_to_char_size_needed", + "get_register_name", + "instr_allocate_raw_bits", "instr_branch_set_prefix_target", + "instr_branch_targets_prefix", + "instr_build", + "instr_build_bits", + "instr_clear_label_callback", + "instr_clone", + "instr_compute_address", + "instr_compute_address_ex", + "instr_compute_address_ex_pos", + "instr_convert_short_meta_jmp_to_long", + "instr_convert_to_isa_regdeps", + "instr_create", + "instr_create_0dst_0src", + "instr_create_0dst_1src", + "instr_create_0dst_2src", + "instr_create_0dst_3src", + "instr_create_0dst_4src", + "instr_create_1dst_0src", + "instr_create_1dst_1src", + "instr_create_1dst_2src", + "instr_create_1dst_3src", + "instr_create_1dst_4src", + "instr_create_1dst_5src", + "instr_create_1dst_6src", + "instr_create_2dst_0src", + "instr_create_2dst_1src", + "instr_create_2dst_2src", + "instr_create_2dst_3src", + "instr_create_2dst_4src", + "instr_create_2dst_5src", + "instr_create_3dst_0src", + "instr_create_3dst_1src", + "instr_create_3dst_2src", + "instr_create_3dst_3src", + "instr_create_3dst_4src", + "instr_create_3dst_5src", + "instr_create_3dst_6src", + "instr_create_4dst_1src", + "instr_create_4dst_2src", + "instr_create_4dst_3src", + "instr_create_4dst_4src", + "instr_create_4dst_5src", + "instr_create_4dst_6src", + "instr_create_4dst_7src", + "instr_create_5dst_3src", + "instr_create_5dst_4src", + "instr_create_5dst_5src", + "instr_create_5dst_8src", + "instr_create_Ndst_Msrc_vardst", + "instr_create_Ndst_Msrc_varsrc", + "instr_destroy", + "instr_disassemble", + "instr_disassemble_to_buffer", + "instr_encode", + "instr_encode_to_copy", + "instr_exit_stub_code", + "instr_free", + "instr_free_raw_bits", + "instr_from_noalloc", + "instr_get_app_pc", + "instr_get_arith_flags", + "instr_get_branch_target_pc", + "instr_get_category", + "instr_get_category_name", + "instr_get_dst", + "instr_get_eflags", + "instr_get_interrupt_number", + "instr_get_isa_mode", + "instr_get_label_data_area", + "instr_get_next", + "instr_get_next_app", + "instr_get_note", + "instr_get_offset", + "instr_get_opcode", + "instr_get_opcode_eflags", + "instr_get_operation_size", + "instr_get_predicate", + "instr_get_prefix_flag", + "instr_get_prefixes", + "instr_get_prev", + "instr_get_prev_app", + "instr_get_raw_bits", + "instr_get_raw_byte", + "instr_get_raw_word", + "instr_get_rel_addr_dst_idx", + "instr_get_rel_addr_src_idx", + "instr_get_rel_addr_target", + "instr_get_rel_data_or_instr_target", + "instr_get_src", + "instr_get_target", + "instr_has_allocated_bits", + "instr_has_encoding_hint", + "instr_has_rel_addr_reference", + "instr_init", + "instr_is_app", + "instr_is_call", + "instr_is_cbr", + "instr_is_cti", + "instr_is_encoding_possible", + "instr_is_exit_cti", + "instr_is_label", + "instr_is_mbr", + "instr_is_meta", + "instr_is_meta_may_fault", + "instr_is_opnd_store_source", + "instr_is_predicated", + "instr_is_reg_spill_or_restore", + "instr_is_ubr", + "instr_is_xrstor", + "instr_is_xsave", + "instr_length", + "instr_make_persistent", + "instr_mem_usage", + "instr_memory_reference_size", + "instr_needs_encoding", + "instr_noalloc_init", + "instr_num_dsts", + "instr_num_memory_read_access", + "instr_num_memory_write_access", + "instr_num_srcs", + "instr_ok_to_emit", + "instr_ok_to_mangle", + "instr_opcode_valid", + "instr_operands_valid", + "instr_raw_bits_valid", + "instr_reads_from_exact_reg", + "instr_reads_from_reg", + "instr_reads_memory", + "instr_reg_in_dst", + "instr_reg_in_src", + "instr_remove_dsts", + "instr_remove_srcs", + "instr_replace_reg_resize", + "instr_replace_src_opnd", + "instr_reset", + "instr_reuse", + "instr_same", + "instr_set_app", + "instr_set_branch_target_pc", + "instr_set_category", + "instr_set_dst", + "instr_set_encoding_hint", "instr_set_exit_stub_code", + "instr_set_label_callback", + "instr_set_meta", + "instr_set_meta_may_fault", + "instr_set_meta_no_translation", + "instr_set_next", + "instr_set_note", + "instr_set_num_opnds", + "instr_set_ok_to_emit", + "instr_set_ok_to_mangle", + "instr_set_opcode", + "instr_set_operands_valid", + "instr_set_operation_size", + "instr_set_predicate", + "instr_set_prefix_flag", + "instr_set_prefixes", + "instr_set_prev", + "instr_set_raw_bits", + "instr_set_raw_bits_valid", + "instr_set_raw_byte", + "instr_set_raw_bytes", + "instr_set_raw_word", + "instr_set_src", + "instr_set_target", + "instr_set_translation", + "instr_shrink_to_16_bits", + "instr_shrink_to_32_bits", + "instr_uses_fp_reg", + "instr_uses_reg", + "instr_valid", + "instr_writes_memory", + "instr_writes_to_exact_reg", + "instr_writes_to_reg", + "instr_zeroes_ymmh", + "instr_zeroes_zmmh", + "instrlist_disassemble", + "opnd_add_flags", + "opnd_compute_address", + "opnd_create_abs_addr", + "opnd_create_base_disp", + "opnd_create_base_disp_ex", + "opnd_create_far_abs_addr", + "opnd_create_far_base_disp", + "opnd_create_far_base_disp_ex", + "opnd_create_far_instr", + "opnd_create_far_pc", + "opnd_create_far_rel_addr", + "opnd_create_immed_double", + "opnd_create_immed_float", + "opnd_create_immed_int", + "opnd_create_immed_int64", + "opnd_create_immed_uint", + "opnd_create_increment_reg", + "opnd_create_instr", + "opnd_create_instr_ex", + "opnd_create_mem_instr", + "opnd_create_null", + "opnd_create_pc", + "opnd_create_reg", + "opnd_create_reg_element_vector", + "opnd_create_reg_ex", + "opnd_create_reg_partial", + "opnd_create_rel_addr", + "opnd_defines_use", + "opnd_disassemble", + "opnd_disassemble_to_buffer", + "opnd_get_addr", + "opnd_get_base", + "opnd_get_disp", + "opnd_get_flags", + "opnd_get_immed_double", "opnd_get_immed_float", - "dr_client_thread_set_suspendable", - "dr_syscall_invoke_another", - "dr_get_stdout_file", - "dr_syscall_set_sysnum", - "dr_syscall_set_result", - "dr_syscall_set_param", - "dr_get_stderr_file", + "opnd_get_immed_int", + "opnd_get_immed_int64", + "opnd_get_index", + "opnd_get_instr", + "opnd_get_mem_instr_disp", + "opnd_get_pc", + "opnd_get_reg", + "opnd_get_reg_used", + "opnd_get_scale", + "opnd_get_segment", + "opnd_get_segment_selector", + "opnd_get_shift", + "opnd_get_size", + "opnd_get_vector_element_size", + "opnd_inc_reg", + "opnd_invert_immed_int", + "opnd_is_abs_addr", + "opnd_is_base_disp", + "opnd_is_disp_encode_zero", + "opnd_is_disp_force_full", + "opnd_is_disp_short_addr", + "opnd_is_element_vector_reg", + "opnd_is_far_abs_addr", + "opnd_is_far_base_disp", + "opnd_is_far_instr", + "opnd_is_far_memory_reference", + "opnd_is_far_pc", + "opnd_is_far_rel_addr", + "opnd_is_governing", + "opnd_is_immed", + "opnd_is_immed_float", + "opnd_is_immed_int", + "opnd_is_immed_int64", + "opnd_is_instr", + "opnd_is_mem_instr", + "opnd_is_memory_reference", + "opnd_is_near_abs_addr", + "opnd_is_near_base_disp", + "opnd_is_near_instr", + "opnd_is_near_memory_reference", + "opnd_is_near_pc", + "opnd_is_near_rel_addr", + "opnd_is_null", + "opnd_is_pc", + "opnd_is_predicate_merge", + "opnd_is_predicate_reg", + "opnd_is_predicate_zero", + "opnd_is_reg", + "opnd_is_reg_32bit", + "opnd_is_reg_64bit", + "opnd_is_reg_partial", + "opnd_is_reg_pointer_sized", + "opnd_is_rel_addr", + "opnd_is_vector_base_disp", + "opnd_is_vsib", + "opnd_num_regs_used", + "opnd_replace_reg", + "opnd_replace_reg_resize", + "opnd_same", + "opnd_same_address", + "opnd_set_disp", + "opnd_set_disp_ex", + "opnd_set_flags", + "opnd_set_size", + "opnd_share_reg", + "opnd_shrink_to_16_bits", + "opnd_shrink_to_32_bits", + "opnd_size_from_bytes", + "opnd_size_in_bits", + "opnd_size_in_bytes", + "opnd_uses_reg", + "proc_bump_to_end_of_cache_line", + "proc_get_L1_dcache_size", + "proc_get_L1_icache_size", + "proc_get_L2_cache_size", + "proc_get_all_feature_bits", + "proc_get_brand_string", + "proc_get_cache_line_size", + "proc_get_cache_size_str", + "proc_get_containing_page", + "proc_get_family", + "proc_get_model", + "proc_get_stepping", + "proc_get_type", + "proc_get_vendor", + "proc_is_cache_aligned", + "proc_set_vendor", + "reg_32_to_16", + "reg_32_to_64", + "reg_32_to_8", + "reg_32_to_opsz", + "reg_64_to_32", + "reg_get_bits", + "reg_get_size", + "reg_get_value", + "reg_get_value_ex", + "reg_is_32bit", + "reg_is_64bit", + "reg_is_avx512", + "reg_is_avx512_extended", + "reg_is_extended", + "reg_is_pointer_sized", + "reg_is_stolen", + "reg_overlap", + "reg_parameter_num", + "reg_resize_to_opsz", + "reg_set_value", + "reg_set_value_ex", + "reg_to_pointer_sized", } @@ -73,7 +505,7 @@ def main(): if "aarch64" in path or "arm" in path: continue code = check_open( - f"cpp -DX86_64 -DLINUX -DLINUX_KERNEL -Ilib/include -include dr_api.h {path}".split() + f"cpp -DX86_64 -DLINUX -Ilib/include -include dr_api.h {path}".split() ) for match in re.findall( r"^\s*(?:[a-zA-Z0-9_]+\s+)*([a-zA-Z0-9_]+)\(", code, re.MULTILINE @@ -85,7 +517,7 @@ def main(): functions.difference_update(excluded_functions()) functions.update(extra_symbols()) print( - f"/* This file was automatically generated by {os.path.abspath(sys.argv[0])}. */" + f"/* This file was automatically generated by {os.path.basename(sys.argv[0])}. */" ) print("#include ") for function in functions: diff --git a/core/fcache.c b/core/fcache.c index 945235452..9fce38114 100644 --- a/core/fcache.c +++ b/core/fcache.c @@ -47,8 +47,8 @@ #ifdef HOT_PATCHING_INTERFACE # include "hotpatch.h" #endif -#include /* for offsetof */ -#include /* for UCHAR_MAX */ +#include "stddef_wrapper.h" /* for offsetof */ +#include "limits_wrapper.h" /* for UCHAR_MAX */ #include "perscache.h" #include "synch.h" #include "instrument.h" @@ -764,7 +764,7 @@ fcache_free_unit(dcontext_t *dcontext, fcache_unit_t *unit, bool dealloc_or_reus * options, returns true if modified the value of any options to make them * compatible. This is called while the options are writable. */ bool -fcache_check_option_compatibility() +fcache_check_option_compatibility(void) { bool ret = false; uint i; @@ -4229,7 +4229,7 @@ fcache_reset_all_caches_proactively(uint target) LOG(GLOBAL, LOG_CACHE, 2, "fcache_reset_all_caches_proactively: walking the threads\n"); char buf[16]; - snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), "%d", + snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), SSZFMT, GLOBAL_STAT(num_bbs) + GLOBAL_STAT(num_traces)); NULL_TERMINATE_BUFFER(buf); SYSLOG(SYSLOG_INFORMATION, INFO_RESET_IN_PROGRESS, 3, buf, get_application_name(), diff --git a/core/fragment.c b/core/fragment.c index a8b5680fb..8703d68e9 100644 --- a/core/fragment.c +++ b/core/fragment.c @@ -46,8 +46,8 @@ #include "emit.h" #include "monitor.h" #include "instrument.h" -#include /* for offsetof */ -#include /* UINT_MAX */ +#include "stddef_wrapper.h" /* for offsetof */ +#include "limits_wrapper.h" /* UINT_MAX */ #include "perscache.h" #include "synch.h" #ifdef UNIX @@ -3036,7 +3036,7 @@ fragment_delete(dcontext_t *dcontext, fragment_t *f, uint actions) release_recursive_lock(&change_linking_lock); } -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) if (TEST(FRAG_HAS_RSEQ_ENDPOINT, f->flags)) rseq_remove_fragment(dcontext, f); #endif @@ -4289,6 +4289,8 @@ fragment_add_ibl_target(dcontext_t *dcontext, app_pc tag, ibl_branch_type_t bran } else if (!INTERNAL_OPTION(link_ibl)) { reason = "-no_link_ibl prevents ibl"; STATS_INC(num_ibt_exit_nolink); + } else if (dcontext->last_exit == get_ibl_unlinked_found_linkstub()) { + reason = "IBL routine unlinked for interrupt."; } else if (DYNAMO_OPTION(disable_traces) && !TEST(FRAG_LINKED_OUTGOING, dcontext->last_fragment->flags)) { reason = "IBL fragment unlinked in signal handler"; @@ -5430,7 +5432,7 @@ check_flush_queue(dcontext_t *dcontext, fragment_t *was_I_flushed) ATOMIC_4BYTE_ALIGNED_READ(&flushtime_global, &local_flushtime_global); if (DYNAMO_OPTION(shared_deletion) && pt->flushtime_last_update < local_flushtime_global) { -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) rseq_shared_fragment_flushtime_update(dcontext); #endif /* dec ref count on any pending shared areas */ diff --git a/core/globals.h b/core/globals.h index 60446e069..1a0320064 100644 --- a/core/globals.h +++ b/core/globals.h @@ -95,8 +95,10 @@ /* currently we always export statistics structure */ #define DYNAMORIO_STATS_EXPORTS 1 -#ifdef WINDOWS +#if defined(WINDOWS) # define DYNAMORIO_EXPORT __declspec(dllexport) +#elif defined(LINUX_KERNEL) +# define DYNAMORIO_EXPORT /* nothing */ #elif defined(USE_VISIBILITY_ATTRIBUTES) /* PR 262804: we use "protected" instead of "default" to ensure our * own uses won't be preempted. Note that for DR_APP_API in @@ -154,8 +156,10 @@ #define INLINE_ONCE inline -#include -#include +#ifndef LINUX_KERNEL +# include +# include +#endif /* N.B.: some of these typedefs and defines are duplicated in * lib/globals_shared.h! @@ -186,7 +190,8 @@ typedef HANDLE file_t; # if defined(MACOS) || defined(ANDROID) typedef unsigned long ulong; # endif -# include /* for wait */ +# include "stddef_wrapper.h" /* for wchar_t */ +# include "types_wrapper.h" /* for wait */ # define DIRSEP '/' # define ALT_DIRSEP DIRSEP #endif @@ -407,7 +412,9 @@ typedef struct _client_data_t { bool suspended; /* 2 other ways to point at a context for dr_{g,s}et_mcontext() */ priv_mcontext_t *cur_mc; +#ifndef LINUX_KERNEL os_cxt_ptr_t os_cxt; +#endif /* The error code of last failed API routine. Not updated on successful API calls * but only upon failures. @@ -533,9 +540,9 @@ DYNAMORIO_EXPORT int dynamorio_app_init(void); /* dynamorio_app_init() can be called in two parts: */ void -dynamorio_app_init_part_one_options(); +dynamorio_app_init_part_one_options(void); int -dynamorio_app_init_part_two_finalize(); +dynamorio_app_init_part_two_finalize(void); int dynamorio_app_exit(void); dcontext_t * @@ -773,6 +780,11 @@ struct _dcontext_t { * XXX: change to a union? */ app_pc next_tag; +#ifdef LINUX_KERNEL + /* Holds the address of the next application instruction to execute. Unlike + * next_tag, this is not overwritten with a code cache address. */ + app_pc next_app_tag; +#endif linkstub_t *last_exit; /* last exit from cache */ byte *dstack; /* thread-private dynamo stack */ @@ -1092,7 +1104,7 @@ enum { DUMP_XML = true, DUMP_NOT_XML = false }; /* io.c */ /* to avoid transparency problems we must have our own vnsprintf and sscanf */ -#include /* for va_list */ +#include "stdarg_wrapper.h" /* for va_list */ int d_r_snprintf(char *s, size_t max, const char *fmt, ...); int @@ -1194,13 +1206,15 @@ strtoul(const char *str, char **end, int base); #endif #if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE) -# undef printf -# define printf printf_forbidden_function -# undef sprintf /* defined on macos */ -# define sprintf sprintf_forbidden_function -# define swprintf swprintf_forbidden_function -# undef vsprintf /* defined on macos */ -# define vsprintf vsprintf_forbidden_function +# ifndef LINUX_KERNEL +# undef printf +# define printf printf_forbidden_function +# undef sprintf /* defined on macos */ +# define sprintf sprintf_forbidden_function +# define swprintf swprintf_forbidden_function +# undef vsprintf /* defined on macos */ +# define vsprintf vsprintf_forbidden_function +# endif /* libc independence */ # define mprotect mprotect_forbidden_function diff --git a/core/heap.c b/core/heap.c index 92c89629e..f0da68146 100644 --- a/core/heap.c +++ b/core/heap.c @@ -40,7 +40,7 @@ */ #include "globals.h" -#include +#include "limits_wrapper.h" #include "fragment.h" /* for struct sizes */ #include "link.h" /* for struct sizes */ @@ -350,7 +350,7 @@ release_landing_pad_mem(void); * DR areas lock first, to retry */ static bool -safe_to_allocate_or_free_heap_units() +safe_to_allocate_or_free_heap_units(void) { return ((!self_owns_recursive_lock(&global_alloc_lock) && !self_owns_recursive_lock(&heap_unit_lock)) || @@ -1552,7 +1552,7 @@ reached_beyond_vmm(which_vmm_t which) } void -vmm_heap_handle_pending_low_on_memory_event_trigger() +vmm_heap_handle_pending_low_on_memory_event_trigger(void) { bool trigger = false; @@ -1569,7 +1569,7 @@ vmm_heap_handle_pending_low_on_memory_event_trigger() } static void -schedule_low_on_memory_event_trigger() +schedule_low_on_memory_event_trigger(void) { bool value = true; ATOMIC_1BYTE_WRITE(&low_on_memory_pending, value, false); @@ -2209,7 +2209,7 @@ vmm_heap_fork_init(dcontext_t *dcontext) * modified the value of any options to make them compatible */ bool -heap_check_option_compatibility() +heap_check_option_compatibility(void) { bool ret = false; @@ -2460,7 +2460,7 @@ d_r_heap_exit() } void -heap_post_exit() +heap_post_exit(void) { heap_exiting = false; } @@ -2471,7 +2471,7 @@ heap_post_exit() * need a test for hitting 2GB (or 3GB!) user mode limit. */ static void -heap_low_on_memory() +heap_low_on_memory(void) { /* free some memory! */ heap_unit_t *u, *next_u; @@ -2565,7 +2565,7 @@ report_low_on_memory(which_vmm_t which, oom_source_t source, { if (TESTANY(DYNAMO_OPTION(silent_oom_mask), source) || silent_oom_for_process(source)) { - SYSLOG_INTERNAL_WARNING("Mostly silent OOM: %s " PFX ".\n", + SYSLOG_INTERNAL_WARNING("Mostly silent OOM: %s 0x%x.\n", get_oom_source_name(source), os_error_code); /* still produce an ldmp for internal use */ if (TEST(DUMPCORE_OUT_OF_MEM_SILENT, DYNAMO_OPTION(dumpcore_mask))) @@ -2573,10 +2573,10 @@ report_low_on_memory(which_vmm_t which, oom_source_t source, } else { const char *oom_source_code = get_oom_source_name(source); char type_hex[19]; - snprintf(type_hex, BUFFER_SIZE_ELEMENTS(type_hex), PFX, which); + snprintf(type_hex, BUFFER_SIZE_ELEMENTS(type_hex), "0x%x", which); NULL_TERMINATE_BUFFER(type_hex); char status_hex[19]; - snprintf(status_hex, BUFFER_SIZE_ELEMENTS(status_hex), PFX, os_error_code); + snprintf(status_hex, BUFFER_SIZE_ELEMENTS(status_hex), "0x%x", os_error_code); NULL_TERMINATE_BUFFER(status_hex); /* SYSLOG first */ SYSLOG(SYSLOG_CRITICAL, OUT_OF_MEMORY, 4, get_application_name(), diff --git a/core/heap.h b/core/heap.h index e072f4d3d..c80c2f430 100644 --- a/core/heap.h +++ b/core/heap.h @@ -146,21 +146,21 @@ vmm_heap_fork_init(dcontext_t *dcontext); void print_vmm_heap_data(file_t outf); byte * -vmcode_get_start(); +vmcode_get_start(void); byte * -vmcode_get_end(); +vmcode_get_end(void); void iterate_vmm_regions(void (*cb)(byte *region_start, byte *region_end, void *user_data), void *user_data); byte * -vmcode_unreachable_pc(); +vmcode_unreachable_pc(void); byte * vmcode_get_writable_addr(byte *exec_addr); byte * vmcode_get_executable_addr(byte *write_addr); void -vmm_heap_handle_pending_low_on_memory_event_trigger(); +vmm_heap_handle_pending_low_on_memory_event_trigger(void); bool heap_check_option_compatibility(void); diff --git a/core/hotpatch.c b/core/hotpatch.c index 8de404513..bf9565374 100644 --- a/core/hotpatch.c +++ b/core/hotpatch.c @@ -52,7 +52,7 @@ #ifndef WINDOWS #endif -#include /* for ULLONG_MAX */ +#include "limits_wrapper.h" /* for ULLONG_MAX */ #include "fcache.h" /* for fcache_reset_all_caches_proactively */ #ifdef GBOP diff --git a/core/io.c b/core/io.c index 5c2eacb8e..153b464d2 100644 --- a/core/io.c +++ b/core/io.c @@ -50,7 +50,7 @@ typedef unsigned long ulong; void dr_fpu_exception_init(void); #endif -#include /* for varargs */ +#include "stdarg_wrapper.h" /* for varargs */ #ifdef UNIX # ifdef MACOS @@ -58,7 +58,9 @@ dr_fpu_exception_init(void); # define _EXTERNALIZE_CTYPE_INLINES_TOP_ # define _EXTERNALIZE_CTYPE_INLINES_ # endif -# include +# ifndef LINUX_KERNEL +# include +# endif #endif #ifdef NOT_DYNAMORIO_CORE_PROPER @@ -89,6 +91,7 @@ const static double zerof = 0.0; # define neg_inf (-1.0 / zerof) #endif +#ifndef LINUX_KERNEL /* assumes that d > 0 */ long /* exported to utils.c */ double2int_trunc(double d) @@ -116,6 +119,7 @@ double2int(double d) else return i; } +#endif #ifdef WINDOWS /***************************************************************************** diff --git a/core/iox.h b/core/iox.h index 90eb840bb..1e78ca564 100644 --- a/core/iox.h +++ b/core/iox.h @@ -38,7 +38,7 @@ * iox.h: i/o routines for both Linux and Windows */ -#include /* for UCHAR_MAX */ +#include "limits_wrapper.h" /* for UCHAR_MAX */ #ifdef IOX_WIDE_CHAR # define TCHAR wchar_t @@ -114,6 +114,7 @@ TNAME(ulong_to_str)(ulong num, int base, TCHAR *buf, int decimal, bool caps) return p; } +#ifndef LINUX_KERNEL /* N.B.: when building with /QIfist casting rounds instead of truncating (i#763)! * Thus, use double2int_trunc() instead of casting. */ @@ -272,6 +273,7 @@ TNAME(d_r_vsnprintf_float)(double val, const TCHAR *c, } return str; } +#endif /* Returns number of chars printed, not including the null terminator. * If number is larger than max, @@ -479,7 +481,7 @@ TNAME(d_r_vsnprintf)(TCHAR *s, size_t max, const TCHAR *fmt, va_list ap) decimal, false); break; } - /* note no break */ + DR_FALLTHROUGH; case _T('x'): case _T('X'): case _T('o'): @@ -543,7 +545,7 @@ TNAME(d_r_vsnprintf)(TCHAR *s, size_t max, const TCHAR *fmt, va_list ap) str = va_arg(ap, TCHAR *); break; } - /* fall-through */ + DR_FALLTHROUGH; case _T('S'): #ifdef IOX_WIDE_CHAR h_type = true; @@ -557,14 +559,19 @@ TNAME(d_r_vsnprintf)(TCHAR *s, size_t max, const TCHAR *fmt, va_list ap) case _T('G'): if (decimal == 0 || decimal == -1) decimal = 1; /* default */ - /* no break */ + DR_FALLTHROUGH; case _T('e'): case _T('E'): case _T('f'): { +#ifdef LINUX_KERNEL + ASSERT_NOT_REACHED(); + str = _T("FLOATING POINT NOT SUPPORTED."); +#else /* pretty sure will always be promoted to a double in arg list */ double val = va_arg(ap, double); str = TNAME(d_r_vsnprintf_float)(val, c, prefixbuf, buf, decimal, space_flag, plus_flag, pound_flag); +#endif break; } case _T('n'): { diff --git a/core/ir/disassemble_shared.c b/core/ir/disassemble_shared.c index 7aee19dea..0ab38a81c 100644 --- a/core/ir/disassemble_shared.c +++ b/core/ir/disassemble_shared.c @@ -683,6 +683,7 @@ internal_opnd_disassemble(char *buf, size_t bufsz, size_t *sofar DR_PARAM_INOUT, } } break; case IMMED_FLOAT_kind: { +#ifndef LINUX_KERNEL /* Save floating state for float printing. */ PRESERVE_FLOATING_POINT_STATE({ uint top; @@ -692,6 +693,10 @@ internal_opnd_disassemble(char *buf, size_t bufsz, size_t *sofar DR_PARAM_INOUT, print_to_buffer(buf, bufsz, sofar, "%s%s%u.%.6u", immed_prefix(), sign, top, bottom); }); +#else + print_to_buffer(buf, bufsz, sofar, "%s0x%08x /* float */", immed_prefix(), + *(const uint *)(&opnd.value.immed_float)); +#endif break; } #ifndef WINDOWS @@ -700,6 +705,7 @@ internal_opnd_disassemble(char *buf, size_t bufsz, size_t *sofar DR_PARAM_INOUT, * not equal EXPECTED_SIZEOF_OPND, triggering the ASSERT in d_r_arch_init(). */ case IMMED_DOUBLE_kind: { +# ifndef LINUX_KERNEL PRESERVE_FLOATING_POINT_STATE({ uint top; uint bottom; @@ -708,6 +714,10 @@ internal_opnd_disassemble(char *buf, size_t bufsz, size_t *sofar DR_PARAM_INOUT, print_to_buffer(buf, bufsz, sofar, "%s%s%u.%.6u", immed_prefix(), sign, top, bottom); }); +# else + print_to_buffer(buf, bufsz, sofar, "%s0x" ZHEX64_FORMAT_STRING " /* double */", + immed_prefix(), *(const uint64 *)(&opnd.value.immed_double)); +# endif break; } #endif diff --git a/core/ir/instr.h b/core/ir/instr.h index c193897c3..1e5a593ea 100644 --- a/core/ir/instr.h +++ b/core/ir/instr.h @@ -68,7 +68,9 @@ struct instr_info_t; */ /* Inlining macro controls. */ #ifndef INSTR_INLINE -# ifdef DR_FAST_IR +# ifdef LINUX_KERNEL +# define INSTR_INLINE static inline +# elif defined(DR_FAST_IR) # define INSTR_INLINE inline # else # define INSTR_INLINE @@ -858,6 +860,7 @@ enum { /* XXX: vs RAW_OPCODE_* enum */ JECXZ_OPCODE = 0xe3, JMP_SHORT_OPCODE = 0xeb, JMP_OPCODE = 0xe9, + INTN_OPCODE = 0xcd, JNE_OPCODE_1 = 0x0f, SAHF_OPCODE = 0x9e, LAHF_OPCODE = 0x9f, diff --git a/core/ir/instr_api.h b/core/ir/instr_api.h index 5956bfaaa..48bd782e4 100644 --- a/core/ir/instr_api.h +++ b/core/ir/instr_api.h @@ -43,7 +43,9 @@ */ #ifndef INSTR_INLINE -# ifdef DR_FAST_IR +# ifdef LINUX_KERNEL +# define INSTR_INLINE static inline +# elif defined(DR_FAST_IR) # define INSTR_INLINE inline # else # define INSTR_INLINE @@ -2585,10 +2587,14 @@ enum { EFLAGS_AF = 0x00000010, /**< The bit in the eflags register of AF (Aux Carry Flag). */ EFLAGS_ZF = 0x00000040, /**< The bit in the eflags register of ZF (Zero Flag). */ EFLAGS_SF = 0x00000080, /**< The bit in the eflags register of SF (Sign Flag). */ + EFLAGS_IF = + 0x00000200, /**< The bit in the eflags register of IF (Interrupt Enable Flag). */ EFLAGS_DF = 0x00000400, /**< The bit in the eflags register of DF (Direction Flag). */ EFLAGS_OF = 0x00000800, /**< The bit in the eflags register of OF (Overflow Flag). */ /** The bits in the eflags register of CF, PF, AF, ZF, SF, OF. */ EFLAGS_ARITH = EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF | EFLAGS_SF | EFLAGS_OF, + EFLAGS_NON_SYSTEM = EFLAGS_ARITH | EFLAGS_DF, + EFLAGS_SYSTEM = ~EFLAGS_NON_SYSTEM, }; #elif defined(AARCHXX) diff --git a/core/ir/instr_create_shared_api.h b/core/ir/instr_create_shared_api.h index a6a236696..3e8db72f6 100644 --- a/core/ir/instr_create_shared_api.h +++ b/core/ir/instr_create_shared_api.h @@ -56,7 +56,18 @@ # include "dr_ir_instr.h" // IWYU pragma: export # include "dr_ir_utils.h" // IWYU pragma: export #endif -#include /* For SCHAR_MAX, SCHAR_MIN. */ + +#ifdef LINUX_KERNEL +# include +# ifndef SCHAR_MAX +# define SCHAR_MAX 127 +# endif +# ifndef SCHAR_MIN +# define SCHAR_MIN (-128) +# endif +#else +# include /* For SCHAR_MAX, SCHAR_MIN. */ +#endif /** * Set the translation field for an instruction. For example: diff --git a/core/ir/opnd.h b/core/ir/opnd.h index c8054023e..a564691f8 100644 --- a/core/ir/opnd.h +++ b/core/ir/opnd.h @@ -81,7 +81,9 @@ */ /* Inlining macro controls. */ #ifndef INSTR_INLINE -# ifdef DR_FAST_IR +# ifdef LINUX_KERNEL +# define INSTR_INLINE static inline +# elif defined(DR_FAST_IR) # define INSTR_INLINE inline # else # define INSTR_INLINE diff --git a/core/ir/opnd_api.h b/core/ir/opnd_api.h index fedf332e6..6da1cdaec 100644 --- a/core/ir/opnd_api.h +++ b/core/ir/opnd_api.h @@ -48,7 +48,9 @@ #endif #ifndef INSTR_INLINE -# ifdef DR_FAST_IR +# ifdef LINUX_KERNEL +# define INSTR_INLINE static inline +# elif defined(DR_FAST_IR) # define INSTR_INLINE inline # else # define INSTR_INLINE diff --git a/core/ir/opnd_shared.c b/core/ir/opnd_shared.c index bf8269490..785da7c37 100644 --- a/core/ir/opnd_shared.c +++ b/core/ir/opnd_shared.c @@ -532,6 +532,7 @@ opnd_get_immed_int64(opnd_t opnd) (uint64)(uint)opnd.value.immed_int_multi_part.low; } +#ifndef LINUX_KERNEL /* NOTE: requires caller to be under PRESERVE_FLOATING_POINT_STATE */ float opnd_get_immed_float(opnd_t opnd) @@ -544,7 +545,7 @@ opnd_get_immed_float(opnd_t opnd) return opnd.value.immed_float; } -#ifndef WINDOWS +# ifndef WINDOWS /* XXX i#4488: x87 floating point immediates should be double precision. * Type double currently not included for Windows because sizeof(opnd_t) does * not equal EXPECTED_SIZEOF_OPND, triggering the ASSERT in d_r_arch_init(). @@ -556,7 +557,8 @@ opnd_get_immed_double(opnd_t opnd) "opnd_get_immed_double called on non-immed-float"); return opnd.value.immed_double; } -#endif +# endif +#endif /* !LINUX_KERNEL */ /* address operands */ diff --git a/core/ir/x86/decode.c b/core/ir/x86/decode.c index 06acaaea5..f9d8161ac 100644 --- a/core/ir/x86/decode.c +++ b/core/ir/x86/decode.c @@ -1989,7 +1989,7 @@ decode_operand(decode_info_t *di, byte optype, opnd_size_t opsize, opnd_t *opnd) /* ensure referencing memory */ if (di->mod >= 3) return false; - /* fall through */ + DR_FALLTHROUGH; case TYPE_E: case TYPE_Q: case TYPE_W: return decode_modrm(di, optype, opsize, NULL, opnd); @@ -2187,7 +2187,7 @@ decode_operand(decode_info_t *di, byte optype, opnd_size_t opsize, opnd_t *opnd) if (di->mod != 3) { return decode_modrm(di, optype, opsize, NULL, opnd); } - /* fall through*/ + DR_FALLTHROUGH; } case TYPE_K_MODRM_R: { /* part of AVX-512: modrm.rm selects opmask register */ @@ -2894,7 +2894,7 @@ unit_check_decode_ff_opcode() instr_encode(dcontext, instr, pc); \ instr_reset(dcontext, instr); \ decode(dcontext, pc, instr); \ - /* XXX: use EXPECT */ \ + /* XXX: use EXPECT */ \ CLIENT_ASSERT(instr_get_opcode(instr) == OP_##opc, "unit test"); \ instr_destroy(dcontext, instr); diff --git a/core/ir/x86/decode_fast.c b/core/ir/x86/decode_fast.c index 7f34a7629..fbfd5ba72 100644 --- a/core/ir/x86/decode_fast.c +++ b/core/ir/x86/decode_fast.c @@ -511,9 +511,7 @@ decode_sizeof_ex(void *drcontext, byte *start_pc, int *num_prefixes, uint *rip_r sz += 1; break; case REPNE_PREFIX_OPCODE: - case REP_PREFIX_OPCODE: /* REP */ - rep_prefix = true; - /* fall through */ + case REP_PREFIX_OPCODE: /* REP */ rep_prefix = true; DR_FALLTHROUGH; case RAW_PREFIX_lock: /* LOCK */ case CS_SEG_OPCODE: /* segment overrides */ case DS_SEG_OPCODE: @@ -535,7 +533,7 @@ decode_sizeof_ex(void *drcontext, byte *start_pc, int *num_prefixes, uint *rip_r if (X64_MODE_DC(dcontext) || TEST(0x10, *(pc + 1))) { evex_prefix = true; } - /* Fall-through is deliberate, EVEX is handled through VEX below */ + DR_FALLTHROUGH; } case VEX_3BYTE_PREFIX_OPCODE: case VEX_2BYTE_PREFIX_OPCODE: { @@ -1367,7 +1365,7 @@ decode_cti(void *drcontext, byte *pc, instr_t *instr) break; case EVEX_PREFIX_OPCODE: instr_set_prefix_flag(instr, PREFIX_EVEX); - /* fall-through */ + DR_FALLTHROUGH; case VEX_3BYTE_PREFIX_OPCODE: { /* EVEX and VEX 3-byte prefixes imply instruction opcodes by encoding mm * bits in the second prefix byte. In theory, there are 5 VEX mm bits, but diff --git a/core/ir/x86/disassemble.c b/core/ir/x86/disassemble.c index 3ba7b7267..a1340cbff 100644 --- a/core/ir/x86/disassemble.c +++ b/core/ir/x86/disassemble.c @@ -172,7 +172,7 @@ opnd_disassemble_noimplicit(char *buf, size_t bufsz, size_t *sofar DR_PARAM_INOU /* if has implicit st0 then don't print it */ (opnd_get_reg(opnd) == REG_ST0 && instr_memory_reference_size(instr) > 0)) return false; - /* else fall through */ + DR_FALLTHROUGH; case TYPE_A: case TYPE_B: case TYPE_C: @@ -225,6 +225,7 @@ opnd_disassemble_noimplicit(char *buf, size_t bufsz, size_t *sofar DR_PARAM_INOU reg_disassemble(buf, bufsz, sofar, opnd_get_segment(opnd), 0, "", ""); return true; } + DR_FALLTHROUGH; case TYPE_Y: case TYPE_FLOATCONST: case TYPE_XREG: @@ -332,6 +333,8 @@ instr_opcode_name_suffix(instr_t *instr) return "d"; else if (sz == 8) return "q"; + + DR_FALLTHROUGH; } case OP_pusha: case OP_popa: { @@ -340,6 +343,8 @@ instr_opcode_name_suffix(instr_t *instr) return "w"; else if (sz == 32) return "d"; + + DR_FALLTHROUGH; } case OP_iret: { uint sz = instr_memory_reference_size(instr); diff --git a/core/ir/x86/encode.c b/core/ir/x86/encode.c index 290ac4ce3..1fcf093df 100644 --- a/core/ir/x86/encode.c +++ b/core/ir/x86/encode.c @@ -1562,7 +1562,7 @@ opnd_type_ok(decode_info_t *di /*prefixes field is IN/OUT; x86_mode is IN*/, opn if (!reg_is_strictly_zmm(opnd_get_index(opnd))) return false; } - /* fall through */ + DR_FALLTHROUGH; case TYPE_FLOATMEM: case TYPE_M: return mem_size_ok(di, opnd, optype, opsize); case TYPE_E: @@ -1801,7 +1801,7 @@ opnd_type_ok(decode_info_t *di /*prefixes field is IN/OUT; x86_mode is IN*/, opn case TYPE_K_MODRM: if (mem_size_ok(di, opnd, optype, opsize)) return true; - /* fall through */ + DR_FALLTHROUGH; case TYPE_K_MODRM_R: /* Same comment above. */ return (opnd_is_reg(opnd) && reg_is_opmask(opnd_get_reg(opnd))); @@ -3523,7 +3523,7 @@ instr_encode_arch(dcontext_t *dcontext, instr_t *instr, byte *copy_pc, byte *fin instr_set_rip_rel_pos(instr, (byte)(disp_relativize_at - di.start_pc)); } -#if DEBUG_DISABLE /* turn back on if want to debug */ +#ifdef DEBUG_DISABLE /* turn back on if want to debug */ if (d_r_stats->loglevel >= 3) { byte *pc = cache_pc; LOG(THREAD, LOG_EMIT, 3, "instr_encode on: "); diff --git a/core/ir/x86/instr.c b/core/ir/x86/instr.c index 72578ce4a..ad4c063c2 100644 --- a/core/ir/x86/instr.c +++ b/core/ir/x86/instr.c @@ -1652,7 +1652,7 @@ instr_predicate_triggered(instr_t *instr, dr_mcontext_t *mc) ptr_int_t val; if (!d_r_safe_read(opnd_compute_address(src, mc), MIN(opnd_get_size(src), sizeof(val)), &val)) - return false; + return (dr_pred_trigger_t) false; return (val != 0) ? DR_PRED_TRIGGER_MATCH : DR_PRED_TRIGGER_MISMATCH; } else CLIENT_ASSERT(false, "invalid predicate/instr combo"); diff --git a/core/ir/x86/instr_create_api.h b/core/ir/x86/instr_create_api.h index 7df551929..60f07a0a7 100644 --- a/core/ir/x86/instr_create_api.h +++ b/core/ir/x86/instr_create_api.h @@ -50,7 +50,9 @@ * the instr_shrink_to_16_bits() routine. */ -#include /* for floating-point math constants */ +#ifndef LINUX_KERNEL +# include /* for floating-point math constants */ +#endif /* instruction modification convenience routines */ /** diff --git a/core/jit_opt.c b/core/jit_opt.c index 94abda28b..c1c9b6664 100644 --- a/core/jit_opt.c +++ b/core/jit_opt.c @@ -547,7 +547,7 @@ fragment_tree_clear(fragment_tree_t *tree) static fragment_tree_t *fragment_tree; void -jitopt_init() +jitopt_init(void) { if (DYNAMO_OPTION(opt_jit)) { fragment_tree = fragment_tree_create(); @@ -564,7 +564,7 @@ jitopt_init() } void -jitopt_exit() +jitopt_exit(void) { if (DYNAMO_OPTION(opt_jit)) fragment_tree_destroy(fragment_tree); diff --git a/core/jit_opt.h b/core/jit_opt.h index 961fc8b05..7402e73a0 100644 --- a/core/jit_opt.h +++ b/core/jit_opt.h @@ -2,10 +2,10 @@ #define _JIT_OPT_H_ void -jitopt_init(); +jitopt_init(void); void -jitopt_exit(); +jitopt_exit(void); /* Account for a DGC basic block having the specified span in app space. */ void diff --git a/core/kernel_linux/kernel_interface.c b/core/kernel_linux/kernel_interface.c index f727b0c5e..ea168b884 100644 --- a/core/kernel_linux/kernel_interface.c +++ b/core/kernel_linux/kernel_interface.c @@ -12,7 +12,6 @@ #include "dynamorio_module_interface.h" #include "dynamorio_module_assert_interface.h" #include "kernel_interface.h" -#include "hypercall_guest.h" #include "page_table.h" /* Used by kernel_find_dynamorio_module_bounds. */ @@ -167,11 +166,6 @@ kernel_module_init(size_t dr_heap_size) kallsyms_lookup_name_ptr = (void *)kp.addr; unregister_kprobe(&kp); -#ifdef HYPERCALL_DEBUGGING - if (!hypercall_init()) { - return false; - } -#endif /* Some OS interfaces, such as get_thread_private_dcontext, rely on the TLS * initially being all 0. */ zero_cpu_private_data(); @@ -238,7 +232,7 @@ kernel_interface_exit(void) } void * -kernel_load_shared_library(char *name) +kernel_load_shared_library(const char *name) { struct module *module; @@ -254,7 +248,7 @@ kernel_load_shared_library(char *name) #define MAX_QUALIFIED_NAME_LEN 256 void * -kernel_lookup_library_routine(void *lib, char *name) +kernel_lookup_library_routine(void *lib, const char *name) { struct module *module = (struct module *)lib; /* Build a string with the format mod_name:routine_name. */ diff --git a/core/kernel_linux/kernel_interface.h b/core/kernel_linux/kernel_interface.h index 9a3437231..7e5648841 100644 --- a/core/kernel_linux/kernel_interface.h +++ b/core/kernel_linux/kernel_interface.h @@ -68,9 +68,9 @@ is_kernel_code(void *pc); /* None of these routines are safe to use after initilization. */ void * -kernel_load_shared_library(char *name); +kernel_load_shared_library(const char *name); void * -kernel_lookup_library_routine(void *lib, char *name); +kernel_lookup_library_routine(void *lib, const char *name); bool kernel_shared_library_bounds(void *lib, byte *addr, byte **start, byte **end); byte * diff --git a/core/kernel_linux/modules/Makefile b/core/kernel_linux/modules/Makefile index 6bbf54d97..2f8360120 100644 --- a/core/kernel_linux/modules/Makefile +++ b/core/kernel_linux/modules/Makefile @@ -18,66 +18,75 @@ dynamorio_controller-objs :=\ ../../kernel_linux/simple_tests/repstr.o\ ../../kernel_linux/simple_tests.o -################################################################################ -## TODO i#20: Re-enable this module once the previous module has been ported. -## obj-m += dynamorio.o -## dynamorio-objs :=\ -## ../../exports.o\ -## ../../buildmark.o\ -## ../../config.o\ -## ../../dispatch.o\ -## ../../dynamo.o\ -## ../../emit.o\ -## ../../fcache.o\ -## ../../fragment.o\ -## ../../hashtable.o\ -## ../../heap.o\ -## ../../hotpatch.o\ -## ../../instrlist.o\ -## ../../io.o\ -## ../../link.o\ -## ../../loader_shared.o\ -## ../../moduledb.o\ -## ../../module_list.o\ -## ../../monitor.o\ -## ../../nudge.o\ -## ../../options.o\ -## ../../perfctr.o\ -## ../../perscache.o\ -## ../../rct.o\ -## ../../stats.o\ -## ../../synch.o\ -## ../../unit-rct.o\ -## ../../utils.o\ -## ../../barrier.o\ -## ../../vmareas.o\ -## ../../x86/arch.o\ -## ../../x86/decode.o\ -## ../../x86/decode_fast.o\ -## ../../x86/decode_table.o\ -## ../../x86/disassemble.o\ -## ../../x86/emit_utils.o\ -## ../../x86/encode.o\ -## ../../x86/instr.o\ -## ../../x86/instrument.o\ -## ../../x86/interp.o\ -## ../../x86/loadtoconst.o\ -## ../../x86/mangle.o\ -## ../../x86/optimize.o\ -## ../../x86/proc.o\ -## ../../x86/retcheck.o\ -## ../../x86/sideline.o\ -## ../../x86/steal_reg.o\ -## ../../x86/x86_code.o\ -## ../../x86/asm_defines.o\ -## ../../x86/x86.o\ -## ../../kernel_linux/os.o\ -## ../../kernel_linux/hypercall_guest.o\ -## ../../kernel_linux/page_table.o\ -## ../../kernel_linux/kernel_interface.o\ -## ../../kernel_linux/dynamorio_module_interface.o\ -## ../../kernel_linux/dynamorio_module.o -################################################################################ +obj-m += dynamorio.o +dynamorio-objs :=\ +../../exports.o\ +../../buildmark.o\ +../../config.o\ +../../dispatch.o\ +../../dynamo.o\ +../../emit.o\ +../../fcache.o\ +../../fragment.o\ +../../hashtable.o\ +../../heap.o\ +../../hotpatch.o\ +../../ir/instrlist.o\ +../../io.o\ +../../link.o\ +../../loader_shared.o\ +../../moduledb.o\ +../../module_list.o\ +../../monitor.o\ +../../nudge.o\ +../../options.o\ +../../perfctr.o\ +../../perscache.o\ +../../rct.o\ +../../stats.o\ +../../synch.o\ +../../utils.o\ +../../barrier.o\ +../../vmareas.o\ +../../translate.o\ +../../string.o\ +../../lib/module_api.o\ +../../arch/arch.o\ +../../arch/emit_utils_shared.o\ +../../arch/proc_shared.o\ +../../arch/mangle_shared.o\ +../../arch/clean_call_opt_shared.o\ +../../arch/x86/clean_call_opt.o\ +../../ir/decode_shared.o\ +../../ir/disassemble_shared.o\ +../../ir/encode_shared.o\ +../../ir/instr_shared.o\ +../../ir/ir_utils_shared.o\ +../../ir/opnd_shared.o\ +../../ir/x86/decode.o\ +../../ir/x86/decode_fast.o\ +../../ir/x86/decode_table.o\ +../../ir/x86/disassemble.o\ +../../arch/x86/emit_utils.o\ +../../ir/x86/encode.o\ +../../ir/x86/instr.o\ +../../ir/x86/opnd.o\ +../../ir/x86/ir_utils.o\ +../../lib/instrument.o\ +../../arch/interp.o\ +../../arch/loadtoconst.o\ +../../arch/x86/mangle.o\ +../../arch/x86/optimize.o\ +../../arch/x86/proc.o\ +../../arch/retcheck.o\ +../../arch/sideline.o\ +../../arch/asm_defines.o\ +../../arch/x86/x86.o\ +../../kernel_linux/os.o\ +../../kernel_linux/page_table.o\ +../../kernel_linux/kernel_interface.o\ +../../kernel_linux/dynamorio_module_interface.o\ +../../kernel_linux/dynamorio_module.o ################################################################################ ## TODO i#20: Re-enable this module once the previous modules have been ported. diff --git a/core/kernel_linux/os.c b/core/kernel_linux/os.c index 0a96fa791..f1771db04 100644 --- a/core/kernel_linux/os.c +++ b/core/kernel_linux/os.c @@ -1,6 +1,4 @@ #include "globals.h" -#include "hypercall.h" -#include "hypercall_guest.h" #include "kernel_interface.h" #include "module_shared.h" #include "page_table.h" @@ -18,9 +16,7 @@ #include "cr.h" #include "monitor.h" -#ifdef CLIENT_INTERFACE -# include "instrument.h" -#endif +#include "instrument.h" /* For inline ASM. */ #ifdef X64 @@ -35,9 +31,10 @@ # define ASM_XSP "esp" #endif +app_pc vsyscall_page_start = NULL; app_pc vsyscall_syscall_end_pc = NULL; app_pc vsyscall_sysenter_return_pc = NULL; -app_pc vsyscall_page_start = NULL; +app_pc vsyscall_sysenter_displaced_pc = NULL; DR_API file_t our_stdin = 0; DR_API file_t our_stdout = 1; @@ -90,10 +87,8 @@ typedef enum { INTERRUPTED_FRAGMENT, INTERRUPTED_GENCODE, INTERRUPTED_DYNAMORIO, -#ifdef CLIENT_INTERFACE INTERRUPTED_CLIENT_LIB, INTERRUPTED_CLIENT_GENCODE, -#endif } interrupted_location_t; typedef struct { @@ -103,7 +98,7 @@ typedef struct { interrupt_stack_frame_t frame; /* Keep a pointer around so we can reset eflags.IF. */ interrupt_stack_frame_t *raw_frame; - dr_mcontext_t *mcontext; + priv_mcontext_t *mcontext; interrupt_vector_t vector; interrupted_location_t location; } interrupt_context_t; @@ -122,7 +117,7 @@ typedef struct { /* State of the pending interrupt. */ bool pending_interrupt; bool use_interrupted_mcontext; - dr_mcontext_t interrupted_mcontext; + priv_mcontext_t interrupted_mcontext; interrupt_vector_t interrupt_vector; reg_t interrupt_error_code; reg_t interrupt_system_xflags; @@ -141,7 +136,7 @@ typedef struct { int num_patches; cache_pc patch_pc[MAX_NUM_PATCHES]; - byte patch_buffer[MAX_NUM_PATCHES][INTN_LENGTH]; + byte patch_buffer[MAX_NUM_PATCHES][INT_LENGTH]; #ifdef DEBUG /* Keep track of where the interrupt came from for debugging. */ interrupted_location_t interrupted_location; @@ -166,9 +161,7 @@ typedef struct { typedef struct os_local_state_t { local_state_extended_t state; struct os_local_state_t *self; -#ifdef CLIENT_INTERFACE void *client_tls[MAX_NUM_CLIENT_TLS]; -#endif } os_local_state_t; /* Offsets from the GS segment. */ @@ -206,7 +199,8 @@ os_module_area_reset(module_area_t *ma HEAPACCT(which_heap_t which)) } generic_func_t -get_proc_address_ex(module_handle_t lib, const char *name, bool *is_indirect_code OUT) +get_proc_address_ex(module_base_t lib, const char *name, + bool *is_indirect_code DR_PARAM_OUT) { ASSERT_NOT_PORTED(false); return 0; @@ -273,7 +267,7 @@ module_get_header_size(app_pc module_base) } bool -module_has_text_relocs(app_pc base) +module_has_text_relocs(app_pc base, bool at_map) { ASSERT_NOT_PORTED(false); return true; @@ -381,6 +375,20 @@ os_tls_offset(tls_offset_t tls_offs) return (tls_offset_t)(tls_local_state_offset + tls_offs); } +ushort +os_get_app_tls_base_offset(reg_id_t reg) +{ + ASSERT_NOT_REACHED(); + return 0; +} + +ushort +os_get_app_tls_reg_offset(reg_id_t reg) +{ + ASSERT_NOT_REACHED(); + return 0; +} + char * get_application_pid() { @@ -493,7 +501,7 @@ os_tls_exit(local_state_t *local_state, bool other_thread) } bool -os_tls_calloc(OUT uint *offset, uint num_slots, uint alignment) +os_tls_calloc(DR_PARAM_OUT uint *offset, uint num_slots, uint alignment) { ASSERT_NOT_PORTED(false); return true; @@ -508,7 +516,7 @@ os_tls_cfree(uint offset, uint num_slots) /* TODO(peter): Move this to arch. */ static void -get_interrupted_context(interrupt_context_t *interrupt, dr_mcontext_t *out) +get_interrupted_context(interrupt_context_t *interrupt, priv_mcontext_t *out) { copy_mcontext(interrupt->mcontext, out); out->xip = interrupt->frame.xip; @@ -530,7 +538,7 @@ redirect_iret_to_fcache_return(dcontext_t *dcontext, /* TODO(peter): Move this to arch. */ static void -emulate_interrupt_arrival(dr_mcontext_t *mcontext, interrupt_vector_t vector, +emulate_interrupt_arrival(priv_mcontext_t *mcontext, interrupt_vector_t vector, byte *handler, reg_t error_code, reg_t system_xflags, bool frame_if) { @@ -608,7 +616,7 @@ handle_user_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt) interrupt->mcontext->rsp += sizeof(interrupt->frame.error_code); } set_last_exit(dcontext, (linkstub_t *)get_user_interrupt_entry_linkstub()); - transfer_to_dispatch(dcontext, 0 /* errno */, interrupt->mcontext); + transfer_to_dispatch(dcontext, interrupt->mcontext, false); ASSERT_NOT_REACHED(); } @@ -630,7 +638,7 @@ get_interrupted_location(dcontext_t *dcontext, interrupt_stack_frame_t *frame) ASSERT(dcontext->whereami != WHERE_USERMODE); } - if (dcontext->whereami == WHERE_FCACHE) { + if (dcontext->whereami == DR_WHERE_FCACHE) { if (in_generated_routine(dcontext, pc)) { return INTERRUPTED_GENCODE; } else if (!is_on_dstack(dcontext, (byte *)frame->xsp)) { @@ -638,18 +646,12 @@ get_interrupted_location(dcontext_t *dcontext, interrupt_stack_frame_t *frame) * kernel entry gencode, then interrupts are disabled and we * shouldn't trigger exceptions). Hence it's safe to call in_fcache. */ -#ifdef CLIENT_INTERFACE if (in_fcache(pc)) { -#else - ASSERT(in_fcache(pc)); -#endif return INTERRUPTED_FRAGMENT; -#ifdef CLIENT_INTERFACE } else { ASSERT(is_dynamo_address(frame->xip)); return INTERRUPTED_CLIENT_GENCODE; } -#endif } else { /* At this point, we could be * - in some DR code that uses exceptions (e.g., TRY..EXCEPT) @@ -663,17 +665,15 @@ get_interrupted_location(dcontext_t *dcontext, interrupt_stack_frame_t *frame) * * Note that the following is impossible * - in some random kernel code that we call (e.g., memcpy) after - * whereami = WHERE_FCACHE was set in the dispatcher + * whereami = DR_WHERE_FCACHE was set in the dispatcher * because we don't do anything that can fault after we set - * whereami = WHERE_FCACHE + * whereami = DR_WHERE_FCACHE */ if (is_in_dynamo_dll(pc)) { return INTERRUPTED_DYNAMORIO; -#ifdef CLIENT_INTERFACE } else if (is_in_client_lib(pc)) { /* clean callee */ return INTERRUPTED_CLIENT_LIB; -#endif } else { /* Could either be in clean call preparation or a crashing clean * call argument. If we're in the kernel code, then in_fcache @@ -692,20 +692,18 @@ get_interrupted_location(dcontext_t *dcontext, interrupt_stack_frame_t *frame) } } } else { - /* Whenever whereami != WHERE_FCACHE, we should be on the dstack. */ + /* Whenever whereami != DR_WHERE_FCACHE, we should be on the dstack. */ ASSERT(is_on_dstack(dcontext, (byte *)frame->xsp)); if (in_generated_routine(dcontext, pc)) { - /* We run some generated code when whereami != WHERE_FCACHE and not + /* We run some generated code when whereami != DR_WHERE_FCACHE and not * on the dstack (namely the kernel entry points), but these * routines should not generate exceptions and they run with * interrupts disabled. */ ASSERT_NOT_REACHED(); return INTERRUPTED_GENCODE; -#ifdef CLIENT_INTERFACE } else if (is_in_client_lib(pc)) { return INTERRUPTED_CLIENT_LIB; -#endif } else { /* If we're in kernel code called by DynamoRIO, then we're screwed * because interrupts should be disabled and we don't know how to @@ -715,6 +713,8 @@ get_interrupted_location(dcontext_t *dcontext, interrupt_stack_frame_t *frame) return INTERRUPTED_DYNAMORIO; } } + ASSERT_NOT_REACHED(); + return INTERRUPTED_DYNAMORIO; } bool @@ -779,7 +779,7 @@ is_patch_interrupt(os_thread_data_t *ostd, interrupt_context_t *interrupt) * all classified as "traps" rather than "faults" (see Section 6.5 in * Intel's Programmer Reference Manual, volume 3A). */ - if (ostd->patch_pc[i] + INTN_LENGTH == interrupt->frame.xip) { + if (ostd->patch_pc[i] + INT_LENGTH == interrupt->frame.xip) { return true; } second_patch(); @@ -846,7 +846,7 @@ receive_pending_interrupt(dcontext_t *dcontext) static void record_pending_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt, - dr_mcontext_t *interrupted_mcontext, bool modify_if) + priv_mcontext_t *interrupted_mcontext, bool modify_if) { os_thread_data_t *ostd = (os_thread_data_t *)dcontext->os_field; ostd->pending_interrupt = true; @@ -880,7 +880,7 @@ is_loop_opc(uint opc) static void handle_fragment_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt) { - dr_mcontext_t mcontext; + priv_mcontext_t mcontext; fragment_t wrapper; recreate_success_t res; bool waslinking = is_couldbelinking(dcontext); @@ -897,7 +897,7 @@ handle_fragment_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt) KSTART(recreate_app_state_frag_intr); if (!waslinking) enter_couldbelinking(dcontext, NULL, false); - res = recreate_app_state(dcontext, &mcontext, true); + res = recreate_app_state(dcontext, &mcontext, true, NULL); if (!waslinking) enter_nolinking(dcontext, NULL, false); KSTOP(recreate_app_state_frag_intr); @@ -917,7 +917,7 @@ handle_fragment_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt) dcontext->next_tag = mcontext.xip; set_last_exit(dcontext, (linkstub_t *)get_kernel_interrupt_entry_linkstub()); STATS_INC(num_ndelayed_frag_intr); - transfer_to_dispatch(dcontext, 0, &mcontext); + transfer_to_dispatch(dcontext, &mcontext, false); } else if (res == RECREATE_DELAY_UNTIL_DISPATCH) { /* Switch from kernel_interrupt_handling */ KSWITCH(kernel_interrupt_frag_delay_dispatch); @@ -1000,7 +1000,7 @@ handle_fcache_enter_interrupt(dcontext_t *dcontext, interrupt_context_t *interru ASSERT(!is_dynamo_address(dcontext->next_tag)); ASSERT(is_kernel_code(dcontext->next_tag)); STATS_INC(num_fcache_enter_interrupts); - transfer_to_dispatch(dcontext, 0, get_mcontext(dcontext)); + transfer_to_dispatch(dcontext, get_mcontext(dcontext), false); } static void @@ -1046,9 +1046,9 @@ handle_kernel_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt) ASSERT(!vector_is_synchronous(interrupt->vector)); if (in_indirect_branch_lookup_code(dcontext, interrupt->frame.xip)) { handle_ibl_interrupt(dcontext, interrupt); - } else if (in_fcache_enter_code(dcontext, interrupt->frame.xip)) { + } else if (in_fcache_enter(dcontext, interrupt->frame.xip)) { handle_fcache_enter_interrupt(dcontext, interrupt); - } else if (in_fcache_return_code(dcontext, interrupt->frame.xip)) { + } else if (in_fcache_return(dcontext, interrupt->frame.xip)) { handle_fcache_return_interrupt(dcontext, interrupt); } else { /* We don't expect interrupts for any other gencode. */ @@ -1068,7 +1068,6 @@ handle_kernel_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt) ASSERT_NOT_REACHED(); os_terminate(dcontext, TERMINATE_PROCESS); break; -#ifdef CLIENT_INTERFACE case INTERRUPTED_CLIENT_LIB: case INTERRUPTED_CLIENT_GENCODE: /* The client should have handled this interrupt and either suppressed @@ -1077,7 +1076,6 @@ handle_kernel_interrupt(dcontext_t *dcontext, interrupt_context_t *interrupt) ASSERT_NOT_REACHED(); os_terminate(dcontext, TERMINATE_PROCESS); break; -#endif default: os_terminate(dcontext, TERMINATE_PROCESS); } } @@ -1087,23 +1085,25 @@ nmi_handler(void) { } -#ifdef CLIENT_INTERFACE static bool send_interrupt_to_client(dcontext_t *dcontext, interrupt_context_t *interrupt) { bool res; dr_interrupt_t dr_interrupt; + dr_mcontext_t mc; dr_interrupt.frame = &interrupt->frame; dr_interrupt.raw_frame = interrupt->raw_frame; dr_interrupt.vector = interrupt->vector; - dr_interrupt.mcontext = interrupt->mcontext; + dr_mcontext_init(&mc); + priv_mcontext_to_dr_mcontext(&mc, interrupt->mcontext); + dr_interrupt.mcontext = &mc; res = instrument_interrupt(dcontext, &dr_interrupt); + dr_mcontext_to_priv_mcontext(interrupt->mcontext, &mc); return res; } -#endif static void -handle_interrupt(interrupt_stack_frame_t *frame, dr_mcontext_t *mcontext, +handle_interrupt(interrupt_stack_frame_t *frame, priv_mcontext_t *mcontext, interrupt_vector_t vector) { dcontext_t *dcontext; @@ -1157,15 +1157,13 @@ handle_interrupt(interrupt_stack_frame_t *frame, dr_mcontext_t *mcontext, if (is_patch_interrupt(ostd, &interrupt)) { unpatch_fragments(dcontext, ostd); - interrupt.raw_frame->xip -= INTN_LENGTH; - interrupt.frame.xip -= INTN_LENGTH; + interrupt.raw_frame->xip -= INT_LENGTH; + interrupt.frame.xip -= INT_LENGTH; interrupt.raw_frame->xflags |= EFLAGS_IF; interrupt.frame.xflags |= EFLAGS_IF; } -#ifdef CLIENT_INTERFACE if (send_interrupt_to_client(dcontext, &interrupt)) { -#endif #ifdef DEBUG DOKSTATS({ kstat_stack_t *ks = &dcontext->thread_kstats->stack_kstats; @@ -1211,9 +1209,7 @@ handle_interrupt(interrupt_stack_frame_t *frame, dr_mcontext_t *mcontext, /* If this returns, then we iret to whatever was interrupted. The * state in mcontext and frame are restored. */ } -#ifdef CLIENT_INTERFACE } -#endif if (local) SELF_PROTECT_LOCAL(dcontext, READONLY); @@ -1232,8 +1228,7 @@ optimize_syscall_entry(dcontext_t *dcontext) os_thread_data_t *ostd = (os_thread_data_t *)dcontext->os_field; fragment_t *f; f = build_basic_block_fragment(dcontext, (byte *)ostd->native_state.msr_lstar, - FRAG_CANNOT_DELETE, true, - true _IF_CLIENT(false) _IF_CLIENT(NULL)); + FRAG_CANNOT_DELETE, true, true, false, NULL); ostd->syscall_entry_frag = f; optimize_syscall_code(dcontext, f); } @@ -1450,7 +1445,7 @@ os_get_native_syscall_entry(dcontext_t *dcontext) } void -os_thread_init(dcontext_t *dcontext) +os_thread_init(dcontext_t *dcontext, void *os_data) { os_thread_data_t *ostd = (os_thread_data_t *)heap_alloc( dcontext, sizeof(os_thread_data_t) HEAPACCT(ACCT_OTHER)); @@ -1467,7 +1462,7 @@ os_thread_after_arch_init(dcontext_t *dcontext) } void -os_thread_exit(dcontext_t *dcontext) +os_thread_exit(dcontext_t *dcontext, bool other_thread) { os_thread_data_t *ostd = (os_thread_data_t *)dcontext->os_field; @@ -1489,7 +1484,7 @@ os_thread_under_dynamo(dcontext_t *dcontext) } void -os_thread_not_under_dynamo(dcontext_t *dcontext) +os_thread_not_under_dynamo(dcontext_t *dcontext, bool restore_sigblocked) { /* This is called when a CPU returns to user space. */ } @@ -1650,7 +1645,7 @@ os_heap_get_commit_limit(size_t *commit_used, size_t *commit_limit) } void -thread_yield() +thread_yield(void) { SPINLOCK_PAUSE(); } @@ -1690,14 +1685,21 @@ is_thread_terminated(dcontext_t *dcontext) } bool -thread_get_mcontext(thread_record_t *tr, dr_mcontext_t *mc) +os_wait_thread_terminated(dcontext_t *dcontext) { ASSERT_NOT_PORTED(false); return true; } bool -thread_set_mcontext(thread_record_t *tr, dr_mcontext_t *mc) +thread_get_mcontext(thread_record_t *tr, priv_mcontext_t *mc) +{ + ASSERT_NOT_PORTED(false); + return true; +} + +bool +thread_set_mcontext(thread_record_t *tr, priv_mcontext_t *mc) { ASSERT_NOT_PORTED(false); return true; @@ -1724,13 +1726,13 @@ get_num_processors() } shlib_handle_t -load_shared_library(char *name) +load_shared_library(const char *name, bool reachable) { return kernel_load_shared_library(name); } shlib_routine_ptr_t -lookup_library_routine(shlib_handle_t lib, char *name) +lookup_library_routine(shlib_handle_t lib, const char *name) { return kernel_lookup_library_routine(lib, name); } @@ -1751,8 +1753,9 @@ shared_library_error(char *buf, int maxlen) } bool -shared_library_bounds(IN shlib_handle_t lib, IN byte *addr, OUT byte **start, - OUT byte **end) +shared_library_bounds(DR_PARAM_IN shlib_handle_t lib, DR_PARAM_IN byte *addr, + DR_PARAM_IN const char *name, DR_PARAM_OUT byte **start, + DR_PARAM_OUT byte **end) { return kernel_shared_library_bounds(lib, addr, start, end); } @@ -1948,8 +1951,8 @@ os_delete_mapped_file(const char *filename) } byte * -os_map_file(file_t f, size_t *size INOUT, uint64 offs, app_pc addr, uint prot, - bool copy_on_write, bool image, bool fixed) +os_map_file(file_t f, size_t *size DR_PARAM_INOUT, uint64 offs, app_pc addr, uint prot, + map_flags_t map_flags) { ASSERT_NOT_PORTED(false); return NULL; @@ -2093,7 +2096,7 @@ vm_region_prot(const vm_region_t *region) } bool -query_memory_ex_from_os(const byte *pc, OUT dr_mem_info_t *info) +query_memory_ex_from_os(const byte *pc, DR_PARAM_OUT dr_mem_info_t *info) { vm_region_t region; /* TODO(peter): page_table_get_region is way too slow. For now, just get the @@ -2215,13 +2218,6 @@ get_stack_bounds(dcontext_t *dcontext, byte **base, byte **top) return true; } -bool -ignorable_system_call(int num) -{ - ASSERT_NOT_PORTED(false); - return true; -} - bool is_clone_thread_syscall(dcontext_t *dcontext) { @@ -2296,29 +2292,25 @@ extern void deadlock_avoidance_unlock(mutex_t *lock, bool ownable); void -mutex_wait_contended_lock(mutex_t *lock) +mutex_wait_contended_lock(mutex_t *lock, priv_mcontext_t *mc) { - IF_CLIENT_INTERFACE(dcontext_t *dcontext = get_thread_private_dcontext();) + dcontext_t *dcontext = get_thread_private_dcontext(); /* FIXME: we don't actually use system calls to synchronize on Linux, * one day we would use futex(2) on this path (PR 295561). * For now we use a busy-wait lock. * If we do use a true wait need to set client_thread_safe_for_synch around it */ /* we now have to undo our earlier request */ - atomic_dec_and_test(&lock->lock_requests); + d_r_atomic_dec_and_test(&lock->lock_requests); - while (!mutex_trylock(lock)) { -#ifdef CLIENT_INTERFACE + while (!d_r_mutex_trylock(lock)) { if (dcontext != NULL && IS_CLIENT_THREAD(dcontext) && (mutex_t *)dcontext->client_data->client_grab_mutex == lock) dcontext->client_data->client_thread_safe_for_synch = true; -#endif thread_yield(); -#ifdef CLIENT_INTERFACE if (dcontext != NULL && IS_CLIENT_THREAD(dcontext) && (mutex_t *)dcontext->client_data->client_grab_mutex == lock) dcontext->client_data->client_thread_safe_for_synch = false; -#endif } #ifdef DEADLOCK_AVOIDANCE @@ -2398,27 +2390,27 @@ destroy_event(event_t e) void signal_event(event_t e) { - mutex_lock(&e->lock); + d_r_mutex_lock(&e->lock); e->signaled = true; LOG(THREAD_GET, LOG_THREADS, 3, "thread %d signalling event " PFX "\n", get_thread_id(), e); - mutex_unlock(&e->lock); + d_r_mutex_unlock(&e->lock); } void reset_event(event_t e) { - mutex_lock(&e->lock); + d_r_mutex_lock(&e->lock); e->signaled = false; LOG(THREAD_GET, LOG_THREADS, 3, "thread %d resetting event " PFX "\n", get_thread_id(), e); - mutex_unlock(&e->lock); + d_r_mutex_unlock(&e->lock); } /* FIXME: compare use and implementation with man pthread_cond_wait */ /* FIXME PR 295561: use futex */ -void -wait_for_event(event_t e) +bool +wait_for_event(event_t e, int timeout_ms) { #ifdef DEBUG dcontext_t *dcontext = get_thread_private_dcontext(); @@ -2428,19 +2420,19 @@ wait_for_event(event_t e) e); while (true) { if (e->signaled) { - mutex_lock(&e->lock); + d_r_mutex_lock(&e->lock); if (!e->signaled) { /* some other thread beat us to it */ LOG(THREAD, LOG_THREADS, 3, "thread %d was beaten to event " PFX "\n", get_thread_id(), e); - mutex_unlock(&e->lock); + d_r_mutex_unlock(&e->lock); } else { /* reset the event */ e->signaled = false; - mutex_unlock(&e->lock); + d_r_mutex_unlock(&e->lock); LOG(THREAD, LOG_THREADS, 3, "thread %d finished waiting for event " PFX "\n", get_thread_id(), e); - return; + return true; } } thread_yield(); @@ -2450,10 +2442,12 @@ wait_for_event(event_t e) /* End of code copied from the original Linux port. */ uint -os_random_seed() +os_random_seed(void) { /* Return the low 32 bits of the cycle count. */ - return (uint)get_cycle_count(); + uint64 cycles = 0; + RDTSC_LL(cycles); + return (uint)cycles; } void @@ -2555,7 +2549,7 @@ os_check_option_compatibility(void) } void -report_diagnostics(IN const char *message, IN const char *name, +report_diagnostics(DR_PARAM_IN const char *message, DR_PARAM_IN const char *name, security_violation_t violation_type) { /* Not implemented in the original Linux version. */ @@ -2614,13 +2608,13 @@ signal_thread_inherit(dcontext_t *dcontext, void *clone_record) } void -thread_set_self_context(void *cxt) +thread_set_self_context(void *cxt, bool is_detach_external) { ASSERT_NOT_PORTED(false); } void -thread_set_self_mcontext(dr_mcontext_t *mc) +thread_set_self_mcontext(priv_mcontext_t *mc, bool is_detach_external) { ASSERT_NOT_PORTED(false); } @@ -2639,7 +2633,7 @@ receive_pending_signal(dcontext_t *dcontext) } void -os_forge_exception(app_pc target_pc, exception_type_t type) +os_forge_exception(app_pc target_pc, dr_exception_type_t type) { ASSERT_NOT_PORTED(false); } @@ -2653,7 +2647,8 @@ os_dump_core(const char *msg) bool set_itimer_callback(dcontext_t *dcontext, int which, uint millisec, - void (*func)(dcontext_t *, dr_mcontext_t *)) + void (*func)(dcontext_t *, priv_mcontext_t *), + void (*func_api)(dcontext_t *, dr_mcontext_t *)) { ASSERT_NOT_PORTED(false); return true; diff --git a/core/kernel_linux/os_exports.h b/core/kernel_linux/os_exports.h index 7ac57c020..3b3d599e2 100644 --- a/core/kernel_linux/os_exports.h +++ b/core/kernel_linux/os_exports.h @@ -70,6 +70,14 @@ # define ASM_SEG "%fs" #endif +#define LIB_SEG_TLS DR_REG_NULL + +ushort +os_get_app_tls_base_offset(reg_id_t reg); + +ushort +os_get_app_tls_reg_offset(reg_id_t reg); + void * get_tls(tls_offset_t tls_offs); void @@ -88,6 +96,8 @@ thread_id_t get_sys_thread_id(void); bool is_thread_terminated(dcontext_t *dcontext); +bool +os_wait_thread_terminated(dcontext_t *dcontext); void os_tls_pre_init(int gdt_index); @@ -174,6 +184,8 @@ extern app_pc vsyscall_page_start; extern app_pc vsyscall_syscall_end_pc; /* pc where kernel returns control after sysenter vsyscall */ extern app_pc vsyscall_sysenter_return_pc; +/* pc where our hook-displaced code was copied */ +extern app_pc vsyscall_sysenter_displaced_pc; #define VSYSCALL_PAGE_MAPS_NAME "[vdso]" bool @@ -184,8 +196,6 @@ bool is_sigreturn_syscall(dcontext_t *dcontext); bool was_sigreturn_syscall(dcontext_t *dcontext); -bool -ignorable_system_call(int num); bool kernel_is_64bit(void); @@ -284,7 +294,8 @@ signal_fork_init(dcontext_t *dcontext); bool set_itimer_callback(dcontext_t *dcontext, int which, uint millisec, - void (*func)(dcontext_t *, dr_mcontext_t *)); + void (*func)(dcontext_t *, priv_mcontext_t *), + void (*func_api)(dcontext_t *, dr_mcontext_t *)); uint get_itimer_frequency(dcontext_t *dcontext, int which); diff --git a/core/kernel_linux/segment_descriptor.h b/core/kernel_linux/segment_descriptor.h index 7daf5d3d8..bb0cff09f 100644 --- a/core/kernel_linux/segment_descriptor.h +++ b/core/kernel_linux/segment_descriptor.h @@ -56,7 +56,7 @@ typedef enum { typedef struct { union { struct { - uint32 requestor_privilige_level : 2; + uint32 requestor_privilege_level : 2; table_indicator_t table_indicator : 1; uint32 index : 13; } __attribute__((__packed__)); @@ -70,7 +70,7 @@ ASSERT_TYPE_SIZE(2, segment_selector_t); static inline void segment_selector_decode(int selector, segment_selector_t *output) { - output->requestor_privilige_level = selector & 0x3; + output->requestor_privilege_level = selector & 0x3; output->table_indicator = (selector & 0x4) >> 2; output->index = (selector & 0xfff8) >> 3; } diff --git a/core/lib/dr_app.h b/core/lib/dr_app.h index 058af34de..bf50a1658 100644 --- a/core/lib/dr_app.h +++ b/core/lib/dr_app.h @@ -47,7 +47,8 @@ # define DR_APP_API __declspec(dllimport) # endif #else /* UNIX */ -# if defined(DR_APP_EXPORTS) && defined(USE_VISIBILITY_ATTRIBUTES) +# if defined(DR_APP_EXPORTS) && defined(USE_VISIBILITY_ATTRIBUTES) && \ + !defined(LINUX_KERNEL) # define DR_APP_API __attribute__((visibility("default"))) # else # define DR_APP_API diff --git a/core/lib/dr_events.h b/core/lib/dr_events.h index ea5ce1fac..d4aa3c57b 100644 --- a/core/lib/dr_events.h +++ b/core/lib/dr_events.h @@ -808,6 +808,30 @@ bool dr_unregister_restore_state_ex_event(bool (*func)(void *drcontext, bool restore_memory, dr_restore_state_info_t *info)); +#ifdef LINUX_KERNEL +typedef struct _dr_interrupt_t { + interrupt_stack_frame_t *frame; + interrupt_stack_frame_t *raw_frame; + interrupt_vector_t vector; + dr_mcontext_t *mcontext; +} dr_interrupt_t; + +DR_API +/** + * Registers a callback function for kernel interrupt events. + */ +void +dr_register_interrupt_event(bool (*func)(void *drcontext, dr_interrupt_t *interrupt)); + +DR_API +/** + * Unregister a callback function for kernel interrupt events. \return true if + * unregistration is successful and false if it is not (e.g., \p func was not registered). + */ +bool +dr_unregister_interrupt_event(bool (*func)(void *drcontext, dr_interrupt_t *interrupt)); +#endif + DR_API /** * Registers a callback function for the thread initialization event. @@ -1478,7 +1502,7 @@ DR_API * whenever virtual memory is tight and enables the client to help free space. */ void -dr_register_low_on_memory_event(void (*func)()); +dr_register_low_on_memory_event(void (*func)(void)); DR_API /** @@ -1487,7 +1511,7 @@ DR_API * (e.g., the function was not registered). */ bool -dr_unregister_low_on_memory_event(void (*func)()); +dr_unregister_low_on_memory_event(void (*func)(void)); DR_API /** diff --git a/core/lib/globals_api.h b/core/lib/globals_api.h index 799cce2b4..415d51ca5 100644 --- a/core/lib/globals_api.h +++ b/core/lib/globals_api.h @@ -52,10 +52,17 @@ # include # include #else -# include -# include +# ifndef LINUX_KERNEL +# include +# include +# endif +#endif + +#ifdef LINUX_KERNEL +# include /* for varargs */ +#else +# include /* for varargs */ #endif -#include /* for varargs */ #ifndef DYNAMORIO_INTERNAL /* A client's target operating system and architecture must be specified. */ @@ -105,7 +112,7 @@ * behave properly. It indents the guard. There seems to be no workaround. */ /* clang-format off */ -# ifndef __cplusplus +# if !defined(__cplusplus) && !defined(LINUX_KERNEL) # ifdef WINDOWS # define inline __inline # else @@ -129,7 +136,11 @@ typedef char bool; # endif # ifdef UNIX -# include /* for pid_t (non-glibc, e.g. musl) */ +# ifdef LINUX_KERNEL +# include +# else +# include /* for pid_t (non-glibc, e.g. musl) */ +# endif # endif # ifdef WINDOWS /* allow nameless struct/union */ @@ -153,7 +164,7 @@ typedef char bool; * (in gcc >= 3.4) to not export symbols by default, setting * USE_VISIBILITY_ATTRIBUTES will properly export. */ -# ifdef USE_VISIBILITY_ATTRIBUTES +# if defined(USE_VISIBILITY_ATTRIBUTES) && !defined(LINUX_KERNEL) # define DR_EXPORT __attribute__((visibility("default"))) # else # define DR_EXPORT @@ -162,7 +173,7 @@ typedef char bool; # define ALIGN_VAR(x) __attribute__((aligned(x))) # define INLINE_FORCED inline # define WEAK __attribute__((weak)) -# define NOINLINE __attribute__((noinline)) +# define NOINLINE __attribute__((__noinline__)) #endif /* We want a consistent size so we stay away from MAX_PATH. @@ -209,7 +220,7 @@ typedef signed char sbyte; #endif typedef byte *app_pc; -typedef void (*generic_func_t)(); +typedef void (*generic_func_t)(void); #ifdef DR_DEFINE_FOR_uint64 # undef DR_DO_NOT_DEFINE_uint64 diff --git a/core/lib/globals_shared.h b/core/lib/globals_shared.h index c048249e7..a27f834dd 100644 --- a/core/lib/globals_shared.h +++ b/core/lib/globals_shared.h @@ -76,9 +76,14 @@ #include "globals_api.h" // IWYU pragma: export -#include /* for USHRT_MAX */ +#include "../limits_wrapper.h" /* for USHRT_MAX */ + #ifdef UNIX -# include +# ifdef LINUX_KERNEL +# include +# else +# include +# endif #endif #include "c_defines.h" // IWYU pragma: export @@ -954,9 +959,9 @@ typedef struct { uint flags : 2; int ignored2; /* siginfo_t.si_code: has meaning to kernel so we avoid using */ #else - uint version; /* version number for future proofing */ - uint nudge_action_mask; /* drawn from NUDGE_DEFS above */ - uint flags; /* flags drawn from above enum */ + uint version; /* version number for future proofing */ + uint nudge_action_mask; /* drawn from NUDGE_DEFS above */ + uint flags; /* flags drawn from above enum */ #endif client_id_t client_id; /* unique ID identifying client */ uint64 client_arg; /* argument for a client nudge */ @@ -1109,4 +1114,14 @@ typedef struct _priv_mcontext_t { #include "mcxtx_api.h" // IWYU pragma: export } priv_mcontext_t; +#ifdef __has_attribute +# if __has_attribute(__fallthrough__) +# define DR_FALLTHROUGH __attribute__((__fallthrough__)) +# else +# define DR_FALLTHROUGH ((void)0) /* fallthrough */ +# endif +#else +# define DR_FALLTHROUGH ((void)0) /* fallthrough */ +#endif + #endif /* _GLOBALS_SHARED_H_ */ diff --git a/core/lib/instrument.c b/core/lib/instrument.c index 1e220f75a..232836c77 100644 --- a/core/lib/instrument.c +++ b/core/lib/instrument.c @@ -54,13 +54,13 @@ #include "../fcache.h" #include "../emit.h" #include "../link.h" -#include "../monitor.h" /* for mark_trace_head */ -#include /* for varargs */ -#include "../nudge.h" /* for nudge_internal() */ +#include "../monitor.h" /* for mark_trace_head */ +#include "../stdarg_wrapper.h" /* for varargs */ +#include "../nudge.h" /* for nudge_internal() */ #include "../synch.h" #include "../annotations.h" #include "../translate.h" -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) # include /* ITIMER_* */ # include "../unix/module.h" /* redirect_* functions */ #endif @@ -252,9 +252,15 @@ static callback_list_t pre_syscall_callbacks = { static callback_list_t post_syscall_callbacks = { 0, }; +#ifndef LINUX_KERNEL static callback_list_t kernel_xfer_callbacks = { 0, }; +#else +static callback_list_t interrupt_callbacks = { + 0, +}; +#endif #ifdef WINDOWS static callback_list_t exception_callbacks = { 0, @@ -851,7 +857,7 @@ free_callback_list(callback_list_t *vec) } static void -free_all_callback_lists() +free_all_callback_lists(void) { free_callback_list(&exit_callbacks); free_callback_list(&post_attach_callbacks); @@ -873,7 +879,11 @@ free_all_callback_lists() free_callback_list(&filter_syscall_callbacks); free_callback_list(&pre_syscall_callbacks); free_callback_list(&post_syscall_callbacks); +#ifndef LINUX_KERNEL free_callback_list(&kernel_xfer_callbacks); +#else + free_callback_list(&interrupt_callbacks); +#endif #ifdef WINDOWS free_callback_list(&exception_callbacks); #else @@ -903,22 +913,19 @@ instrument_exit_event(void) /* support dr_get_mcontext() from the exit event */ if (!standalone_library) get_thread_private_dcontext()->client_data->mcontext_in_dcontext = true; - call_all(exit_callbacks, int (*)(), - /* It seems the compiler is confused if we pass no var args - * to the call_all macro. Bogus NULL arg */ - NULL); + call_all(exit_callbacks, int (*)(void)); } void instrument_post_attach_event(void) { - call_all(post_attach_callbacks, int (*)(), NULL); + call_all(post_attach_callbacks, int (*)(void)); } void instrument_pre_detach_event(void) { - call_all(pre_detach_callbacks, int (*)(), NULL); + call_all(pre_detach_callbacks, int (*)(void)); } void @@ -1212,13 +1219,13 @@ dr_unregister_fork_init_event(void (*func)(void *drcontext)) #endif void -dr_register_low_on_memory_event(void (*func)()) +dr_register_low_on_memory_event(void (*func)(void)) { add_callback(&low_on_memory_callbacks, (void (*)(void))func, true); } bool -dr_unregister_low_on_memory_event(void (*func)()) +dr_unregister_low_on_memory_event(void (*func)(void)) { return remove_callback(&low_on_memory_callbacks, (void (*)(void))func, true); } @@ -1314,6 +1321,7 @@ dr_unregister_post_syscall_event(void (*func)(void *drcontext, int sysnum)) return remove_callback(&post_syscall_callbacks, (void (*)(void))func, true); } +#ifndef LINUX_KERNEL void dr_register_kernel_xfer_event(void (*func)(void *drcontext, const dr_kernel_xfer_info_t *info)) @@ -1327,6 +1335,7 @@ dr_unregister_kernel_xfer_event(void (*func)(void *drcontext, { return remove_callback(&kernel_xfer_callbacks, (void (*)(void))func, true); } +#endif #ifdef PROGRAM_SHEPHERDING void @@ -1415,13 +1424,13 @@ dr_nudge_client_ex(process_id_t process_id, client_id_t client_id, uint64 argume if (client_libs[i].id == client_id) { if (client_libs[i].nudge_callbacks.num == 0) { CLIENT_ASSERT(false, "dr_nudge_client: no nudge handler registered"); - return false; + return (dr_config_status_t) false; } return nudge_internal(process_id, NUDGE_GENERIC(client), argument, client_id, timeout_ms); } } - return false; + return (dr_config_status_t) false; } else { return nudge_internal(process_id, NUDGE_GENERIC(client), argument, client_id, timeout_ms); @@ -1521,9 +1530,9 @@ instrument_fork_init(dcontext_t *dcontext) #endif void -instrument_low_on_memory() +instrument_low_on_memory(void) { - call_all(low_on_memory_callbacks, int (*)()); + call_all(low_on_memory_callbacks, int (*)(void)); } /* PR 536058: split the exit event from thread cleanup, to provide a @@ -1950,6 +1959,39 @@ instrument_restore_nonfcache_state(dcontext_t *dcontext, bool restore_memory, &client_mcontext); } +#ifdef LINUX_KERNEL +void +dr_register_interrupt_event(bool (*func)(void *dcontext, dr_interrupt_t *interrupt)) +{ + if (func == NULL) { + CLIENT_ASSERT(false, "trying to register a NULL callback"); + return; + } + if (!INTERNAL_OPTION(code_api)) { + CLIENT_ASSERT(false, "asking for interrupt event when code_api disabled"); + return; + } + add_callback(&interrupt_callbacks, (void (*)(void))func, true); +} + +bool +dr_unregister_interrupt_event(bool (*func)(void *drcontext, dr_interrupt_t *interrupt)) +{ + return remove_callback(&interrupt_callbacks, (void (*)(void))func, true); +} + +bool +instrument_interrupt(dcontext_t *dcontext, dr_interrupt_t *interrupt) +{ + bool res = true; + if (interrupt_callbacks.num > 0) { + call_all_ret(res, = res &&, , interrupt_callbacks, + bool (*)(void *, dr_interrupt_t *), (void *)dcontext, interrupt); + } + return res; +} +#endif + /* Ask whether to end trace prior to adding next_tag fragment. * Return values: * CUSTOM_TRACE_DR_DECIDES = use standard termination criteria @@ -1987,7 +2029,7 @@ instrument_module_load_trigger(app_pc pc) if (ma != NULL && !TEST(MODULE_LOAD_EVENT, ma->flags)) { /* switch to write lock */ os_get_module_info_unlock(); -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) /* i#3385: re-try to initialize dynamic information, because * it failed during the first flat-mmap that loaded the module. * We don't perform this if there are no clients, assuming @@ -2131,6 +2173,7 @@ instrument_invoke_another_syscall(dcontext_t *dcontext) return dcontext->client_data->invoke_another_syscall; } +#ifndef LINUX_KERNEL bool instrument_kernel_xfer(dcontext_t *dcontext, dr_kernel_xfer_type_t type, os_cxt_ptr_t source_os_cxt, dr_mcontext_t *source_dmc, @@ -2172,6 +2215,7 @@ instrument_kernel_xfer(dcontext_t *dcontext, dr_kernel_xfer_type_t type, dcontext->client_data->cur_mc = NULL; return true; } +#endif #ifdef WINDOWS /* Notify user of exceptions. Note: not called for RaiseException */ @@ -2513,7 +2557,10 @@ dr_exit_process(int exit_code) } #endif if (!is_currently_on_dstack(dcontext) - IF_UNIX(&&!is_currently_on_sigaltstack(dcontext))) { +#if defined(UNIX) && !defined(LINUX_KERNEL) + && !is_currently_on_sigaltstack(dcontext) +#endif + ) { /* if on app stack or sigaltstack, avoid incorrect leak assert at exit */ SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); dr_api_exit = true; @@ -2534,7 +2581,7 @@ dr_create_memory_dump(dr_memory_dump_spec_t *spec) if (TEST(DR_MEMORY_DUMP_LDMP, spec->flags)) return os_dump_core_live(spec->label, spec->ldmp_path, spec->ldmp_path_size); /* XXX i#2154: Add Android AArch64 support. */ -#elif defined(LINUX) && \ +#elif defined(LINUX) && !defined(LINUX_KERNEL) && \ ((defined(X64) && defined(X86)) || (defined(AARCH64) && !defined(ANDROID64))) if (TEST(DR_MEMORY_DUMP_ELF, spec->flags)) { return os_dump_core_live(get_thread_private_dcontext(), @@ -6204,10 +6251,10 @@ dr_insert_cbr_instrumentation_help(void *drcontext, instrlist_t *ilist, instr_t app_flags_ok = instr_get_prev(instr); if (has_fallthrough) { - ptr_uint_t fallthrough = address + instr_length(drcontext, instr); + ptr_uint_t fallthrough_addr = address + instr_length(drcontext, instr); CLIENT_ASSERT(!opnd_uses_reg(user_data, DR_REG_XBX), "register ebx should not be used"); - CLIENT_ASSERT(fallthrough > address, "wrong fallthrough address"); + CLIENT_ASSERT(fallthrough_addr > address, "wrong fallthrough address"); dr_insert_clean_call_ex( drcontext, ilist, instr, callee, /* Many users will ask for mcontexts; some will set; it doesn't seem worth @@ -6220,7 +6267,7 @@ dr_insert_cbr_instrumentation_help(void *drcontext, instrlist_t *ilist, instr_t /* target is 2nd parameter */ OPND_CREATE_INTPTR(target), /* fall-throug is 3rd parameter */ - OPND_CREATE_INTPTR(fallthrough), + OPND_CREATE_INTPTR(fallthrough_addr), /* branch direction (put in ebx below) is 4th parameter */ opnd_create_reg(REG_XBX), /* user defined data is 5th parameter */ @@ -6506,8 +6553,8 @@ dr_insert_cbr_instrumentation_help(void *drcontext, instrlist_t *ilist, instr_t } if (has_fallthrough) { - ptr_uint_t fallthrough = address + instr_length(drcontext, instr); - CLIENT_ASSERT(fallthrough > address, "wrong fallthrough address"); + ptr_uint_t fallthrough_addr = address + instr_length(drcontext, instr); + CLIENT_ASSERT(fallthrough_addr > address, "wrong fallthrough address"); dr_insert_clean_call_ex( drcontext, ilist, instr, callee, /* Many users will ask for mcontexts; some will set; it doesn't seem worth @@ -6520,7 +6567,7 @@ dr_insert_cbr_instrumentation_help(void *drcontext, instrlist_t *ilist, instr_t /* Target is 2nd parameter. */ OPND_CREATE_INTPTR(target), /* Fall-through is 3rd parameter. */ - OPND_CREATE_INTPTR(fallthrough), + OPND_CREATE_INTPTR(fallthrough_addr), /* Branch direction is 4th parameter. */ opnd_create_reg(dir), /* User defined data is 5th parameter. */ @@ -6677,9 +6724,11 @@ dr_get_mcontext_priv(dcontext_t *dcontext, dr_mcontext_t *dmc, priv_mcontext_t * return true; } +#ifndef LINUX_KERNEL if (!is_os_cxt_ptr_null(dcontext->client_data->os_cxt)) { return os_context_to_mcontext(dmc, mc, dcontext->client_data->os_cxt); } +#endif if (dcontext->client_data->suspended) { /* A thread suspended by dr_suspend_all_other_threads() has its @@ -6794,6 +6843,7 @@ dr_set_mcontext(void *drcontext, dr_mcontext_t *context) if (dcontext->client_data->cur_mc != NULL) { return dr_mcontext_to_priv_mcontext(dcontext->client_data->cur_mc, context); } +#ifndef LINUX_KERNEL if (!is_os_cxt_ptr_null(dcontext->client_data->os_cxt)) { /* It would be nice to fail for #DR_XFER_CALLBACK_RETURN but we'd need to * store yet more state to do so. The pc will be ignored, and xsi @@ -6801,6 +6851,7 @@ dr_set_mcontext(void *drcontext, dr_mcontext_t *context) */ return mcontext_to_os_context(dcontext->client_data->os_cxt, context, NULL); } +#endif /* copy the machine context to the dstack area created with * dr_prepare_for_call(). note that xmm0-5 copied there @@ -6866,6 +6917,7 @@ dr_redirect_execution(dr_mcontext_t *mcontext) dcontext->next_tag = canonicalize_pc_target(dcontext, mcontext->pc); dcontext->whereami = DR_WHERE_FCACHE; set_last_exit(dcontext, (linkstub_t *)get_client_linkstub()); +#ifndef LINUX_KERNEL if (kernel_xfer_callbacks.num > 0) { /* This can only be called from a clean call or an exception event. * For both of those we can get the current mcontext via dr_get_mcontext() @@ -6880,6 +6932,7 @@ dr_redirect_execution(dr_mcontext_t *mcontext) dr_mcontext_as_priv_mcontext(mcontext), 0)) dcontext->next_tag = canonicalize_pc_target(dcontext, mcontext->pc); } +#endif transfer_to_dispatch(dcontext, dr_mcontext_as_priv_mcontext(mcontext), true /*full_DR_state*/); /* on success we won't get here */ @@ -7560,7 +7613,9 @@ dr_insert_get_seg_base(void *drcontext, instrlist_t *ilist, instr_t *instr, reg_ #ifdef X86 CLIENT_ASSERT(reg_is_segment(seg), "dr_insert_get_seg_base: seg is not a segment register"); -# ifdef UNIX +# ifdef LINUX_KERNEL + return false; +# elif defined(UNIX) # ifndef MACOS64 CLIENT_ASSERT(INTERNAL_OPTION(mangle_app_seg), "dr_insert_get_seg_base is supported with -mangle_app_seg only"); @@ -7599,7 +7654,7 @@ dr_insert_get_seg_base(void *drcontext, instrlist_t *ilist, instr_t *instr, reg_ INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(reg), OPND_CREATE_INTPTR(0))); } else return false; -# endif /* UNIX/Windows */ +# endif /* LINUX_KERNEL/UNIX/Windows */ #elif defined(ARM) /* i#1551: NYI on ARM */ ASSERT_NOT_IMPLEMENTED(false); @@ -7732,7 +7787,7 @@ dr_prepopulate_cache(app_pc *tags, size_t tags_count) /* There could be duplicates if sthg was deleted and re-added during profiling */ fragment_t coarse_f; fragment_t *f; -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) /* We silently skip DR-segment-reading addresses to help out a caller * who sampled and couldn't avoid self-sampling for decoding. */ diff --git a/core/lib/instrument.h b/core/lib/instrument.h index c5bd4989f..a58394e15 100644 --- a/core/lib/instrument.h +++ b/core/lib/instrument.h @@ -118,7 +118,10 @@ bool instrument_restore_nonfcache_state_prealloc(dcontext_t *dcontext, bool restore_memory, DR_PARAM_INOUT priv_mcontext_t *mcontext, DR_PARAM_OUT dr_mcontext_t *client_mcontext); - +#ifdef LINUX_KERNEL +bool +instrument_interrupt(dcontext_t *dcontext, dr_interrupt_t *interrupt); +#endif module_data_t * copy_module_area_to_module_data(const module_area_t *area); void @@ -139,7 +142,8 @@ instrument_post_syscall(dcontext_t *dcontext, int sysnum); bool instrument_invoke_another_syscall(dcontext_t *dcontext); void -instrument_low_on_memory(); +instrument_low_on_memory(void); +#ifndef LINUX_KERNEL /* returns whether a client event was called which might have changed the context */ bool instrument_kernel_xfer(dcontext_t *dcontext, dr_kernel_xfer_type_t type, @@ -148,6 +152,7 @@ instrument_kernel_xfer(dcontext_t *dcontext, dr_kernel_xfer_type_t type, priv_mcontext_t *source_mc, app_pc target_pc, reg_t target_xsp, /* only one of these 2 should be non-NULL */ os_cxt_ptr_t target_os_cxt, priv_mcontext_t *target_mc, int sig); +#endif void instrument_nudge(dcontext_t *dcontext, client_id_t id, uint64 arg); diff --git a/core/lib/kstatsx.h b/core/lib/kstatsx.h index a0c035289..3f62cd57b 100644 --- a/core/lib/kstatsx.h +++ b/core/lib/kstatsx.h @@ -141,6 +141,37 @@ KSTAT_DEF("in rct analysis using relocations [per page loop]", rct_reloc_per_pag KSTAT_DEF("in aslr_generate_relocated_section for validation", aslr_validate_relocate) KSTAT_DEF("in module_contents_compare or aslr_compare_in_place", aslr_compare) +#ifdef LINUX_KERNEL +KSTAT_DEF("delaying patched interrupt", delaying_patched_interrupt) +KSTAT_DEF("user interrupt handling", user_interrupt_handling) +KSTAT_DEF("kernel interrupt handling", kernel_interrupt_handling) +KSTAT_DEF("kernel interrupt fcache enter", kernel_interrupt_fcache_enter) +KSTAT_DEF("kernel interrupt fcache return", kernel_interrupt_fcache_return) +KSTAT_DEF("kernel interrupt ibl", kernel_interrupt_ibl) +KSTAT_DEF("kernel interrupt frag success page fault", + kernel_interrupt_frag_success_page_fault) +KSTAT_DEF("kernel interrupt frag success other sync", + kernel_interrupt_frag_success_other_sync) +KSTAT_DEF("kernel interrupt frag success async", kernel_interrupt_frag_success_async) +KSTAT_DEF("kernel interrupt frag delay dispatch", kernel_interrupt_frag_delay_dispatch) +KSTAT_DEF("kernel interrupt frag delay pc", kernel_interrupt_frag_delay_pc) +KSTAT_DEF("kernel interrupt frag unknown", kernel_interrupt_frag_unknown) +KSTAT_DEF("user mode", usermode) +KSTAT_DEF("recreate app state", recreate_app_state_failure) +KSTAT_DEF("recreate app state", recreate_app_state_success_pc) +KSTAT_DEF("recreate app state", recreate_app_state_success_state) +KSTAT_DEF("recreate app state", recreate_app_state_delay_dispatch) +KSTAT_DEF("recreate app state", recreate_app_state_delay_pc) +KSTAT_DEF("recreate app state", recreate_app_state_unknown) +KSTAT_DEF("recreate app state", recreate_app_state_frag_intr) +KSTAT_DEF("recreate app state", recreate_app_state_from_info) +KSTAT_DEF("recreate app state", recreate_app_state_in_fcache) +KSTAT_DEF("recreate app state", recreate_fragment_lookup) +KSTAT_DEF("recreate app state", recreate_fragment_ilist) +KSTAT_DEF("recreate app state", recreate_app_state_from_ilist) +KSTAT_DEF("fragment interrupt handling", fragment_interrupt_handling) +#endif /* LINUX_KERNEL */ + #ifdef KSTAT_UNIT_TEST KSTAT_DEF("empty block overhead", empty) KSTAT_DEF("total measured", measured) diff --git a/core/lib/statsx.h b/core/lib/statsx.h index 2a4af4ee5..2d8aeff4a 100644 --- a/core/lib/statsx.h +++ b/core/lib/statsx.h @@ -686,6 +686,7 @@ STATS_DEF("Fcache exits, fine targeting non-th coarse", num_exits_fine2non_th_co STATS_DEF("Fcache exits, system call executions", num_exits_syscalls) STATS_DEF("Fcache exits, flushed due to code mod", num_exits_code_mod_flush) STATS_DEF("Fcache exits, deleted but hit in ibl", num_exits_ibl_deleted) +STATS_DEF("Fcache exits, interrupt in ibl hit in ibl", num_exits_ibl_unlinked_found) STATS_DEF("Fcache exits, asynch", num_exits_asynch) STATS_DEF("Fcache exits, native_exec executions", num_exits_native_exec) STATS_DEF("Fcache exits, native_exec syscalls", num_exits_native_exec_syscall) diff --git a/core/limits_wrapper.h b/core/limits_wrapper.h index c61087e50..e32c3afef 100644 --- a/core/limits_wrapper.h +++ b/core/limits_wrapper.h @@ -1,31 +1,35 @@ -#ifndef __LIMITS_H_ -#define __LIMITS_H_ -/* Copied from /usr/include/limits.h */ -/* We don't have #include_next. - Define ANSI for standard 32-bit words. */ +#ifndef __LIMITS_WRAPPER_H_ +#define __LIMITS_WRAPPER_H_ -/* These assume 8-bit `char's, 16-bit `short int's, - and 32-bit `int's and `long int's. */ +#include "configure.h" -#include +#ifdef LINUX_KERNEL +/* The Linux kernel does not have a standard limits.h for basic C types, + * but it defines some integer limits in . We include that + * and manually define the standard C char/byte limits here. + */ +# include /* Number of bits in a `char'. */ -#define CHAR_BIT 8 +# define CHAR_BIT 8 /* Minimum and maximum values a `signed char' can hold. */ -#define SCHAR_MIN (-128) -#define SCHAR_MAX 127 +# define SCHAR_MIN (-128) +# define SCHAR_MAX 127 /* Maximum value an `unsigned char' can hold. (Minimum is 0.) */ -#define UCHAR_MAX 255 +# define UCHAR_MAX 255 /* Minimum and maximum values a `char' can hold. */ -#ifdef __CHAR_UNSIGNED__ -# define CHAR_MIN 0 -# define CHAR_MAX UCHAR_MAX +# ifdef __CHAR_UNSIGNED__ +# define CHAR_MIN 0 +# define CHAR_MAX UCHAR_MAX +# else +# define CHAR_MIN SCHAR_MIN +# define CHAR_MAX SCHAR_MAX +# endif #else -# define CHAR_MIN SCHAR_MIN -# define CHAR_MAX SCHAR_MAX +# include #endif -#endif +#endif /* __LIMITS_WRAPPER_H_ */ diff --git a/core/link.c b/core/link.c index 8371ece84..a4daaf2da 100644 --- a/core/link.c +++ b/core/link.c @@ -161,6 +161,7 @@ static #endif const linkstub_t linkstub_selfmod = { LINK_FAKE, 0 }; static const linkstub_t linkstub_ibl_deleted = { LINK_FAKE, 0 }; +static const linkstub_t linkstub_ibl_unlinked_found = { LINK_FAKE, 0 }; static const linkstub_t linkstub_asynch = { LINK_FAKE, 0 }; static const linkstub_t linkstub_native_exec = { LINK_FAKE, 0 }; /* this one we give the flag LINK_NI_SYSCALL for executing a syscall in d_r_dispatch() */ @@ -187,6 +188,12 @@ static const linkstub_t linkstub_shared_syscall_bb = { static const linkstub_t linkstub_shared_syscall_unlinked = { LINK_FAKE, 0 }; #endif +#ifdef LINUX_KERNEL +static const linkstub_t linkstub_syscall_entry = { LINK_FAKE, 0 }; +static const linkstub_t linkstub_kernel_interrupt_entry = { LINK_FAKE, 0 }; +static const linkstub_t linkstub_user_interrupt_entry = { LINK_FAKE, 0 }; +#endif + /* A unique fragment_t for use when the details don't matter */ static const fragment_t coarse_fragment = { NULL, @@ -739,6 +746,12 @@ get_ibl_deleted_linkstub() return &linkstub_ibl_deleted; } +const linkstub_t * +get_ibl_unlinked_found_linkstub() +{ + return &linkstub_ibl_unlinked_found; +} + const linkstub_t * get_asynch_linkstub() { @@ -777,6 +790,26 @@ get_shared_syscalls_bb_linkstub() } #endif /* WINDOWS */ +#ifdef LINUX_KERNEL +const linkstub_t * +get_syscall_entry_linkstub() +{ + return &linkstub_syscall_entry; +} + +const linkstub_t * +get_user_interrupt_entry_linkstub() +{ + return &linkstub_user_interrupt_entry; +} + +const linkstub_t * +get_kernel_interrupt_entry_linkstub() +{ + return &linkstub_kernel_interrupt_entry; +} +#endif + #ifdef HOT_PATCHING_INTERFACE const linkstub_t * get_hot_patch_linkstub() @@ -797,7 +830,7 @@ is_ibl_sourceless_linkstub(const linkstub_t *l) return (l == &linkstub_ibl_trace_ret || l == &linkstub_ibl_trace_jmp || l == &linkstub_ibl_trace_call || l == &linkstub_ibl_bb_ret || l == &linkstub_ibl_bb_jmp || l == &linkstub_ibl_bb_call || - is_special_ibl_linkstub(l)); + l == &linkstub_ibl_unlinked_found || is_special_ibl_linkstub(l)); } const linkstub_t * diff --git a/core/link.h b/core/link.h index a359c668c..72ffd96d5 100644 --- a/core/link.h +++ b/core/link.h @@ -497,6 +497,8 @@ extern const linkstub_t linkstub_selfmod; #endif const linkstub_t * get_ibl_deleted_linkstub(void); +const linkstub_t * +get_ibl_unlinked_found_linkstub(void); /* This is used for Windows APC, callback, etc. and Linux sigreturn, forge fault, etc. */ const linkstub_t * get_asynch_linkstub(void); @@ -546,6 +548,18 @@ get_shared_syscalls_bb_linkstub(void); # define IS_SHARED_SYSCALLS_TRACE_LINKSTUB(l) false #endif +#ifdef LINUX_KERNEL +const linkstub_t * +get_syscall_entry_linkstub(void); +const linkstub_t * +get_user_interrupt_entry_linkstub(void); +const linkstub_t * +get_kernel_interrupt_entry_linkstub(void); +# define IS_KERNEL_ENTRY_LINKSTUB(l) \ + ((l) == get_syscall_entry_linkstub() || \ + (l) == get_user_interrupt_entry_linkstub()) +#endif + bool should_separate_stub(dcontext_t *dcontext, app_pc target, uint fragment_flags); int diff --git a/core/module_shared.h b/core/module_shared.h index 0ad2bf92d..1934d576b 100644 --- a/core/module_shared.h +++ b/core/module_shared.h @@ -565,7 +565,7 @@ os_loader_init_prologue(void); * will release privload_lock. */ void -os_loader_init_epilogue(); +os_loader_init_epilogue(void); void os_loader_exit(void); diff --git a/core/nudge.c b/core/nudge.c index 789e396e6..2358f0dfc 100644 --- a/core/nudge.c +++ b/core/nudge.c @@ -586,10 +586,14 @@ nudge_internal(process_id_t pid, uint nudge_action_mask, uint64 client_arg, nudge_add_pending(dcontext, &nudge_arg); return DR_SUCCESS; } else { +# ifndef LINUX_KERNEL if (send_nudge_signal(pid, nudge_action_mask, client_id, client_arg)) return DR_SUCCESS; else return DR_FAILURE; +# else + return DR_FAILURE; +# endif } #endif /* WINDOWS -> UNIX */ } diff --git a/core/options.c b/core/options.c index fc6833aad..c9a6ebaa1 100644 --- a/core/options.c +++ b/core/options.c @@ -40,7 +40,7 @@ * */ -#include +#include "stddef_wrapper.h" #ifndef NOT_DYNAMORIO_CORE # include "globals.h" @@ -675,9 +675,6 @@ set_dynamo_options_common(options_t *options, const char *optstr, bool for_this_ char *opt; const char *pos = optstr; bool got_badopt = false; - char badopt[MAX_OPTION_LENGTH]; - - char wordbuffer[MAX_OPTION_LENGTH]; /* used in the OPTION_COMMAND define above, declared here to save stack * space XXX : value_true and value_false could be static const if @@ -691,6 +688,16 @@ set_dynamo_options_common(options_t *options, const char *optstr, bool for_this_ ASSERT_OWN_OPTIONS_LOCK(options == &dynamo_options || options == &temp_options, &options_lock); ASSERT(!OPTIONS_PROTECTED()); + +#ifdef LINUX_KERNEL + /* Kernel stack frames are limited to 4096 bytes; use static (safe under + * options_lock). */ + static char badopt[MAX_OPTION_LENGTH]; + static char wordbuffer[MAX_OPTION_LENGTH]; +#else + char badopt[MAX_OPTION_LENGTH]; + char wordbuffer[MAX_OPTION_LENGTH]; +#endif while ((opt = getword(optstr, &pos, wordbuffer, sizeof(wordbuffer))) != NULL) { if (opt[0] == '-') { value = NULL; @@ -2427,7 +2434,7 @@ check_option_compatibility_helper(int recurse_count) /* returns true if changed any options */ static bool -check_option_compatibility() +check_option_compatibility(void) { ASSERT_OWN_OPTIONS_LOCK(true, &options_lock); ASSERT(!OPTIONS_PROTECTED()); @@ -2436,7 +2443,7 @@ check_option_compatibility() /* returns true if changed any options */ static bool -check_dynamic_option_compatibility() +check_dynamic_option_compatibility(void) { ASSERT_OWN_OPTIONS_LOCK(true, &options_lock); /* NOTE : use non-synch form of USAGE_ERROR in here to avoid diff --git a/core/options.h b/core/options.h index 90d65abc4..594ffb65b 100644 --- a/core/options.h +++ b/core/options.h @@ -235,12 +235,12 @@ extern read_write_lock_t options_lock; /* full access to string requires read lock */ static inline void -string_option_read_lock() +string_option_read_lock(void) { d_r_read_lock(&options_lock); } static inline void -string_option_read_unlock() +string_option_read_unlock(void) { d_r_read_unlock(&options_lock); } diff --git a/core/optionsx.h b/core/optionsx.h index 89bbeda03..b1ecdd8d9 100644 --- a/core/optionsx.h +++ b/core/optionsx.h @@ -3470,6 +3470,11 @@ OPTION_COMMAND( }, "run dr in a light weight mode with nothing but a few hooks", STATIC, OP_PCACHE_NOP) +#ifdef LINUX_KERNEL +OPTION_DEFAULT(bool, optimize_sys_call_ret, true, + "optimize syscall and sysret to avoid dispatch") +#endif + #undef OPTION #undef OPTION_NAME #undef OPTION_DEFAULT diff --git a/core/os_api.h b/core/os_api.h index ec349274c..c46f110fd 100644 --- a/core/os_api.h +++ b/core/os_api.h @@ -396,7 +396,7 @@ dr_convert_pid_to_handle(process_id_t pid); */ typedef void *dr_auxlib_handle_t; /** An exported routine in a loaded client auxiliary library. */ -typedef void (*dr_auxlib_routine_ptr_t)(); +typedef void (*dr_auxlib_routine_ptr_t)(void); #if defined(WINDOWS) && !defined(X64) /** * A handle to a loaded 64-bit client auxiliary library. This is a different diff --git a/core/os_shared.h b/core/os_shared.h index 381ce9dce..48ade2ade 100644 --- a/core/os_shared.h +++ b/core/os_shared.h @@ -113,6 +113,14 @@ os_heap_init(void); void os_heap_exit(void); +#ifdef LINUX_KERNEL +/* Called on each thread right before dispatching starts. This can be used to + * warm the fragment cache. + */ +void +os_warm_fcache(dcontext_t *dcontext); +#endif + /* os provided heap routines */ /* caller is required to handle thread synchronization and to update dynamo vm areas. * size must be PAGE_SIZE-aligned. @@ -249,9 +257,15 @@ get_thread_private_dcontext(void); void set_thread_private_dcontext(dcontext_t *dcontext); +#ifdef LINUX_KERNEL +typedef uint tls_offset_t; +#else +typedef ushort tls_offset_t; +#endif + /* converts a local_state_t offset to a segment offset */ -ushort -os_tls_offset(ushort tls_offs); +tls_offset_t +os_tls_offset(tls_offset_t tls_offs); ushort os_local_state_offset(ushort seg_offs); @@ -302,7 +316,7 @@ char * get_application_name(void); int -num_app_args(); +num_app_args(void); int get_app_args(DR_PARAM_OUT dr_app_arg_t *args_array, int args_count); const char * @@ -418,7 +432,7 @@ os_syslog(syslog_event_type_t priority, uint message_id, uint substitutions_num, * XXX: we're duplicating these types above as dr_auxlib* */ typedef void *shlib_handle_t; -typedef void (*shlib_routine_ptr_t)(); +typedef void (*shlib_routine_ptr_t)(void); shlib_handle_t load_shared_library(const char *name, bool reachable); @@ -1010,7 +1024,7 @@ query_time_millis(void); /* microseconds since 1601 */ uint64 -query_time_micros(); +query_time_micros(void); /* gives a good but not necessarily crypto-strength random seed */ uint diff --git a/core/perscache.c b/core/perscache.c index 7c0424a56..0aa9b3975 100644 --- a/core/perscache.c +++ b/core/perscache.c @@ -48,7 +48,7 @@ #include "hotpatch.h" #include "synch.h" #include "module_shared.h" -#include /* for offsetof */ +#include "stddef_wrapper.h" /* for offsetof */ #include "instrument.h" #ifdef DEBUG diff --git a/core/stats.c b/core/stats.c index 036cb9081..21dcc9b8b 100644 --- a/core/stats.c +++ b/core/stats.c @@ -225,7 +225,7 @@ kstat_exit() } static void -kstat_calibrate() +kstat_calibrate(void) { uint i; static bool kstats_calibrated = false; diff --git a/core/stddef_wrapper.h b/core/stddef_wrapper.h index bbe953c6e..3114dee99 100644 --- a/core/stddef_wrapper.h +++ b/core/stddef_wrapper.h @@ -5,6 +5,10 @@ #ifdef LINUX_KERNEL # include +# ifndef _WCHAR_T_DEFINED +# define _WCHAR_T_DEFINED +typedef __WCHAR_TYPE__ wchar_t; +# endif #else # include #endif diff --git a/core/synch.c b/core/synch.c index c4d020dc7..c55f4f2b0 100644 --- a/core/synch.c +++ b/core/synch.c @@ -2170,7 +2170,7 @@ detach_on_permanent_stack(bool internal, bool do_cleanup, dr_stats_t *drstats) wait_for_outstanding_nudges(); #endif -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) /* i#2270: we ignore alarm signals during detach to reduce races. */ signal_remove_alarm_handlers(my_dcontext); #endif @@ -2272,7 +2272,7 @@ detach_on_permanent_stack(bool internal, bool do_cleanup, dr_stats_t *drstats) * the thread_initexit_lock is held so that we can clean up thread * data later. */ -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) os_signal_thread_detach(threads[i]->dcontext); #endif LOG(GLOBAL, LOG_ALL, 1, "Detach: thread " TIDFMT " is being resumed as native\n", @@ -2281,7 +2281,7 @@ detach_on_permanent_stack(bool internal, bool do_cleanup, dr_stats_t *drstats) } ASSERT(my_idx != -1 || !internal); -#ifdef UNIX +#if defined(UNIX) && !defined(LINUX_KERNEL) LOG(GLOBAL, LOG_ALL, 1, "Detach: waiting for threads to fully detach\n"); for (i = 0; i < num_threads; i++) { if (i != my_idx && !IS_CLIENT_THREAD(threads[i]->dcontext)) @@ -2329,7 +2329,7 @@ detach_on_permanent_stack(bool internal, bool do_cleanup, dr_stats_t *drstats) #ifdef LINUX void -detach_externally_on_new_stack() +detach_externally_on_new_stack(void) { dcontext_t *my_dcontext; priv_mcontext_t my_mcontext; @@ -2371,8 +2371,10 @@ detach_externally_on_new_stack() /* synch with flush */ if (my_dcontext != NULL) enter_threadexit(my_dcontext); - /* i#2270: we ignore alarm signals during detach to reduce races. */ + /* i#2270: we ignore alarm signals during detach to reduce races. */ +# ifndef LINUX_KERNEL signal_remove_alarm_handlers(my_dcontext); +# endif /* suspend all DR-controlled threads at safe locations */ if (!synch_with_all_threads(THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT, &threads, &num_threads, @@ -2435,16 +2437,20 @@ detach_externally_on_new_stack() * the thread_initexit_lock is held so that we can clean up thread * data later. */ +# ifndef LINUX_KERNEL os_signal_thread_detach(threads[i]->dcontext); +# endif LOG(GLOBAL, LOG_ALL, 1, "Detach: thread " TIDFMT " is being resumed as native\n", threads[i]->id); os_thread_resume(threads[i]); } +# ifndef LINUX_KERNEL LOG(GLOBAL, LOG_ALL, 1, "Detach: waiting for threads to fully detach\n"); for (i = 0; i < num_threads; i++) { if (i != my_idx && !IS_CLIENT_THREAD(threads[i]->dcontext)) os_wait_thread_detached(threads[i]->dcontext); } +# endif /* Clean up each thread now that everyone has gone native. Needs to be * done with the thread_initexit_lock held, which is true within a synched * region. diff --git a/core/synch.h b/core/synch.h index 92e8e1817..9ca80cdb6 100644 --- a/core/synch.h +++ b/core/synch.h @@ -268,7 +268,7 @@ detach_on_permanent_stack(bool internal, bool do_cleanup, dr_stats_t *drstats); #ifdef LINUX void -detach_externally_on_new_stack(); +detach_externally_on_new_stack(void); #endif /*** exported for detach only ***/ diff --git a/core/translate.c b/core/translate.c index 29b0b3309..3b97638ca 100644 --- a/core/translate.c +++ b/core/translate.c @@ -167,7 +167,7 @@ instr_is_seg_ref_load(dcontext_t *dcontext, instr_t *inst) static inline bool instr_is_rseq_mangling(dcontext_t *dcontext, instr_t *inst) { -# ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) /* This won't fault but we don't want it marked as unsupported. */ if (!instr_is_our_mangling(inst)) return false; @@ -214,7 +214,7 @@ instr_is_rseq_mangling(dcontext_t *dcontext, instr_t *inst) opnd_get_disp(instr_get_dst(inst, 0)) == rseq_get_tls_ptr_offset()) return true; # endif -# endif +# endif /* LINUX && !LINUX_KERNEL */ return false; } #endif /* UNIX */ @@ -768,7 +768,7 @@ translate_restore_clean_call(dcontext_t *tdcontext, translate_walk_t *walk) app_pc translate_restore_special_cases(dcontext_t *dcontext, app_pc pc) { -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) app_pc handler; if (rseq_get_region_info(pc, NULL, NULL, &handler, NULL, NULL)) { LOG(THREAD_GET, LOG_INTERP, 2, @@ -786,7 +786,7 @@ translate_restore_special_cases(dcontext_t *dcontext, app_pc pc) app_pc translate_last_direct_translation(dcontext_t *dcontext, app_pc pc) { -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) app_pc handler; if (dcontext->client_data->last_special_xl8 != NULL && rseq_get_region_info(dcontext->client_data->last_special_xl8, NULL, NULL, @@ -800,7 +800,7 @@ translate_last_direct_translation(dcontext_t *dcontext, app_pc pc) void translate_clear_last_direct_translation(dcontext_t *dcontext) { -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) dcontext->client_data->last_special_xl8 = NULL; #endif } diff --git a/core/translate.h b/core/translate.h index 7c88623f8..a62ec8d46 100644 --- a/core/translate.h +++ b/core/translate.h @@ -93,6 +93,8 @@ typedef enum { RECREATE_FAILURE, RECREATE_SUCCESS_PC, RECREATE_SUCCESS_STATE, + RECREATE_DELAY_UNTIL_DISPATCH, + RECREATE_DELAY_UNTIL_PC, } recreate_success_t; recreate_success_t diff --git a/core/unix/os.c b/core/unix/os.c index dcd1c39d8..162a51086 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -1275,7 +1275,7 @@ set_app_args(DR_PARAM_IN int *app_argc_in, DR_PARAM_IN char **app_argv_in) /* Returns the number of application's command-line arguments. */ int -num_app_args() +num_app_args(void) { if (!DYNAMO_OPTION(early_inject)) { set_client_error_code(NULL, DR_ERROR_NOT_IMPLEMENTED); @@ -1442,7 +1442,7 @@ query_time_millis() /* microseconds since 1601 */ uint64 -query_time_micros() +query_time_micros(void) { struct timeval current_time; uint64 val = dynamorio_syscall(SYS_gettimeofday, 2, ¤t_time, NULL); @@ -10684,7 +10684,7 @@ mutex_wait_contended_lock(mutex_t *lock, priv_mcontext_t *mc) } } else { /* we now have to undo our earlier request */ - atomic_dec_and_test(&lock->lock_requests); + d_r_atomic_dec_and_test(&lock->lock_requests); while (!d_r_mutex_trylock(lock)) { if (set_client_safe_for_synch) diff --git a/core/utils.c b/core/utils.c index 370bbb06d..cd217b1a3 100644 --- a/core/utils.c +++ b/core/utils.c @@ -45,20 +45,26 @@ #include "dr_tools.h" #include "utils.h" #include "module_shared.h" -#include + +#ifndef LINUX_KERNEL +# include +#endif #ifdef PROCESS_CONTROL # include "moduledb.h" /* for process control macros */ #endif #ifdef UNIX -# include -# include -# include -# include -# include -# include -# include +# include "types_wrapper.h" +# ifndef LINUX_KERNEL +# include +# include +# include +# include +# include +# include +# include +# endif #else # include /* XXX : remove when syslog macros fixed */ @@ -73,8 +79,8 @@ # include "synch.h" /* all_threads_synch_lock */ #endif -#include /* for varargs */ -#include /* for offsetof */ +#include "stdarg_wrapper.h" /* for varargs */ +#include "stddef_wrapper.h" /* for offsetof */ try_except_t global_try_except; #ifdef UNIX @@ -92,7 +98,7 @@ sideline_exit(void); * performs some cleanup and then calls os_terminate */ static void -soft_terminate() +soft_terminate(void) { #ifdef SIDELINE /* kill child threads */ @@ -749,7 +755,8 @@ utils_init() ASSERT(sizeof(uint) == 4); ASSERT(sizeof(reg_t) == sizeof(void *)); -#ifdef UNIX /* after options_init(), before we open logfile or call instrument_init() */ +#if defined(UNIX) && !defined(LINUX_KERNEL) + /* after options_init(), before we open logfile or call instrument_init() */ os_file_init(); #endif @@ -878,7 +885,7 @@ d_r_mutex_lock_app(mutex_t *lock, priv_mcontext_t *mc) } /* we have strong intentions to grab this lock, increment requests */ - acquired = atomic_inc_and_test(&lock->lock_requests); + acquired = d_r_atomic_inc_and_test(&lock->lock_requests); DEADLOCK_AVOIDANCE_LOCK(lock, acquired, ownable); if (!acquired) { @@ -929,7 +936,7 @@ d_r_mutex_unlock(mutex_t *lock) ASSERT(lock->lock_requests > LOCK_FREE_STATE && "lock not owned"); DEADLOCK_AVOIDANCE_UNLOCK(lock, ownable); - if (atomic_dec_and_test(&lock->lock_requests)) + if (d_r_atomic_dec_and_test(&lock->lock_requests)) return; /* if we were not the last one to hold the lock, (i.e. final value is not LOCK_FREE_STATE) @@ -1185,7 +1192,7 @@ d_r_read_lock(read_write_lock_t *rw) /* no need to pause */ } /* Even if we didn't wait another reader may be waiting for notification */ - if (!atomic_dec_becomes_zero(&rw->num_pending_readers)) { + if (!d_r_atomic_dec_becomes_zero(&rw->num_pending_readers)) { /* If we were not the last pending reader, we need to notify another waiting one so that it can get out of the contention path. @@ -1295,7 +1302,7 @@ d_r_read_unlock(read_write_lock_t *rw) to check if the writer is in fact waiting. Even though this is not atomic we don't need to loop here - d_r_write_lock() will loop. */ - if (atomic_dec_becomes_zero(&rw->num_readers)) { + if (d_r_atomic_dec_becomes_zero(&rw->num_readers)) { /* if the writer is waiting it definitely needs to hold the mutex */ if (mutex_testlock(&rw->lock)) { /* test that it was not this thread owning both write and read lock */ @@ -1714,6 +1721,7 @@ divide_uint64_print(uint64 numerator, uint64 denominator, bool percentage, uint extern long double2int_trunc(double d); +#ifndef LINUX_KERNEL /* For printing a float. * NOTE: You must preserve x87 floating point state to call this function, unless * you can prove the compiler will never use x87 state for float operations. @@ -1742,6 +1750,7 @@ double_print(double val, uint precision, uint *top, uint *bottom, const char **s *top = double2int_trunc(val); *bottom = double2int_trunc((val - *top) * precision_multiple); } +#endif #ifdef WINDOWS /* for pre_inject, injector, and core shared files, is just wrapper for syslog @@ -2646,7 +2655,9 @@ static bool basedir_initialized = false; /* below used in the create_log_dir function to avoid having it on the stack * on what is a critical path for stack depth (diagnostics->create_log_dir-> * get_parameter */ +#ifndef LINUX_KERNEL static char old_basedir[MAXIMUM_PATH]; +#endif /* this lock is recursive because current implementation recurses to create the * basedir when called to create the logdir before the basedir is created, is * also useful in case we receive an exception in the create_log_dir function @@ -2665,14 +2676,17 @@ enable_new_log_dir() void create_log_dir(int dir_type) { -#ifdef UNIX +#ifdef LINUX_KERNEL + ASSERT_NOT_IMPLEMENTED(false); +#else +# ifdef UNIX char *pre_execve = getenv(DYNAMORIO_VAR_EXECVE_LOGDIR); DEBUG_DECLARE(bool sharing_logdir = false;) -#endif +# endif /* synchronize */ acquire_recursive_lock(&logdir_mutex); SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); -#ifdef UNIX +# ifdef UNIX if (dir_type == PROCESS_DIR && pre_execve != NULL) { /* if this app has a logdir option or config, that should trump sharing * the pre-execve logdir. a logdir env var should not. @@ -2692,7 +2706,7 @@ create_log_dir(int dir_type) /* check that it's gone: we've had problems with unsetenv */ ASSERT(getenv(DYNAMORIO_VAR_EXECVE_LOGDIR) == NULL); } -#endif +# endif /* used to be an else: leaving indentation though */ if (dir_type == BASE_DIR) { int retval; @@ -2715,21 +2729,21 @@ create_log_dir(int dir_type) basedir_initialized = true; /* skip creating dir basedir if is empty */ if (basedir[0] == '\0') { -#ifndef STATIC_LIBRARY +# ifndef STATIC_LIBRARY SYSLOG(SYSLOG_WARNING, WARNING_EMPTY_OR_NONEXISTENT_LOGDIR_KEY, 2, get_application_name(), get_application_pid()); -#endif +# endif } else { if (!os_create_dir(basedir, CREATE_DIR_ALLOW_EXISTING)) { /* try to create full path */ char swap; char *end = double_strchr(basedir, DIRSEP, ALT_DIRSEP); bool res; -#ifdef WINDOWS +# ifdef WINDOWS /* skip the drive */ if (end != NULL && end > basedir && *(end - 1) == ':') end = double_strchr(++end, DIRSEP, ALT_DIRSEP); -#endif +# endif while (end) { swap = *end; *end = '\0'; @@ -2771,7 +2785,7 @@ create_log_dir(int dir_type) SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); release_recursive_lock(&logdir_mutex); -#ifdef DEBUG +# ifdef DEBUG if (d_r_stats != NULL) { /* if null, we're trying to report an error (probably via a core dump), * so who cares if we lose logdir name */ @@ -2779,12 +2793,13 @@ create_log_dir(int dir_type) d_r_stats->logdir[sizeof(d_r_stats->logdir) - 1] = '\0'; /* if max no null */ } if (dir_type == PROCESS_DIR -# ifdef UNIX +# ifdef UNIX && !sharing_logdir -# endif +# endif ) SYSLOG_INTERNAL_INFO("log dir=%s", logdir); -#endif /* DEBUG */ +# endif /* DEBUG */ +#endif /* LINUX_KERNEL */ } /* Copies the name of the specified directory into buffer, returns true if @@ -3836,7 +3851,7 @@ MD5Transform(uint32 state[4], const unsigned char block[MD5_BLOCK_LENGTH]) { uint32 a, b, c, d, in[MD5_BLOCK_LENGTH / 4]; -#if BYTE_ORDER == LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ memcpy(in, block, sizeof(in)); #else for (a = 0; a < MD5_BLOCK_LENGTH / 4; a++) { diff --git a/core/vmareas.c b/core/vmareas.c index 3b082932d..81d586876 100644 --- a/core/vmareas.c +++ b/core/vmareas.c @@ -1687,7 +1687,7 @@ vm_areas_init() } static void -vm_areas_statistics() +vm_areas_statistics(void) { #ifdef PROGRAM_SHEPHERDING DOLOG(1, LOG_VMAREAS | LOG_STATS, { @@ -8266,7 +8266,7 @@ check_thread_vm_area(dcontext_t *dcontext, app_pc pc, app_pc tag, void **vmlist, if (stop != NULL) { *stop = area->end; ASSERT(*stop != NULL); -#ifdef LINUX +#if defined(LINUX) && !defined(LINUX_KERNEL) if (!vmvector_empty(d_r_rseq_areas)) { /* XXX i#3798: While for core operation we do not need to end a block at * an rseq endpoint, we need clients to treat the endpoint as a barrier and diff --git a/core/win32/ntdll.c b/core/win32/ntdll.c index 906c5bfa3..b5822615d 100644 --- a/core/win32/ntdll.c +++ b/core/win32/ntdll.c @@ -3369,7 +3369,7 @@ query_time_100ns() } uint64 -query_time_micros() +query_time_micros(void) { LONGLONG time100ns = query_time_100ns(); return ((uint64)time100ns / TIMER_UNITS_PER_MICROSECOND); diff --git a/core/win32/os.c b/core/win32/os.c index fc95edb36..55f0028e4 100644 --- a/core/win32/os.c +++ b/core/win32/os.c @@ -3225,7 +3225,7 @@ is_phandle_me(HANDLE phandle) /* Returns the number of application's command-line arguments. */ int -num_app_args() +num_app_args(void) { /* XXX i#2662: Add support for Windows. */ ASSERT_NOT_IMPLEMENTED(false); diff --git a/ext/drmgr/drmgr.c b/ext/drmgr/drmgr.c index 58b98bd3f..0a0a68316 100644 --- a/ext/drmgr/drmgr.c +++ b/ext/drmgr/drmgr.c @@ -459,7 +459,7 @@ static void drmgr_modunload_event(void *drcontext, const module_data_t *info); static void -drmgr_low_on_memory_event(); +drmgr_low_on_memory_event(void); static void drmgr_kernel_xfer_event(void *drcontext, const dr_kernel_xfer_info_t *info); @@ -3500,7 +3500,7 @@ drmgr_unregister_low_on_memory_event_user_data(void (*func)(void *user_data)) } static void -drmgr_low_on_memory_event() +drmgr_low_on_memory_event(void) { void *drcontext = dr_get_current_drcontext(); generic_event_entry_t local[EVENTS_STACK_SZ]; diff --git a/ext/drsyms/drsyms_unix_frontend.c b/ext/drsyms/drsyms_unix_frontend.c index f67cd0728..237b91dad 100644 --- a/ext/drsyms/drsyms_unix_frontend.c +++ b/ext/drsyms/drsyms_unix_frontend.c @@ -206,8 +206,8 @@ drsym_init(int shmid_in) */ } else { hashtable_init_ex(&modtable, MODTABLE_HASH_BITS, HASH_STRING, true /*strdup*/, - false /*!synch: using symbol_lock*/, - (generic_func_t)drsym_unix_unload, NULL, NULL); + false /*!synch: using symbol_lock*/, drsym_unix_unload, NULL, + NULL); } return DRSYM_SUCCESS; } diff --git a/suite/tests/client-interface/drcallstack-test.c b/suite/tests/client-interface/drcallstack-test.c index 13b54d8fe..328458b0b 100644 --- a/suite/tests/client-interface/drcallstack-test.c +++ b/suite/tests/client-interface/drcallstack-test.c @@ -33,7 +33,7 @@ #include "tools.h" #ifdef LINUX -# define NOINLINE __attribute__((noinline)) +# define NOINLINE __attribute__((__noinline__)) #else # error NYI #endif diff --git a/suite/tests/client-interface/events.dll.c b/suite/tests/client-interface/events.dll.c index c06a9a270..5bbcd2214 100644 --- a/suite/tests/client-interface/events.dll.c +++ b/suite/tests/client-interface/events.dll.c @@ -169,7 +169,7 @@ check_result(void) } static void -low_on_memory_event() +low_on_memory_event(void) { /* Do nothing. Testing only register and unregister functions. */ } diff --git a/suite/tests/tools.h b/suite/tests/tools.h index 5526a5a92..edc7cc573 100644 --- a/suite/tests/tools.h +++ b/suite/tests/tools.h @@ -291,7 +291,7 @@ page_size(void) #else /* UNIX */ # define EXPORT __attribute__((visibility("default"))) # define IMPORT extern -# define NOINLINE __attribute__((noinline)) +# define NOINLINE __attribute__((__noinline__)) #endif /* convenience macros for secure string buffer operations */