diff --git a/arch/aarch64/mcount-insn.c b/arch/aarch64/mcount-insn.c index 13488487b..7e0a2dc8a 100644 --- a/arch/aarch64/mcount-insn.c +++ b/arch/aarch64/mcount-insn.c @@ -8,10 +8,19 @@ #include #include +/** + * mcount_disasm_init - initialize the capstone engine once + * @disasm - mcount disassembly engine + */ void mcount_disasm_init(struct mcount_disasm_engine *disasm) { + if (disasm->engine) + return; + + pr_dbg2("initialize disassembly engine\n"); + if (cs_open(CS_ARCH_ARM64, CS_MODE_ARM, &disasm->engine) != CS_ERR_OK) { - pr_dbg("failed to init Capstone disasm engine\n"); + pr_dbg("failed to init capstone disasm engine\n"); return; } diff --git a/arch/x86_64/mcount-insn.c b/arch/x86_64/mcount-insn.c index c8ab6105a..d438034d8 100644 --- a/arch/x86_64/mcount-insn.c +++ b/arch/x86_64/mcount-insn.c @@ -19,8 +19,17 @@ struct disasm_check_data { uint32_t size; }; +/** + * mcount_disasm_init - initialize the capstone engine once + * @disasm - mcount disassembly engine + */ void mcount_disasm_init(struct mcount_disasm_engine *disasm) { + if (disasm->engine) + return; + + pr_dbg2("initialize disassembly engine\n"); + if (cs_open(CS_ARCH_X86, CS_MODE_64, &disasm->engine) != CS_ERR_OK) { pr_dbg("failed to init capstone disasm engine\n"); return; @@ -674,7 +683,7 @@ TEST_CASE(dynamic_x86_handle_lea) .addr = 0x3000, .size = 32, }; - struct mcount_disasm_engine disasm; + struct mcount_disasm_engine disasm = { 0 }; struct mcount_disasm_info info = { .sym = &sym, .addr = ORIGINAL_BASE + sym.addr, @@ -745,7 +754,7 @@ TEST_CASE(dynamic_x86_handle_call) .addr = 0x4000, .size = 32, }; - struct mcount_disasm_engine disasm; + struct mcount_disasm_engine disasm = { 0 }; struct mcount_disasm_info info = { .sym = &sym1, .addr = ORIGINAL_BASE + sym1.addr, @@ -823,7 +832,7 @@ TEST_CASE(dynamic_x86_handle_jmp) .addr = 0x3000, .size = 32, }; - struct mcount_disasm_engine disasm; + struct mcount_disasm_engine disasm = { 0 }; struct mcount_disasm_info info = { .sym = &sym, .addr = ORIGINAL_BASE + sym.addr, @@ -927,7 +936,7 @@ TEST_CASE(dynamic_x86_handle_jcc) .addr = 0x3000, .size = 32, }; - struct mcount_disasm_engine disasm; + struct mcount_disasm_engine disasm = { 0 }; struct mcount_disasm_info info = { .sym = &sym, .addr = ORIGINAL_BASE + sym.addr, @@ -1036,7 +1045,7 @@ TEST_CASE(dynamic_x86_handle_mov_load) .addr = 0x3000, .size = 32, }; - struct mcount_disasm_engine disasm; + struct mcount_disasm_engine disasm = { 0 }; struct mcount_disasm_info info = { .sym = &sym, .addr = ORIGINAL_BASE + sym.addr, diff --git a/libmcount/dynamic.c b/libmcount/dynamic.c index 507162060..462f1b3c9 100644 --- a/libmcount/dynamic.c +++ b/libmcount/dynamic.c @@ -31,6 +31,8 @@ #include "utils/symbol.h" #include "utils/utils.h" +static bool main_loaded; +static bool modules_loaded; static struct mcount_dynamic_info *mdinfo; static struct mcount_dynamic_stats { int total; @@ -60,6 +62,8 @@ static unsigned min_size; /* disassembly engine for dynamic code patch (for capstone) */ static struct mcount_disasm_engine disasm; +static bool mcount_dynamic_is_initialized; + static struct mcount_orig_insn *create_code(struct Hashmap *map, unsigned long addr) { struct mcount_orig_insn *entry; @@ -101,6 +105,9 @@ void mcount_save_code(struct mcount_disasm_info *info, unsigned call_size, void struct mcount_orig_insn *orig; int patch_size; + if (hashmap_get(code_hmap, (void *)info->addr + call_size)) + return; + if (unlikely(info->modified)) { /* it needs to save original instructions as well */ int orig_size = ALIGN(info->orig_size, 16); @@ -247,6 +254,11 @@ __weak void mcount_disasm_finish(struct mcount_disasm_engine *disasm) { } +__weak int mcount_arch_dynamic_init(void) +{ + return -1; +} + __weak int mcount_arch_branch_table_size(struct mcount_disasm_info *info) { return 0; @@ -293,8 +305,15 @@ static struct mcount_dynamic_info *create_mdi(struct dl_phdr_info *info) return mdi; } -/* callback for dl_iterate_phdr() */ -static int find_dynamic_module(struct dl_phdr_info *info, size_t sz, void *data) +/** + * prepare_dynamic_module - store dynamic module info and install trampoline; + * callback for dl_iterate_phdr() + * @info - module info + * @sz - data size (unused) + * @data - callback data: symbol info and module parsing flag + * @return - stop iteration on non-zero value + */ +static int prepare_dynamic_module(struct dl_phdr_info *info, size_t sz, void *data) { struct mcount_dynamic_info *mdi; struct find_module_data *fmd = data; @@ -302,6 +321,15 @@ static int find_dynamic_module(struct dl_phdr_info *info, size_t sz, void *data) struct uftrace_mmap *map; bool is_executable = mcount_is_main_executable(info->dlpi_name, sym_info->filename); + if (is_executable) { + if (main_loaded) + return !fmd->needs_modules; + } + else if (!fmd->needs_modules) + return main_loaded; + else if (modules_loaded) + return 0; + mdi = create_mdi(info); map = find_map(sym_info, mdi->base_addr); @@ -309,8 +337,14 @@ static int find_dynamic_module(struct dl_phdr_info *info, size_t sz, void *data) mdi->map = map; mcount_arch_find_module(mdi, &map->mod->symtab); + if (mcount_setup_trampoline(mdi) < 0) + mdi->trampoline = 0; + mdi->next = mdinfo; mdinfo = mdi; + if (is_executable) + main_loaded = true; + pr_dbg3("load dynamic info for '%s'\n", mdi->map->libname); } else { free(mdi); @@ -319,20 +353,51 @@ static int find_dynamic_module(struct dl_phdr_info *info, size_t sz, void *data) return !fmd->needs_modules && is_executable; } +/** + * prepare_dynamic_update - create dynamic data structures and load dynamic + * modules info + * @sinfo - dynamic symbol info + * @needs_modules - whether to prepare dynamic modules or only the main binary + */ static void prepare_dynamic_update(struct uftrace_sym_info *sinfo, bool needs_modules) { + int rc = 0; struct find_module_data fmd = { .sinfo = sinfo, .needs_modules = needs_modules, }; int hash_size = sinfo->exec_map->mod->symtab.nr_sym * 3 / 4; - if (needs_modules) - hash_size *= 2; + if (code_hmap) { + /* main executable already loaded */ + if (!needs_modules || modules_loaded) + return; + } + else { /* nothing loaded */ + if (needs_modules) + hash_size *= 2; + code_hmap = hashmap_create(hash_size, hashmap_ptr_hash, hashmap_ptr_equals); + } + + rc = dl_iterate_phdr(prepare_dynamic_module, &fmd); + + if (needs_modules && rc == 0) + modules_loaded = true; +} + +/** + * setup_size_filter - initialize size filter if not set + */ +void setup_size_filter(void) +{ + char *size_filter; - code_hmap = hashmap_create(hash_size, hashmap_ptr_hash, hashmap_ptr_equals); + if (min_size) + return; - dl_iterate_phdr(find_dynamic_module, &fmd); + size_filter = getenv("UFTRACE_MIN_SIZE"); + if (size_filter) + min_size = strtoul(size_filter, NULL, 0); } struct mcount_dynamic_info *setup_trampoline(struct uftrace_mmap *map) @@ -384,17 +449,10 @@ static bool match_pattern_module(char *pathname) return ret; } -/** - * match_pattern_list - match a symbol name against a pattern list - * @map - memory map of the symbol - * @soname - name of the module - * @sym_name - name of the symbol - * @return - -1 if match negative, 1 if match positive, 0 if no match - */ -static int match_pattern_list(struct uftrace_mmap *map, char *soname, char *sym_name) +static bool match_pattern_list(struct uftrace_mmap *map, char *soname, char *sym_name) { struct patt_list *pl; - int ret = 0; + bool ret = false; char *libname = basename(map->libname); list_for_each_entry(pl, &patterns, list) { @@ -405,7 +463,7 @@ static int match_pattern_list(struct uftrace_mmap *map, char *soname, char *sym_ continue; if (match_filter_pattern(&pl->patt, sym_name)) - ret = pl->positive ? 1 : -1; + ret = pl->positive; } return ret; @@ -417,6 +475,7 @@ static void parse_pattern_list(char *patch_funcs, char *def_mod, enum uftrace_pa char *name; int j; struct patt_list *pl; + bool all_negative = true; strv_split(&funcs, patch_funcs, ";"); @@ -427,8 +486,10 @@ static void parse_pattern_list(char *patch_funcs, char *def_mod, enum uftrace_pa if (name[0] == '!') name++; - else + else { pl->positive = true; + all_negative = false; + } delim = strchr(name, '@'); if (delim == NULL) { @@ -443,6 +504,20 @@ static void parse_pattern_list(char *patch_funcs, char *def_mod, enum uftrace_pa list_add_tail(&pl->list, &patterns); } + /* prepend match-all pattern, if all patterns are negative */ + if (all_negative) { + pl = xzalloc(sizeof(*pl)); + pl->positive = true; + pl->module = xstrdup(def_mod); + + if (ptype == PATT_REGEX) + init_filter_pattern(ptype, &pl->patt, "."); + else + init_filter_pattern(PATT_GLOB, &pl->patt, "*"); + + list_add(&pl->list, &patterns); + } + strv_free(&funcs); } @@ -477,6 +552,12 @@ static bool skip_sym(struct uftrace_symbol *sym, struct mcount_dynamic_info *mdi if (sym->type != ST_LOCAL_FUNC && sym->type != ST_GLOBAL_FUNC && sym->type != ST_WEAK_FUNC) return true; + if (!match_pattern_list(map, soname, sym->name)) { + if (mcount_unpatch_func(mdi, sym, &disasm) == 0) + stats.unpatch++; + return true; + } + return false; } @@ -555,7 +636,6 @@ static void patch_normal_func_matched(struct mcount_dynamic_info *mdi, struct uf unsigned i; struct uftrace_symbol *sym; bool found = false; - int match; char *soname = get_soname(map->libname); symtab = &map->mod->symtab; @@ -565,15 +645,9 @@ static void patch_normal_func_matched(struct mcount_dynamic_info *mdi, struct uf if (skip_sym(sym, mdi, map, soname)) continue; - found = true; - match = match_pattern_list(map, soname, sym->name); - if (!match) - continue; - else if (match == 1) - mcount_patch_func_with_stats(mdi, sym); - else - mcount_unpatch_func(mdi, sym, NULL); + found = true; + mcount_patch_func_with_stats(mdi, sym); } if (!found) @@ -597,10 +671,18 @@ static void patch_func_matched(struct mcount_dynamic_info *mdi, struct uftrace_m patch_normal_func_matched(mdi, map); } +/** + * do_dynamic_update - apply (un)patching across loaded modules' maps + * @sinfo - dynamic symbol info + * @patch_funcs - spec of symbols to (un)patch + * @ptype - matching pattern type + * @return - 0 (unused) + */ static int do_dynamic_update(struct uftrace_sym_info *sinfo, char *patch_funcs, enum uftrace_pattern_type ptype) { struct uftrace_mmap *map; + struct mcount_dynamic_info *mdi; char *def_mod; if (patch_funcs == NULL) @@ -609,17 +691,22 @@ static int do_dynamic_update(struct uftrace_sym_info *sinfo, char *patch_funcs, def_mod = basename(sinfo->exec_map->libname); parse_pattern_list(patch_funcs, def_mod, ptype); - for_each_map(sinfo, map) { - struct mcount_dynamic_info *mdi; - - /* TODO: filter out unsuppported libs */ - mdi = setup_trampoline(map); - if (mdi == NULL) - continue; + /* TODO: filter out unsupported libs */ + for (mdi = mdinfo; mdi != NULL; mdi = mdi->next) { + if (mdi->type == DYNAMIC_NONE && !mcount_dynamic_is_initialized) { + if (mcount_arch_dynamic_init() >= 0) + mcount_dynamic_is_initialized = true; + else + continue; + } - patch_func_matched(mdi, map); + map = mdi->map; + if (mdi->trampoline) + patch_func_matched(mdi, map); } + release_pattern_list(); + if (stats.failed + stats.skipped + stats.nomatch == 0) { pr_dbg("patched all (%d) functions in '%s'\n", stats.total, basename(sinfo->filename)); @@ -628,24 +715,6 @@ static int do_dynamic_update(struct uftrace_sym_info *sinfo, char *patch_funcs, return 0; } -static void freeze_dynamic_update(void) -{ - struct mcount_dynamic_info *mdi, *tmp; - - mdi = mdinfo; - while (mdi) { - tmp = mdi->next; - - mcount_arch_dynamic_recover(mdi, &disasm); - mcount_cleanup_trampoline(mdi); - free(mdi); - - mdi = tmp; - } - - mcount_freeze_code(); -} - /* do not use floating-point in libmcount */ static int calc_percent(int n, int total, int *rem) { @@ -655,20 +724,23 @@ static int calc_percent(int n, int total, int *rem) return quot; } +/** + * mcount_dynamic_update - prepare and perform dynamic patching or unpatching, + * then display statistics + * @sinfo - dynamic symbol info + * @patch_funcs - spec of symbols to patch or unpatch + * @ptype - matching pattern type + * @return - 0 (unused) + */ int mcount_dynamic_update(struct uftrace_sym_info *sinfo, char *patch_funcs, enum uftrace_pattern_type ptype) { int ret = 0; - char *size_filter; bool needs_modules = !!strchr(patch_funcs, '@'); mcount_disasm_init(&disasm); - prepare_dynamic_update(sinfo, needs_modules); - - size_filter = getenv("UFTRACE_MIN_SIZE"); - if (size_filter != NULL) - min_size = strtoul(size_filter, NULL, 0); + setup_size_filter(); ret = do_dynamic_update(sinfo, patch_funcs, ptype); @@ -687,7 +759,7 @@ int mcount_dynamic_update(struct uftrace_sym_info *sinfo, char *patch_funcs, pr_dbg("no match: %8d\n", stats.nomatch); } - freeze_dynamic_update(); + mcount_freeze_code(); return ret; } @@ -733,9 +805,29 @@ void mcount_dynamic_dlopen(struct uftrace_sym_info *sinfo, struct dl_phdr_info * mcount_freeze_code(); } +/** + * mcount_free_mdinfo - free all dynamic info structures + */ +static void mcount_free_mdinfo(void) +{ + struct mcount_dynamic_info *mdi, *tmp; + + mdi = mdinfo; + while (mdi) { + tmp = mdi->next; + + mcount_arch_dynamic_recover(mdi, &disasm); + mcount_cleanup_trampoline(mdi); + free(mdi); + + mdi = tmp; + } +} + void mcount_dynamic_finish(void) { release_pattern_list(); + mcount_free_mdinfo(); mcount_disasm_finish(&disasm); } @@ -849,27 +941,27 @@ TEST_CASE(dynamic_pattern_list) pr_dbg("check simple match with default module\n"); parse_pattern_list("abc;!def", "main", PATT_SIMPLE); - TEST_EQ(match_pattern_list(main_map, NULL, "abc"), 1); - TEST_EQ(match_pattern_list(main_map, NULL, "def"), -1); - TEST_EQ(match_pattern_list(other_map, NULL, "xyz"), 0); + TEST_EQ(match_pattern_list(main_map, NULL, "abc"), true); + TEST_EQ(match_pattern_list(main_map, NULL, "def"), false); + TEST_EQ(match_pattern_list(other_map, NULL, "xyz"), false); release_pattern_list(); pr_dbg("check negative regex match with default module\n"); parse_pattern_list("!^a", "main", PATT_REGEX); - TEST_EQ(match_pattern_list(main_map, NULL, "abc"), -1); - TEST_EQ(match_pattern_list(main_map, NULL, "def"), 0); - TEST_EQ(match_pattern_list(other_map, NULL, "xyz"), 0); + TEST_EQ(match_pattern_list(main_map, NULL, "abc"), false); + TEST_EQ(match_pattern_list(main_map, NULL, "def"), true); + TEST_EQ(match_pattern_list(other_map, NULL, "xyz"), false); release_pattern_list(); pr_dbg("check wildcard match with other module\n"); parse_pattern_list("*@other", "main", PATT_GLOB); - TEST_EQ(match_pattern_list(main_map, NULL, "abc"), 0); - TEST_EQ(match_pattern_list(main_map, NULL, "def"), 0); - TEST_EQ(match_pattern_list(other_map, NULL, "xyz"), 1); + TEST_EQ(match_pattern_list(main_map, NULL, "abc"), false); + TEST_EQ(match_pattern_list(main_map, NULL, "def"), false); + TEST_EQ(match_pattern_list(other_map, NULL, "xyz"), true); release_pattern_list();