From eabd382a7963030f99630efe0c587c440141d876 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 15 Feb 2024 16:33:51 +0100 Subject: [PATCH 1/2] Add umf_ba_linear_pool_contains_pointer Add umf_ba_linear_pool_contains_pointer(). It returns: - 0 if ptr does not belong to the pool or - size (> 0) of the memory region from ptr to the end of the pool if ptr belongs to the pool. It will be useful to implement realloc() in the proxy library. Added also the baseAllocLinearPoolContainsPointer test. Signed-off-by: Lukasz Dorau --- src/base_alloc/base_alloc_linear.c | 30 ++++++++++++++++++++++++++++++ src/base_alloc/base_alloc_linear.h | 2 ++ test/test_base_alloc_linear.cpp | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/src/base_alloc/base_alloc_linear.c b/src/base_alloc/base_alloc_linear.c index 4948640b1..d3059d664 100644 --- a/src/base_alloc/base_alloc_linear.c +++ b/src/base_alloc/base_alloc_linear.c @@ -175,3 +175,33 @@ void umf_ba_linear_destroy(umf_ba_linear_pool_t *pool) { util_mutex_destroy_not_free(&pool->metadata.lock); ba_os_free(pool, pool->metadata.pool_size); } + +// umf_ba_linear_pool_contains_pointer() returns: +// - 0 if ptr does not belong to the pool or +// - size (> 0) of the memory region from ptr +// to the end of the pool if ptr belongs to the pool +size_t umf_ba_linear_pool_contains_pointer(umf_ba_linear_pool_t *pool, + void *ptr) { + util_mutex_lock(&pool->metadata.lock); + char *cptr = (char *)ptr; + if (cptr >= pool->data && + cptr < ((char *)(pool)) + pool->metadata.pool_size) { + size_t size = ((char *)(pool)) + pool->metadata.pool_size - cptr; + util_mutex_unlock(&pool->metadata.lock); + return size; + } + + umf_ba_next_linear_pool_t *next_pool = pool->next_pool; + while (next_pool) { + if (cptr >= next_pool->data && + cptr < ((char *)(next_pool)) + next_pool->pool_size) { + size_t size = ((char *)(next_pool)) + next_pool->pool_size - cptr; + util_mutex_unlock(&pool->metadata.lock); + return size; + } + next_pool = next_pool->next_pool; + } + + util_mutex_unlock(&pool->metadata.lock); + return 0; +} diff --git a/src/base_alloc/base_alloc_linear.h b/src/base_alloc/base_alloc_linear.h index 710bc560d..ccb408b53 100644 --- a/src/base_alloc/base_alloc_linear.h +++ b/src/base_alloc/base_alloc_linear.h @@ -26,6 +26,8 @@ typedef struct umf_ba_linear_pool umf_ba_linear_pool_t; umf_ba_linear_pool_t *umf_ba_linear_create(size_t pool_size); void *umf_ba_linear_alloc(umf_ba_linear_pool_t *pool, size_t size); void umf_ba_linear_destroy(umf_ba_linear_pool_t *pool); +size_t umf_ba_linear_pool_contains_pointer(umf_ba_linear_pool_t *pool, + void *ptr); #ifdef __cplusplus } diff --git a/test/test_base_alloc_linear.cpp b/test/test_base_alloc_linear.cpp index e43e03286..9d7d656dc 100644 --- a/test/test_base_alloc_linear.cpp +++ b/test/test_base_alloc_linear.cpp @@ -27,6 +27,24 @@ TEST_F(test, baseAllocLinearAllocMoreThanPoolSize) { memset(ptr, 0, new_size); } +TEST_F(test, baseAllocLinearPoolContainsPointer) { + auto pool = std::shared_ptr( + umf_ba_linear_create(0 /* minimal pool size (page size) */), + umf_ba_linear_destroy); + + size_t size = 16; + void *ptr = umf_ba_linear_alloc(pool.get(), size); + UT_ASSERTne(ptr, NULL); + memset(ptr, 0, size); + + // assert pool contains pointer ptr + UT_ASSERTne(umf_ba_linear_pool_contains_pointer(pool.get(), ptr), 0); + + // assert pool does NOT contain pointer 0x0123 + UT_ASSERTeq(umf_ba_linear_pool_contains_pointer(pool.get(), (void *)0x0123), + 0); +} + TEST_F(test, baseAllocLinearMultiThreadedAllocMemset) { static constexpr int NTHREADS = 10; static constexpr int ITERATIONS = 1000; From 47a98a4c6fc93979ad246d95cee01ccb978550f7 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 21 Feb 2024 12:40:08 +0100 Subject: [PATCH 2/2] Add proxy library for intercepting user allocation requests UMF proxy library is a library for intercepting user allocation requests. It intercepts following APIs: - aligned_alloc() - calloc() - free() - malloc() - malloc_usable_size() - realloc() Signed-off-by: Lukasz Dorau --- .github/workflows/pr_push.yml | 3 + .github/workflows/proxy_lib.yml | 62 +++++++ src/CMakeLists.txt | 28 ++++ src/base_alloc/base_alloc.c | 22 ++- src/base_alloc/base_alloc_linear.c | 8 + src/provider/provider_tracking.c | 40 +++-- src/proxy_lib/CMakeLists.txt | 54 ++++++ src/proxy_lib/proxy_lib.c | 261 +++++++++++++++++++++++++++++ src/proxy_lib/proxy_lib.def | 15 ++ src/proxy_lib/proxy_lib.h | 24 +++ src/proxy_lib/proxy_lib.map | 17 ++ src/proxy_lib/proxy_lib_linux.c | 29 ++++ src/proxy_lib/proxy_lib_windows.c | 27 +++ src/utils/utils_common.h | 40 ++++- test/test_make_install.txt | 1 + 15 files changed, 612 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/proxy_lib.yml create mode 100644 src/proxy_lib/CMakeLists.txt create mode 100644 src/proxy_lib/proxy_lib.c create mode 100644 src/proxy_lib/proxy_lib.def create mode 100644 src/proxy_lib/proxy_lib.h create mode 100644 src/proxy_lib/proxy_lib.map create mode 100644 src/proxy_lib/proxy_lib_linux.c create mode 100644 src/proxy_lib/proxy_lib_windows.c diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 505bf8b4b..f4516f784 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -91,3 +91,6 @@ jobs: Benchmark: needs: [Build] uses: ./.github/workflows/benchmarks.yml + Proxy_lib: + needs: [Build] + uses: ./.github/workflows/proxy_lib.yml diff --git a/.github/workflows/proxy_lib.yml b/.github/workflows/proxy_lib.yml new file mode 100644 index 000000000..635113378 --- /dev/null +++ b/.github/workflows/proxy_lib.yml @@ -0,0 +1,62 @@ +name: Proxy library + +on: workflow_call + +permissions: + contents: read + +jobs: + ubuntu-build: + name: Build - Ubuntu + + strategy: + matrix: + os: ['ubuntu-22.04'] + build_type: [Release, Debug] + compiler: [{c: gcc, cxx: g++}] + runs-on: ${{matrix.os}} + + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Install apt packages + run: | + sudo apt-get update + sudo apt-get install -y cmake libhwloc-dev libjemalloc-dev libtbb-dev + + - name: Configure build + run: > + cmake + -B ${{github.workspace}}/build + -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + -DCMAKE_C_COMPILER=${{matrix.compiler.c}} + -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} + -DUMF_BUILD_SHARED_LIBRARY=ON + -DUMF_BUILD_BENCHMARKS=ON + -DUMF_BUILD_TESTS=ON + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=OFF + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON + -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON + -DUMF_ENABLE_POOL_TRACKING=OFF + + - name: Build UMF + run: cmake --build ${{github.workspace}}/build -j $(nproc) + + - name: Run "ctest --output-on-failure" with proxy library + working-directory: ${{github.workspace}}/build + run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure + + - name: Run "./test/umf_test-memoryPool" with proxy library + working-directory: ${{github.workspace}}/build + run: LD_PRELOAD=./lib/libumf_proxy.so ./test/umf_test-memoryPool + + - name: Run "/usr/bin/ls" with proxy library + working-directory: ${{github.workspace}}/build + run: LD_PRELOAD=./lib/libumf_proxy.so /usr/bin/ls + + - name: Run "/usr/bin/date" with proxy library + working-directory: ${{github.workspace}}/build + run: LD_PRELOAD=./lib/libumf_proxy.so /usr/bin/date diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3768a01c6..a83d394d1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,11 @@ include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) +# set UMF_PROXY_LIB_BASED_ON_POOL to one of: +# - SCALABLE +# - JEMALLOC +set(UMF_PROXY_LIB_BASED_ON_POOL SCALABLE CACHE STRING "A UMF pool the proxy library is based on (SCALABLE or JEMALLOC)" FORCE) + add_subdirectory(utils) set(UMF_LIBS umf_utils) @@ -103,3 +108,26 @@ install(TARGETS umf ) add_subdirectory(pool) + +# TODO: enable proxy_lib on Windows +if(LINUX) + if(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL SCALABLE) + set(PROXY_LIB_USES_SCALABLE_POOL ON) + set(PROXY_LIBS umf scalable_pool) + if(UMF_BUILD_LIBUMF_POOL_SCALABLE) + add_subdirectory(proxy_lib) + else() + message(STATUS "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==SCALABLE but UMF_BUILD_LIBUMF_POOL_SCALABLE is OFF") + endif() + elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL JEMALLOC) + set(PROXY_LIB_USES_JEMALLOC_POOL ON) + set(PROXY_LIBS umf jemalloc_pool) + if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) + add_subdirectory(proxy_lib) + else() + message(STATUS "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==JEMALLOC but UMF_BUILD_LIBUMF_POOL_JEMALLOC is OFF") + endif() + else() + message(FATAL_ERROR "Proxy library: pool manager not chosen or set to a non-supported one (see UMF_PROXY_LIB_BASED_ON_POOL)") + endif() +endif() diff --git a/src/base_alloc/base_alloc.c b/src/base_alloc/base_alloc.c index feec05d7d..37d5a1bac 100644 --- a/src/base_alloc/base_alloc.c +++ b/src/base_alloc/base_alloc.c @@ -36,9 +36,9 @@ struct umf_ba_main_pool_meta_t { size_t chunk_size; // size of all memory chunks in this pool os_mutex_t free_lock; // lock of free_list umf_ba_chunk_t *free_list; // list of free chunks + size_t n_allocs; // number of allocated chunks #ifndef NDEBUG size_t n_pools; - size_t n_allocs; size_t n_chunks; #endif /* NDEBUG */ }; @@ -135,9 +135,9 @@ umf_ba_pool_t *umf_ba_create(size_t size) { pool->metadata.pool_size = pool_size; pool->metadata.chunk_size = chunk_size; pool->next_pool = NULL; // this is the only pool now + pool->metadata.n_allocs = 0; #ifndef NDEBUG pool->metadata.n_pools = 1; - pool->metadata.n_allocs = 0; pool->metadata.n_chunks = 0; #endif /* NDEBUG */ @@ -187,8 +187,8 @@ void *umf_ba_alloc(umf_ba_pool_t *pool) { umf_ba_chunk_t *chunk = pool->metadata.free_list; pool->metadata.free_list = pool->metadata.free_list->next; -#ifndef NDEBUG pool->metadata.n_allocs++; +#ifndef NDEBUG ba_debug_checks(pool); #endif /* NDEBUG */ util_mutex_unlock(&pool->metadata.free_lock); @@ -230,18 +230,30 @@ void umf_ba_free(umf_ba_pool_t *pool, void *ptr) { assert(pool_contains_pointer(pool, ptr)); chunk->next = pool->metadata.free_list; pool->metadata.free_list = chunk; -#ifndef NDEBUG pool->metadata.n_allocs--; +#ifndef NDEBUG ba_debug_checks(pool); #endif /* NDEBUG */ util_mutex_unlock(&pool->metadata.free_lock); } void umf_ba_destroy(umf_ba_pool_t *pool) { + // Do not destroy if we are running in the proxy library, + // because it may need those resources till + // the very end of exiting the application. + if (pool->metadata.n_allocs && is_running_in_proxy_lib()) { + return; + } + #ifndef NDEBUG - assert(pool->metadata.n_allocs == 0); ba_debug_checks(pool); + if (pool->metadata.n_allocs) { + fprintf(stderr, "umf_ba_destroy(): pool->metadata.n_allocs = %zu\n", + pool->metadata.n_allocs); + assert(pool->metadata.n_allocs == 0); + } #endif /* NDEBUG */ + size_t size = pool->metadata.pool_size; umf_ba_next_pool_t *current_pool; umf_ba_next_pool_t *next_pool = pool->next_pool; diff --git a/src/base_alloc/base_alloc_linear.c b/src/base_alloc/base_alloc_linear.c index d3059d664..c12de81a2 100644 --- a/src/base_alloc/base_alloc_linear.c +++ b/src/base_alloc/base_alloc_linear.c @@ -161,9 +161,17 @@ void *umf_ba_linear_alloc(umf_ba_linear_pool_t *pool, size_t size) { } void umf_ba_linear_destroy(umf_ba_linear_pool_t *pool) { + // Do not destroy if we are running in the proxy library, + // because it may need those resources till + // the very end of exiting the application. + if (is_running_in_proxy_lib()) { + return; + } + #ifndef NDEBUG ba_debug_checks(pool); #endif /* NDEBUG */ + umf_ba_next_linear_pool_t *current_pool; umf_ba_next_linear_pool_t *next_pool = pool->next_pool; while (next_pool) { diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 39a3c1e80..4f9db7804 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -10,6 +10,7 @@ #include "provider_tracking.h" #include "base_alloc_global.h" #include "critnib.h" +#include "utils_common.h" #include "utils_concurrency.h" #include @@ -357,18 +358,23 @@ static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, } if (n_items) { - if (pool) { - fprintf(stderr, - "ASSERT: tracking provider of pool %p is not empty! (%zu " - "items left)\n", - (void *)pool, n_items); - } else { - fprintf( - stderr, - "ASSERT: tracking provider is not empty! (%zu items left)\n", - n_items); + // Do not assert if we are running in the proxy library, + // because it may need those resources till + // the very end of exiting the application. + if (!is_running_in_proxy_lib()) { + if (pool) { + fprintf(stderr, + "ASSERT: tracking provider of pool %p is not empty! " + "(%zu items left)\n", + (void *)pool, n_items); + } else { + fprintf(stderr, + "ASSERT: tracking provider is not empty! (%zu items " + "left)\n", + n_items); + } + assert(n_items == 0); } - assert(n_items == 0); } } #endif /* NDEBUG */ @@ -505,12 +511,24 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { return; } + // Do not destroy if we are running in the proxy library, + // because it may need those resources till + // the very end of exiting the application. + if (is_running_in_proxy_lib()) { + return; + } + #ifndef NDEBUG check_if_tracker_is_empty(handle, NULL); #endif /* NDEBUG */ + // We have to zero all inner pointers, + // because the tracker handle can be copied + // and used in many places. critnib_delete(handle->map); + handle->map = NULL; util_mutex_destroy_not_free(&handle->splitMergeMutex); umf_ba_destroy(handle->tracker_allocator); + handle->tracker_allocator = NULL; umf_ba_global_free(handle, sizeof(struct umf_memory_tracker_t)); } diff --git a/src/proxy_lib/CMakeLists.txt b/src/proxy_lib/CMakeLists.txt new file mode 100644 index 000000000..02315f720 --- /dev/null +++ b/src/proxy_lib/CMakeLists.txt @@ -0,0 +1,54 @@ +# Copyright (C) 2023-2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) + +set(PROXY_SOURCES + proxy_lib.c +) + +set(PROXY_SOURCES_LINUX + proxy_lib_linux.c +) + +set(PROXY_SOURCES_WINDOWS + proxy_lib_windows.c +) + +set(PROXY_SOURCES_MACOSX + proxy_lib_linux.c +) + +if(LINUX) + set(PROXY_SOURCES ${PROXY_SOURCES} ${PROXY_SOURCES_LINUX}) +elseif(WINDOWS) + set(PROXY_SOURCES ${PROXY_SOURCES} ${PROXY_SOURCES_WINDOWS}) +elseif(MACOSX) + set(PROXY_SOURCES ${PROXY_SOURCES} ${PROXY_SOURCES_MACOSX}) +endif() + +add_umf_library(NAME umf_proxy + TYPE SHARED + SRCS ${PROXY_SOURCES} + LIBS ${PROXY_LIBS} + LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/proxy_lib.map + WINDOWS_DEF_FILE ${CMAKE_CURRENT_SOURCE_DIR}/proxy_lib.def) + +add_library(${PROJECT_NAME}::proxy ALIAS umf_proxy) + +if(PROXY_LIB_USES_SCALABLE_POOL) + target_compile_definitions(umf_proxy PRIVATE PROXY_LIB_USES_SCALABLE_POOL=1) +elseif(PROXY_LIB_USES_JEMALLOC_POOL) + target_compile_definitions(umf_proxy PRIVATE PROXY_LIB_USES_JEMALLOC_POOL=1) +endif() + +target_include_directories(umf_proxy PUBLIC + $ + $ + $ + $ +) + +install(TARGETS umf_proxy + EXPORT ${PROJECT_NAME}-targets) diff --git a/src/proxy_lib/proxy_lib.c b/src/proxy_lib/proxy_lib.c new file mode 100644 index 000000000..084bf6a10 --- /dev/null +++ b/src/proxy_lib/proxy_lib.c @@ -0,0 +1,261 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +/* + * UMF proxy library - a library for intercepting user allocation requests + * + * It intercepts following APIs: + * - aligned_alloc() + * - calloc() + * - free() + * - malloc() + * - malloc_usable_size() + * - realloc() + */ + +#if (defined PROXY_LIB_USES_JEMALLOC_POOL) +#include +#define umfPoolManagerOps umfJemallocPoolOps +#elif (defined PROXY_LIB_USES_SCALABLE_POOL) +#include +#define umfPoolManagerOps umfScalablePoolOps +#else +#error Pool manager not defined +#endif + +#include +#include +#include +#include + +#include +#include +#include + +#include "base_alloc_linear.h" +#include "proxy_lib.h" +#include "utils_common.h" +#include "utils_concurrency.h" + +/* + * The UMF proxy library uses two memory allocators: + * 1) the "LEAK" internal linear base allocator based on the anonymous mapped + * memory that will NOT be destroyed (with API ba_leak_*()). + * 2) the main one - UMF pool allocator. + * + * Ad 1) + * The "LEAK" internal linear base allocator is used from the very beginning + * to the creation of a UMF pool in the constructor of the proxy library. + * It is used to allocate memory for OS specific data used during loading and unloading + * applications (for example _dl_init() and _dl_fini() on Linux storing data of all + * constructors and destructors that have to be called) and also memory needed + * by umfMemoryProviderCreate() and umfPoolCreate(). + * That memory will be leaked on purpose (OS will have to free it during destroying + * the process), because we cannot free the memory containing data of destructors + * that have to be called at the end (for example memory allocated by _dl_init() + * and used internally by _dl_fini() on Linux). + * The "LEAK" internal linear base allocator uses about 900 kB on Linux. + * + * Ad 2) + * The UMF pool allocator (the main one) is used from the creation to the destruction + * of a UMF pool to allocate memory needed by an application. It should be freed + * by an application. + */ + +static UTIL_ONCE_FLAG Base_alloc_leak_initialized = UTIL_ONCE_FLAG_INIT; +static umf_ba_linear_pool_t *Base_alloc_leak = NULL; +static umf_memory_provider_handle_t OS_memory_provider = NULL; +static umf_memory_pool_handle_t Proxy_pool = NULL; + +// it protects us from recursion in umfPool*() +static __TLS int was_called_from_umfPool = 0; + +/*****************************************************************************/ +/*** The constructor and destructor of the proxy library *********************/ +/*****************************************************************************/ + +void proxy_lib_create_common(void) { + umf_os_memory_provider_params_t os_params = + umfOsMemoryProviderParamsDefault(); + enum umf_result_t umf_result; + + umf_result = umfMemoryProviderCreate(umfOsMemoryProviderOps(), &os_params, + &OS_memory_provider); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "error: creating OS memory provider failed\n"); + exit(-1); + } + + umf_result = umfPoolCreate(umfPoolManagerOps(), OS_memory_provider, NULL, 0, + &Proxy_pool); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "error: creating UMF pool manager failed\n"); + exit(-1); + } + // The UMF pool has just been created (Proxy_pool != NULL). Stop using + // the linear allocator and start using the UMF pool allocator from now on. +} + +void proxy_lib_destroy_common(void) { + // We cannot destroy 'Base_alloc_leak' nor 'Proxy_pool' nor 'OS_memory_provider', + // because it could lead to use-after-free in the program's unloader + // (for example _dl_fini() on Linux). +} + +/*****************************************************************************/ +/*** Generic version of realloc() of linear base allocator *******************/ +/*****************************************************************************/ + +static inline void *ba_generic_realloc(umf_ba_linear_pool_t *pool, void *ptr, + size_t new_size, size_t max_size) { + assert(ptr); // it should be verified in the main realloc() + assert(new_size); // it should be verified in the main realloc() + assert(max_size); // max_size should be set in the main realloc() + + void *new_ptr = umf_ba_linear_alloc(pool, new_size); + if (!new_ptr) { + return NULL; + } + + if (new_size > max_size) { + new_size = max_size; + } + + memcpy(new_ptr, ptr, new_size); + + return new_ptr; +} + +/*****************************************************************************/ +/*** The "LEAK" linear base allocator functions ******************************/ +/*****************************************************************************/ + +static void ba_leak_create(void) { Base_alloc_leak = umf_ba_linear_create(0); } + +// it does not implement destroy(), because it will not free memory at all + +static inline void *ba_leak_malloc(size_t size) { + util_init_once(&Base_alloc_leak_initialized, ba_leak_create); + return umf_ba_linear_alloc(Base_alloc_leak, size); +} + +static inline void *ba_leak_calloc(size_t nmemb, size_t size) { + util_init_once(&Base_alloc_leak_initialized, ba_leak_create); + // umf_ba_linear_alloc() returns zeroed memory + return umf_ba_linear_alloc(Base_alloc_leak, nmemb * size); +} + +static inline void *ba_leak_realloc(void *ptr, size_t size, size_t max_size) { + util_init_once(&Base_alloc_leak_initialized, ba_leak_create); + return ba_generic_realloc(Base_alloc_leak, ptr, size, max_size); +} + +static inline void *ba_leak_aligned_alloc(size_t alignment, size_t size) { + util_init_once(&Base_alloc_leak_initialized, ba_leak_create); + void *ptr = umf_ba_linear_alloc(Base_alloc_leak, size + alignment); + return (void *)ALIGN_UP((uintptr_t)ptr, alignment); +} + +static inline size_t ba_leak_pool_contains_pointer(void *ptr) { + return umf_ba_linear_pool_contains_pointer(Base_alloc_leak, ptr); +} + +/*****************************************************************************/ +/*** The UMF pool allocator functions (the public API) ***********************/ +/*****************************************************************************/ + +void *malloc(size_t size) { + if (!was_called_from_umfPool && Proxy_pool) { + was_called_from_umfPool = 1; + void *ptr = umfPoolMalloc(Proxy_pool, size); + was_called_from_umfPool = 0; + return ptr; + } + + return ba_leak_malloc(size); +} + +void *calloc(size_t nmemb, size_t size) { + if (!was_called_from_umfPool && Proxy_pool) { + was_called_from_umfPool = 1; + void *ptr = umfPoolCalloc(Proxy_pool, nmemb, size); + was_called_from_umfPool = 0; + return ptr; + } + + return ba_leak_calloc(nmemb, size); +} + +void *realloc(void *ptr, size_t size) { + if (ptr == NULL) { + return malloc(size); + } + + if (size == 0) { + free(ptr); + return NULL; + } + + size_t leak_pool_contains_pointer = ba_leak_pool_contains_pointer(ptr); + if (leak_pool_contains_pointer) { + return ba_leak_realloc(ptr, size, leak_pool_contains_pointer); + } + + if (Proxy_pool) { + was_called_from_umfPool = 1; + void *new_ptr = umfPoolRealloc(Proxy_pool, ptr, size); + was_called_from_umfPool = 0; + return new_ptr; + } + + assert(0); + return NULL; +} + +void free(void *ptr) { + if (ptr == NULL) { + return; + } + + if (ba_leak_pool_contains_pointer(ptr)) { + // allocations from the leak linear base allocator will not be freed at all + return; + } + + if (Proxy_pool) { + if (umfPoolFree(Proxy_pool, ptr) != UMF_RESULT_SUCCESS) { + fprintf(stderr, "error: umfPoolFree() failed\n"); + assert(0); + } + return; + } + + assert(0); + return; +} + +void *aligned_alloc(size_t alignment, size_t size) { + if (!was_called_from_umfPool && Proxy_pool) { + was_called_from_umfPool = 1; + void *ptr = umfPoolAlignedMalloc(Proxy_pool, size, alignment); + was_called_from_umfPool = 0; + return ptr; + } + + return ba_leak_aligned_alloc(alignment, size); +} + +size_t malloc_usable_size(void *ptr) { + if (!was_called_from_umfPool && Proxy_pool) { + was_called_from_umfPool = 1; + size_t size = umfPoolMallocUsableSize(Proxy_pool, ptr); + was_called_from_umfPool = 0; + return size; + } + + return 0; // unsupported in this case +} diff --git a/src/proxy_lib/proxy_lib.def b/src/proxy_lib/proxy_lib.def new file mode 100644 index 000000000..0a67554e1 --- /dev/null +++ b/src/proxy_lib/proxy_lib.def @@ -0,0 +1,15 @@ +;;;; Begin Copyright Notice +; Copyright (C) 2024 Intel Corporation +; Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +;;;; End Copyright Notice + +LIBRARY UMF_PROXY +EXPORTS + DllMain + aligned_alloc + calloc + free + malloc + malloc_usable_size + realloc diff --git a/src/proxy_lib/proxy_lib.h b/src/proxy_lib/proxy_lib.h new file mode 100644 index 000000000..2e67fbbdd --- /dev/null +++ b/src/proxy_lib/proxy_lib.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_PROXY_LIB_H +#define UMF_PROXY_LIB_H 1 + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void proxy_lib_create_common(void); +void proxy_lib_destroy_common(void); + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_PROXY_LIB_H */ diff --git a/src/proxy_lib/proxy_lib.map b/src/proxy_lib/proxy_lib.map new file mode 100644 index 000000000..5d93d03ba --- /dev/null +++ b/src/proxy_lib/proxy_lib.map @@ -0,0 +1,17 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# linker VERSION script + +{ + global: + aligned_alloc; + calloc; + free; + malloc; + malloc_usable_size; + realloc; + local: + *; +}; diff --git a/src/proxy_lib/proxy_lib_linux.c b/src/proxy_lib/proxy_lib_linux.c new file mode 100644 index 000000000..a6299eeae --- /dev/null +++ b/src/proxy_lib/proxy_lib_linux.c @@ -0,0 +1,29 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#define _GNU_SOURCE +#include + +#include +#include + +#include "proxy_lib.h" + +// The priority 102 is used, because the constructor should be called as the second one +// (just after the first constructor of the base allocator with priority 101) +// and the destructor as the last but one (just before the last destructor +// of the base allocator with priority 101), because this library +// provides the memory allocation API. +void __attribute__((constructor(102))) proxy_lib_create(void) { + proxy_lib_create_common(); +} + +void __attribute__((destructor(102))) proxy_lib_destroy(void) { + proxy_lib_destroy_common(); +} diff --git a/src/proxy_lib/proxy_lib_windows.c b/src/proxy_lib/proxy_lib_windows.c new file mode 100644 index 000000000..1199f102d --- /dev/null +++ b/src/proxy_lib/proxy_lib_windows.c @@ -0,0 +1,27 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include + +#include + +#include "proxy_lib.h" + +static void proxy_lib_create(void) { proxy_lib_create_common(); } + +static void proxy_lib_destroy(void) { proxy_lib_destroy_common(); } + +BOOL APIENTRY DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { + if (fdwReason == DLL_PROCESS_DETACH) { + proxy_lib_destroy(); + } else if (fdwReason == DLL_PROCESS_ATTACH) { + proxy_lib_create(); + } + return TRUE; +} diff --git a/src/utils/utils_common.h b/src/utils/utils_common.h index e30ba0a96..a43c1d3e2 100644 --- a/src/utils/utils_common.h +++ b/src/utils/utils_common.h @@ -19,11 +19,45 @@ extern "C" { #endif -#ifdef _WIN32 +#ifdef _WIN32 /* Windows */ + #define __TLS __declspec(thread) -#else + +static inline char *os_getenv(const char *name) { + char *buffer; + size_t numberOfElements; + errno_t err = _dupenv_s(&buffer, &numberOfElements, name); + if (err) { + return NULL; + } + + return buffer; +} + +static inline void os_free_getenv(char *val) { free(val); } + +#else /* Linux */ + #define __TLS __thread -#endif + +static inline char *os_getenv(const char *name) { return getenv(name); } +static inline void os_free_getenv(const char *val) { + (void)val; // unused +} + +#endif /* _WIN32 */ + +// check if we are running in the proxy library +static inline int is_running_in_proxy_lib(void) { + int is_in_proxy_lib_val = 0; + char *ld_preload = os_getenv("LD_PRELOAD"); + if (ld_preload && strstr(ld_preload, "libumf_proxy.so")) { + is_in_proxy_lib_val = 1; + } + + os_free_getenv(ld_preload); + return is_in_proxy_lib_val; +} #define NOFUNCTION \ do { \ diff --git a/test/test_make_install.txt b/test/test_make_install.txt index 9078d8682..63bf06f26 100644 --- a/test/test_make_install.txt +++ b/test/test_make_install.txt @@ -27,6 +27,7 @@ ./lib/libjemalloc_pool.a ./lib/libscalable_pool.a ./lib/libumf.@LIB_EXT_STR@ +./lib/libumf_proxy.so ./lib/libumf_utils.a ./share ./share/doc