From 5f9b2eab32e01e542121db93d88e35ba790a9013 Mon Sep 17 00:00:00 2001 From: HU Xuesong <87519979+huxs001@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:15:31 +0800 Subject: [PATCH 1/5] When `soloOutFormatFeaturesGeneField3` is `+`, write GeneBiotype as FeaturesGeneField3. --- source/SoloFeature_outputResults.cpp | 5 ++++- source/parametersDefault | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/SoloFeature_outputResults.cpp b/source/SoloFeature_outputResults.cpp index bf07c4cf..31d8c7ed 100644 --- a/source/SoloFeature_outputResults.cpp +++ b/source/SoloFeature_outputResults.cpp @@ -35,7 +35,10 @@ void SoloFeature::outputResults(bool cellFilterYes, string outputPrefixMat) for (uint32 ii=0; ii Date: Wed, 15 Jan 2025 13:51:24 +0800 Subject: [PATCH 2/5] Apply patches/0002-donotuse_own_htslib.patch --- source/BAMfunctions.cpp | 2 +- source/IncludeDefine.h | 4 ++-- source/Makefile | 19 ++++++------------- source/Parameters.cpp | 1 - source/STAR.cpp | 2 +- source/bamRemoveDuplicates.cpp | 2 +- source/bam_cat.c | 4 ++-- source/bam_cat.h | 2 +- source/signalFromBAM.h | 2 +- 9 files changed, 15 insertions(+), 23 deletions(-) diff --git a/source/BAMfunctions.cpp b/source/BAMfunctions.cpp index af6ce237..632deca5 100644 --- a/source/BAMfunctions.cpp +++ b/source/BAMfunctions.cpp @@ -1,5 +1,5 @@ #include "BAMfunctions.h" -#include "htslib/htslib/kstring.h" +#include string bam_cigarString (bam1_t *b) {//output CIGAR string diff --git a/source/IncludeDefine.h b/source/IncludeDefine.h index ddf0a02f..e5511797 100755 --- a/source/IncludeDefine.h +++ b/source/IncludeDefine.h @@ -30,8 +30,8 @@ #define ERROR_OUT string ( __FILE__ ) +":"+ to_string ( (uint) __LINE__ ) +":"+ string ( __FUNCTION__ ) //external libs -#define SAMTOOLS_BGZF_H "htslib/htslib/bgzf.h" -#define SAMTOOLS_SAM_H "htslib/htslib/sam.h" +#define SAMTOOLS_BGZF_H +#define SAMTOOLS_SAM_H using namespace std; diff --git a/source/Makefile b/source/Makefile index a6b5fcb3..f172797b 100644 --- a/source/Makefile +++ b/source/Makefile @@ -12,8 +12,8 @@ CXXFLAGSextra ?= CXX ?= g++ # pre-defined flags -LDFLAGS_shared := -pthread -Lhtslib -Bstatic -lhts -Bdynamic -lz -LDFLAGS_static := -static -static-libgcc -pthread -Lhtslib -lhts -lz +LDFLAGS_shared := -pthread -lhts -Bdynamic -lz +LDFLAGS_static := -static -static-libgcc -pthread -lhts -lz LDFLAGS_Mac :=-pthread -lz htslib/libhts.a LDFLAGS_Mac_static :=-pthread -lz -static-libgcc htslib/libhts.a LDFLAGS_gdb := $(LDFLAGS_shared) @@ -45,7 +45,7 @@ CFLAGS ?= -pipe -Wall -Wextra -O3 CXXFLAGS_SIMD ?= -mavx2 # Unconditionally set essential flags and optimization options -CXXFLAGS_common := -std=c++11 -fopenmp $(COMPTIMEPLACE) $(GIT_BRANCH_COMMIT_DIFF) +CXXFLAGS_common := -std=c++11 -fopenmp $(COMPTIMEPLACE) $(CCFLAGS_common_add) CXXFLAGS_main := -O3 $(CXXFLAGS_common) CXXFLAGS_gdb := -O0 -g3 $(CXXFLAGS_common) @@ -97,10 +97,10 @@ SOURCES := $(wildcard *.cpp) $(wildcard *.c) %.o : %.cpp - $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $(COMPTIMEPLACE) $< %.o : %.c - $(CXX) -c $(CPPFLAGS) $(CFLAGS) $< + $(CXX) -c $(CPPFLAGS) $(CFLAGS) $(COMPTIMEPLACE) $< all: cleanCompileInfo STAR$(SFX) @@ -114,8 +114,6 @@ clean: .PHONY: CLEAN CLEAN: clean - $(MAKE) -C htslib clean - .PHONY: clean_solo clean_solo: @@ -131,7 +129,7 @@ ifneq ($(MAKECMDGOALS),CLEAN) ifneq ($(MAKECMDGOALS),clean_solo) ifneq ($(MAKECMDGOALS),STARforMac) ifneq ($(MAKECMDGOALS),STARforMacGDB) -Depend.list: $(SOURCES) parametersDefault.xxd htslib +Depend.list: $(SOURCES) parametersDefault.xxd echo $(SOURCES) 'rm' -f ./Depend.list $(CXX) $(CXXFLAGS_common) -MM $^ >> Depend.list @@ -143,11 +141,6 @@ endif endif endif -htslib : htslib/libhts.a - -htslib/libhts.a : - $(MAKE) -C htslib lib-static - parametersDefault.xxd: parametersDefault xxd -i parametersDefault > parametersDefault.xxd diff --git a/source/Parameters.cpp b/source/Parameters.cpp index 26f6a2dd..d6246bb7 100755 --- a/source/Parameters.cpp +++ b/source/Parameters.cpp @@ -377,7 +377,6 @@ void Parameters::inputParameters (int argInN, char* argIn[]) {//input parameters inOut->logMain << "STAR version=" << STAR_VERSION << "\n"; inOut->logMain << "STAR compilation time,server,dir=" << COMPILATION_TIME_PLACE << "\n"; - inOut->logMain << "STAR git: " << GIT_BRANCH_COMMIT_DIFF << "\n"; #ifdef COMPILE_FOR_LONG_READS inOut->logMain << "Compiled for LONG reads" << "\n"; #endif diff --git a/source/STAR.cpp b/source/STAR.cpp index f9b84b3e..365c6329 100755 --- a/source/STAR.cpp +++ b/source/STAR.cpp @@ -30,7 +30,7 @@ #include "twoPassRunPass1.h" -#include "htslib/htslib/sam.h" +#include #include "parametersDefault.xxd" void usage(int usageType) diff --git a/source/bamRemoveDuplicates.cpp b/source/bamRemoveDuplicates.cpp index 13958d02..d7cc7ee3 100644 --- a/source/bamRemoveDuplicates.cpp +++ b/source/bamRemoveDuplicates.cpp @@ -1,7 +1,7 @@ #include #include "bamRemoveDuplicates.h" #include -#include "htslib/htslib/sam.h" +#include #include "IncludeDefine.h" #include SAMTOOLS_BGZF_H #include "ErrorWarning.h" diff --git a/source/bam_cat.c b/source/bam_cat.c index 944cafbb..efaabe4c 100644 --- a/source/bam_cat.c +++ b/source/bam_cat.c @@ -52,8 +52,8 @@ THE SOFTWARE. #include #include -#include "htslib/htslib/bgzf.h" -#include "htslib/htslib/sam.h" +#include +#include #include #define BUF_SIZE 0x10000 diff --git a/source/bam_cat.h b/source/bam_cat.h index 337d150c..f8f7fcda 100644 --- a/source/bam_cat.h +++ b/source/bam_cat.h @@ -1,7 +1,7 @@ #ifndef CODE_bam_cat #define CODE_bam_cat -#include "htslib/htslib/sam.h" +#include int bam_cat(int nfn, char * const *fn, const bam_hdr_t *h, const char* outbam); diff --git a/source/signalFromBAM.h b/source/signalFromBAM.h index 0406fdcd..aac44168 100644 --- a/source/signalFromBAM.h +++ b/source/signalFromBAM.h @@ -1,6 +1,6 @@ #ifndef CODE_signalFromBAM #define CODE_signalFromBAM -#include "htslib/htslib/sam.h" +#include #include #include #include "Stats.h" From 665875a5b5d5a178dcb2f565c9ad4b8eb40bd6c6 Mon Sep 17 00:00:00 2001 From: Yuuki Galaxy <863605+galaxy001@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:35:39 +0800 Subject: [PATCH 3/5] Add source/mmap_malloc modified: source/Makefile new file: source/mmap_allocator.hpp new file: source/mmap_malloc.h new file: source/mmap_malloc/default_config.c new file: source/mmap_malloc/mmap_allocator.c --- source/Makefile | 25 +- source/mmap_allocator.hpp | 71 +++++ source/mmap_malloc.h | 20 ++ source/mmap_malloc/constants.h | 14 + source/mmap_malloc/default_config.c | 40 +++ source/mmap_malloc/default_config.h | 19 ++ source/mmap_malloc/heap.c | 261 ++++++++++++++++ source/mmap_malloc/heap.h | 58 ++++ source/mmap_malloc/list.h | 120 ++++++++ source/mmap_malloc/mmap_allocator.c | 446 ++++++++++++++++++++++++++++ source/mmap_malloc/mmap_mgr.c | 91 ++++++ source/mmap_malloc/mmap_mgr.h | 10 + source/mmap_malloc/profiling.c | 36 +++ source/mmap_malloc/profiling.h | 47 +++ 14 files changed, 1255 insertions(+), 3 deletions(-) create mode 100644 source/mmap_allocator.hpp create mode 100644 source/mmap_malloc.h create mode 100644 source/mmap_malloc/constants.h create mode 100644 source/mmap_malloc/default_config.c create mode 100644 source/mmap_malloc/default_config.h create mode 100644 source/mmap_malloc/heap.c create mode 100644 source/mmap_malloc/heap.h create mode 100644 source/mmap_malloc/list.h create mode 100644 source/mmap_malloc/mmap_allocator.c create mode 100644 source/mmap_malloc/mmap_mgr.c create mode 100644 source/mmap_malloc/mmap_mgr.h create mode 100644 source/mmap_malloc/profiling.c create mode 100644 source/mmap_malloc/profiling.h diff --git a/source/Makefile b/source/Makefile index f172797b..65677632 100644 --- a/source/Makefile +++ b/source/Makefile @@ -10,6 +10,7 @@ CXXFLAGSextra ?= # user may define the compiler CXX ?= g++ +CC ?= gcc # pre-defined flags LDFLAGS_shared := -pthread -lhts -Bdynamic -lz @@ -18,6 +19,14 @@ LDFLAGS_Mac :=-pthread -lz htslib/libhts.a LDFLAGS_Mac_static :=-pthread -lz -static-libgcc htslib/libhts.a LDFLAGS_gdb := $(LDFLAGS_shared) +PKGCONFIG ?= pkg-config +PKGLIBS := htslib zlib +PKGCPPFLAGS := $(shell $(PKGCONFIG) --cflags $(PKGLIBS)) +PKGLDFLAGS := $(shell $(PKGCONFIG) --libs $(PKGLIBS)) + +LDFLAGSextra += $(PKGLDFLAGS) +CXXFLAGSextra += $(PKGCPPFLAGS) + DATE_FMT = --iso-8601=seconds ifdef SOURCE_DATE_EPOCH BUILD_DATE ?= $(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u -r "$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u "$(DATE_FMT)") @@ -91,19 +100,29 @@ OBJECTS = systemFunctions.o funPrimaryAlignMark.o \ sjdbLoadFromFiles.o sjdbLoadFromStream.o sjdbPrepare.o sjdbBuildIndex.o sjdbInsertJunctions.o mapThreadsSpawn.o \ Parameters_readFilesInit.o Parameters_openReadsFiles.cpp Parameters_closeReadsFiles.cpp Parameters_readSAMheader.o \ bam_cat.o serviceFuns.o GlobalVariables.cpp \ - BAMoutput.o BAMfunctions.o ReadAlign_alignBAM.o BAMbinSortByCoordinate.o signalFromBAM.o bamRemoveDuplicates.o BAMbinSortUnmapped.o + BAMoutput.o BAMfunctions.o ReadAlign_alignBAM.o BAMbinSortByCoordinate.o signalFromBAM.o bamRemoveDuplicates.o BAMbinSortUnmapped.o \ + $(MMAP_MALLOC_SRC_DIR)/libmmap_allocator.a SOURCES := $(wildcard *.cpp) $(wildcard *.c) +MMAP_MALLOC_SRC_DIR = mmap_malloc +MMAP_MALLOC_LIB_SRCS = $(wildcard $(MMAP_MALLOC_SRC_DIR)/*.c) +MMAP_MALLOC_OBJ_FILES = $(patsubst $(MMAP_MALLOC_SRC_DIR)/%.c,$(MMAP_MALLOC_SRC_DIR)/%.o,$(MMAP_MALLOC_LIB_SRCS)) + +$(MMAP_MALLOC_SRC_DIR)/%.o : $(MMAP_MALLOC_SRC_DIR)/%.c + $(CC) -Wall -fPIC -pthread -O3 -Immap_malloc -c $< -o $@ %.o : %.cpp - $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $(COMPTIMEPLACE) $< + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $(CXXFLAGSextra) $(COMPTIMEPLACE) $< %.o : %.c - $(CXX) -c $(CPPFLAGS) $(CFLAGS) $(COMPTIMEPLACE) $< + $(CXX) -c $(CPPFLAGS) $(CFLAGS) $(CXXFLAGSextra) $(COMPTIMEPLACE) $< all: cleanCompileInfo STAR$(SFX) +$(MMAP_MALLOC_SRC_DIR)/libmmap_allocator.a: $(MMAP_MALLOC_OBJ_FILES) + ar r $@ $^ + opal/opal.o : opal/opal.cpp opal/opal.h cd opal && \ $(CXX) -c -I./ -std=c++11 $(CPPFLAGS) $(CXXFLAGS) $(CXXFLAGSextra) $(CXXFLAGS_SIMD) opal.cpp diff --git a/source/mmap_allocator.hpp b/source/mmap_allocator.hpp new file mode 100644 index 00000000..029119a8 --- /dev/null +++ b/source/mmap_allocator.hpp @@ -0,0 +1,71 @@ +#ifndef _MALLOC_ALLOCATOR_H +#define _MALLOC_ALLOCATOR_H 1 + +// g++ -std=gnu++20 + +#include +#include +#include +#include + +#include "mmap_malloc.h" + +namespace galaxy { + +template +class mmap_allocator { + public: + using value_type = _Tp; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 2103. propagate_on_container_move_assignment + using propagate_on_container_move_assignment = std::true_type; + + constexpr mmap_allocator() noexcept = default; + constexpr mmap_allocator(const mmap_allocator&) noexcept = default; + + template + constexpr mmap_allocator(const mmap_allocator<_Tp1>&) noexcept {}; + + ~mmap_allocator() noexcept = default; + + // NB: __n is permitted to be 0. The C++ standard says nothing + // about what the return value is when __n == 0. + [[nodiscard]] _Tp* allocate(size_type __n, const void* = nullptr) { + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 3308. std::allocator().allocate(n) + static_assert(sizeof(_Tp) != 0, "cannot allocate incomplete types"); + + if (__n > this->_M_max_size()) [[unlikely]] { + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 3190. allocator::allocate sometimes returns too little storage + if (__n > (std::size_t(-1) / sizeof(_Tp))) throw std::bad_array_new_length(); + throw std::bad_alloc(); + } + _Tp* __ret = static_cast<_Tp*>(mmap_malloc(__n * sizeof(_Tp))); + if (!__ret) throw std::bad_alloc(); + return __ret; + } + + // __p is not permitted to be a null pointer. + void deallocate(_Tp* __p, size_type) { mmap_free(static_cast(__p)); } + + template + friend constexpr bool operator==(const mmap_allocator&, const mmap_allocator<_Up>&) noexcept { + return true; + } + + private: + constexpr size_type _M_max_size() const noexcept { +#if __PTRDIFF_MAX__ < __SIZE_MAX__ + return std::size_t(__PTRDIFF_MAX__) / sizeof(_Tp); +#else + return std::size_t(-1) / sizeof(_Tp); +#endif + } +}; + +} // namespace galaxy + +#endif diff --git a/source/mmap_malloc.h b/source/mmap_malloc.h new file mode 100644 index 00000000..82d91a60 --- /dev/null +++ b/source/mmap_malloc.h @@ -0,0 +1,20 @@ +#ifndef _MMAP_MALLOC_H +#define _MMAP_MALLOC_H 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +void* mmap_malloc(size_t size); +void mmap_free(void* addr); +void* mmap_calloc(size_t num_elements, size_t element_size); +void* mmap_realloc(void *addr, size_t size); +void* mmap_reallocarray(void *addr, size_t size, size_t count); + +#ifdef __cplusplus +} +#endif + +#endif /* mmap_malloc.h */ diff --git a/source/mmap_malloc/constants.h b/source/mmap_malloc/constants.h new file mode 100644 index 00000000..cd30a54f --- /dev/null +++ b/source/mmap_malloc/constants.h @@ -0,0 +1,14 @@ +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#include +#include +#include + +#define FORCE_INLINE inline __attribute__((always_inline)) +#define LOCAL_HELPER static FORCE_INLINE + +#define DEBUG_HEAP 0 +#define DEBUG_LIST 0 + +#endif \ No newline at end of file diff --git a/source/mmap_malloc/default_config.c b/source/mmap_malloc/default_config.c new file mode 100644 index 00000000..5ade7615 --- /dev/null +++ b/source/mmap_malloc/default_config.c @@ -0,0 +1,40 @@ +#include "default_config.h" + +char* const env_mmap_heap_size = "ENV_MMAP_HEAP_SIZE"; +char* const env_mmap_alloctor_min_bsize = "ENV_MMAP_ALLOCATOR_MIN_BSIZE"; +char* const env_naming_template = "ENV_NAMING_TEMPLATE"; + +char* const env_profile_file_path = "ENV_PROFILE_FILE_PATH"; +char* const env_profile_frequency = "ENV_PROFILE_FREQUENCY"; + +const size_t default_mmap_heap_size = (size_t) 1024 * 1024 * 1024 * 1024 * 4; +/* +$ grep -m1 "address sizes" /proc/cpuinfo +address sizes : 46 bits physical, 57 bits virtual +*/ +const size_t default_mmap_alloctor_min_bsize = (size_t) 1024 * 1024 * 1024 * 2; +char* const default_naming_template = ".mmap_alloc.XXXXXXXXXX"; + +/* +48 bit is 256T, 44bit:4T, 16-1 possible. + +https://www.linuxquestions.org/questions/linux-hardware-18/address-sizes-in-cpuinfo-757456/ + +Every x86 and x86_64 CPU has an address translation system that translates virtual addresses into physical addresses. + +The virtual addresses are the ones programs actually use (they are the numerical values of pointer variables in the program). + +The physical addresses are the ones the CPU sends to the memory controller to generate the signals needed to select specific memory locations. + +Each process has its own translation tables, so a pointer in one process with identical value to a pointer in another process doesn't normally point to the same physical memory. + +The older x86 architecture has a translation design that translates a 32 bit virtual address to a 32 bit physical address. + +Newer x86 architecture also supports a translation (called PAE) that translates 32 bit virtual to 36 bit physical. + +The x86_64 architecture has 48 bit virtual addresses. Pointers take 64 bits, but the top 17 bits of any 64 bit pointer are required to be identical (all 17 zeros or all 17 ones) so 16 of those 17 bits are redundant. + +The x86_64 address translation takes in 48 bit virtual addresses on all models. But the number of physical addresses generated varies by model of CPU chip. + +The architecture has an upper limit on the number of physical address bits. I think that is 52, but I'm not sure I remember correctly. But since no one will be using that much physical ram before current CPU models are obsolete, CPU chips don't actually support that much. They support various numbers of physical address bits from 36 up. 40, which you have, is pretty typical. +*/ diff --git a/source/mmap_malloc/default_config.h b/source/mmap_malloc/default_config.h new file mode 100644 index 00000000..01b2fb04 --- /dev/null +++ b/source/mmap_malloc/default_config.h @@ -0,0 +1,19 @@ +#ifndef DEFAULT_CONFIG_H +#define DEFAULT_CONFIG_H + +#include + +#define MAX_NAMING_TEMPLATE_SIZE 256 + +extern char* const env_mmap_heap_size; +extern char* const env_mmap_alloctor_min_bsize; +extern char* const env_naming_template; + +extern char* const env_profile_file_path; +extern char* const env_profile_frequency; + +extern const size_t default_mmap_heap_size; +extern const size_t default_mmap_alloctor_min_bsize; +extern char* const default_naming_template; + +#endif \ No newline at end of file diff --git a/source/mmap_malloc/heap.c b/source/mmap_malloc/heap.c new file mode 100644 index 00000000..a4326fd8 --- /dev/null +++ b/source/mmap_malloc/heap.c @@ -0,0 +1,261 @@ +#include +#include + +#include "heap.h" +#include "list.h" +#include + +/* +Since these functions are recursive, they cannot be inlined. +Put them in the source code instead. +*/ + +static void heap_heapify_down(heap_t* heap, const size_t idx); +static void heap_heapify_up(heap_t* heap, const size_t idx); + +void LOCAL_HELPER +heap_set_idx(heap_t* heap, const size_t idx, list_node_t node) { + heap->node_array[idx] = node; + node->idx = idx; +} + +size_t LOCAL_HELPER +heap_get_num_elements(heap_t* heap) { + return heap->size - 1; +} + +size_t LOCAL_HELPER +heap_compare_GT(heap_t* heap, const size_t l, const size_t r) { + return (heap->node_array[l]->size > heap->node_array[r]->size) ? l : r; +} + +void LOCAL_HELPER +heap_swap(heap_t* heap, const size_t a, const size_t b) { + heap->node_array[a]->idx = b; + heap->node_array[b]->idx = a; + list_node_t temp = heap->node_array[a]; + heap->node_array[a] = heap->node_array[b]; + heap->node_array[b] = temp; +} + +void LOCAL_HELPER +heap_remove_idx(heap_t* heap, const size_t idx) { + if (!(idx > 0 && idx < heap->size && "Heap remove invalid idx.")) { + fprintf(stderr, "idx: %ld\n", idx); + assert(0); + } + + const size_t heap_elements = heap_get_num_elements(heap); + if (heap_elements == idx) { + heap->size -= 1; + heap->node_array[idx]->idx = HEAP_IDX_NULL; + return; + } + + heap->node_array[idx]->idx = HEAP_IDX_NULL; + heap_set_idx(heap, idx, heap->node_array[heap->size - 1]); + heap->size -= 1; + heap_heapify_down(heap, idx); +} + +list_node_t LOCAL_HELPER +heap_get_root(heap_t* heap) { + if (heap->size <= HEAP_IDX_ROOT) { + return NULL; + } + return heap->node_array[HEAP_IDX_ROOT]; +} + +// Insert a new node to the heap with heap property maintained. +bool LOCAL_HELPER +heap_insert(heap_t* heap, list_node_t node) { + if (heap->size == heap->capacity) { + // Needs to expand. + list_node_t* new_array = realloc( + heap->node_array, + sizeof(list_node_base_t) * heap->capacity * 2 + ); + if (new_array == NULL) { + fprintf( + stderr, + "Error: failed to allocate more memory for internal heap.\n" + ); + return false; + } + heap->node_array = new_array; + heap->capacity *= 2; + } + + heap_set_idx(heap, heap->size, node); + node->is_free = true; + heap->size += 1; + heap_heapify_up(heap, node->idx); + + return true; +} + +/* +Heapify downward. This happens when we replace a node by a smaller number. +*/ +static void heap_heapify_down(heap_t* heap, const size_t idx) { + const size_t left_idx = HEAP_GET_LEFT_IDX(idx); + const size_t right_idx = HEAP_GET_RIGHT_IDX(idx); + size_t largest_idx = idx; + if (left_idx < heap->size) { + largest_idx = heap_compare_GT(heap, largest_idx, left_idx); + } + if (right_idx < heap->size) { + largest_idx = heap_compare_GT(heap, largest_idx, right_idx); + } + if (largest_idx != idx) { + // Swap. Heapify needs to propagate downward. + heap_swap(heap, idx, largest_idx); + heap_heapify_down(heap, largest_idx); + } +} + +/* +Heapify upward. This happens when we replace a node by a larger number. +*/ +static void heap_heapify_up(heap_t* heap, const size_t idx) { + if (idx <= HEAP_IDX_ROOT) return; + + const size_t parent = HEAP_GET_PARENT_IDX(idx); + if (heap_compare_GT(heap, idx, parent) != parent) { + heap_swap(heap, idx, parent); + heap_heapify_up(heap, parent); + } +} + +list_node_t heap_allocate(heap_t* heap, const size_t size) { + list_node_t root = heap_get_root(heap); + if (!root) { + return NULL; + } + + if (size > root->size) { + // The required size is larger than the largest block size. + return NULL; + } + + // First, try to find the smallest node on the top of the heap >= size + list_node_t victim = root; + const size_t heap_size = heap->size; + const size_t left_idx = HEAP_GET_LEFT_IDX(HEAP_IDX_ROOT); + const size_t right_idx = HEAP_GET_RIGHT_IDX(HEAP_IDX_ROOT); + list_node_t left = left_idx < heap_size ? + heap->node_array[left_idx] : NULL; + list_node_t right = right_idx < heap_size ? + heap->node_array[right_idx] : NULL; + + if (left && left->size >= size) { + victim = left; + } + if (right && right->size >= size && right->size < victim->size) { + victim = right; + } + + if (victim->size == size) { + // Perfect match! + heap_remove_idx(heap, victim->idx); + victim->is_free = false; + return victim; + } + + // The block is larger than what's being asked. Need to split... + list_node_t ret_node = list_create_node( + victim->prev, + victim, + victim->addr, + size, + HEAP_IDX_NULL, + false + ); // Notice that this is the only place that we create new node. + if (!ret_node) { + // Running out of memory to allocate node... + // But we can still return the allocated block + heap_remove_idx(heap, victim->idx); + victim->is_free = false; + return victim; + } + + victim->prev->next = ret_node; + victim->prev = ret_node; + victim->addr += size; + victim->size -= size; + heap_heapify_down(heap, victim->idx); + return ret_node; +} + +bool heap_free(heap_t* heap, list_node_t node) { + // The idea is that we want to merge internal free blocks, + // and try to avoid insert new node into the heap. + assert(!node->is_free && "The current node must not be free already"); + if (node->next->is_free) { + // The next block is free. + if (node->prev->is_free) { + // First check if the previous block is also free. + // Merge and remove the previous is true. + list_node_t previous = node->prev; + node->size += previous->size; + node->addr = previous->addr; + assert(heap->node_array[previous->idx] == previous); + heap_remove_idx(heap, previous->idx); + node_unlink(previous); + free(previous); + } + + // Merge the current into next. + list_node_t next = node->next; + next->size += node->size; + next->addr = node->addr; + node_unlink(node); + free(node); + // Restructure the heap as we increase the size of "next" node. + heap_heapify_up(heap, next->idx); + return true; + + } else if (node->prev->is_free) { + // Merge the current block into the previous. + list_node_t previous = node->prev; + previous->size += node->size; + node_unlink(node); + free(node); + + // Restructure the heap as we increase the size of "next" node. + heap_heapify_up(heap, previous->idx); + return true; + + } else { + // Nothing we can merge. Insert the current node into the heap. + bool ret_val = heap_insert(heap, node); + return ret_val; + } +} + +bool heap_init(heap_t* heap, uint8_t* addr, const size_t size) { + list_node_t* array = + (list_node_t*) malloc(sizeof(list_node_t) * INIT_HEAP_CAPACITY); + if (!array) { + fprintf(stderr, "Error: failed to allocate memory for the heap.\n"); + return false; + } + + array[0] = NULL; + + *heap = (heap_t) { + .capacity = INIT_HEAP_CAPACITY, + .size = 2, + .node_array = array + }; + + if (!list_init(&(heap->node_list), addr, size)) { + free(array); + return false; + } + + list_node_t head = heap->node_list.virtual_head->next; + heap_set_idx(heap, 1, head); + + return true; +} \ No newline at end of file diff --git a/source/mmap_malloc/heap.h b/source/mmap_malloc/heap.h new file mode 100644 index 00000000..35e8f432 --- /dev/null +++ b/source/mmap_malloc/heap.h @@ -0,0 +1,58 @@ +#ifndef HEAP_H +#define HEAP_H + +#include +#include + +#include "constants.h" +#include "list.h" + +#define INIT_HEAP_CAPACITY 8 + +#define HEAP_GET_LEFT_IDX(idx) (idx * 2) +#define HEAP_GET_RIGHT_IDX(idx) (idx * 2 + 1) +#define HEAP_GET_PARENT_IDX(idx) (idx / 2) + +#define HEAP_IDX_NULL 0 +#define HEAP_IDX_ROOT 1 + +typedef struct heap { + size_t capacity; + size_t size; + list_t node_list; + list_node_t* node_array; +} heap_t; + +list_node_t heap_allocate(heap_t* heap, const size_t size); +bool heap_free(heap_t* heap, list_node_t node); +bool heap_init(heap_t* heap, uint8_t* addr, const size_t size); + +void FORCE_INLINE +heap_check(heap_t* heap) { +#if DEBUG_HEAP + list_t* list = &heap->node_list; + assert(!list->virtual_head->is_free && "Virtual head is dirty."); + assert(!list->virtual_tail->is_free && "Virtual tail is dirty."); + list_node_t curr = list->virtual_head->next; + list_node_t end = list->virtual_tail; + while (curr != end) { + assert(curr); + if (curr->is_free) { + if (!(curr->idx > 0 && curr->idx < heap->size)) { + fprintf(stderr, "Idx: %ld; Addr: %p\n", curr->idx, curr); + assert(0 && "Out of bound!"); + } + } else { + assert(curr->idx == 0 && "Invalid null heap idx."); + } + curr = curr->next; + } + for (int i = 1; i < heap->size; ++i) { + assert(heap->node_array[i] && "Node array being NULL."); + assert(heap->node_array[i]->is_free && "Heap not clean."); + assert(heap->node_array[i]->idx == i && "Idx mismatch."); + } +#endif +} + +#endif \ No newline at end of file diff --git a/source/mmap_malloc/list.h b/source/mmap_malloc/list.h new file mode 100644 index 00000000..9d9d30ac --- /dev/null +++ b/source/mmap_malloc/list.h @@ -0,0 +1,120 @@ +#ifndef LIST_H +#define LIST_H + +#include +#include +#include + +#include "constants.h" +#include + +typedef struct list_node_base { + struct list_node_base* prev; + struct list_node_base* next; + uint8_t* addr; // The block actual address + size_t size; // Size of the block + size_t idx; + bool is_free; +} list_node_base_t; + +typedef list_node_base_t* list_node_t; + +typedef struct list { + list_node_t virtual_head; + list_node_t virtual_tail; +} list_t; + +list_node_t FORCE_INLINE +list_create_node( + list_node_t prev, + list_node_t next, + uint8_t* addr, + const size_t size, + const size_t idx, + const bool is_free +) { + list_node_t new_node = (list_node_t) malloc(sizeof(list_node_base_t)); + if (!new_node) { + return NULL; + } + *new_node = (list_node_base_t) { + .prev = prev, + .next = next, + .addr = addr, + .size = size, + .idx = idx, + .is_free = is_free + }; + return new_node; +} + +bool FORCE_INLINE +list_init(list_t* list, uint8_t* addr, const size_t size) { + list_node_t virtual_head = (list_node_t) malloc(sizeof(list_node_base_t)); + list_node_t init_node = (list_node_t) malloc(sizeof(list_node_base_t)); + list_node_t virtual_tail = (list_node_t) malloc(sizeof(list_node_base_t)); + if (!virtual_head || !init_node || !virtual_tail) { + fprintf(stderr, "Error: failed to allocate memory for the node list.\n"); + return false; + } + + *virtual_head = (list_node_base_t) { + .prev = NULL, + .next = init_node, + .addr = NULL, + .size = 0, + .idx = 0, + .is_free = false + }; + + *init_node = (list_node_base_t) { + .prev = virtual_head, + .next = virtual_tail, + .addr = addr, + .size = size, + .idx = 0, + .is_free = true + }; + + *virtual_tail = (list_node_base_t) { + .prev = init_node, + .next = NULL, + .addr = NULL, + .size = 0, + .idx = 0, + .is_free = false + }; + + *list = (list_t) { + .virtual_head = virtual_head, + .virtual_tail = virtual_tail + }; + + return true; +} + +void FORCE_INLINE +node_unlink(list_node_t node) { + assert(node); + assert(node->next != node->prev->next && "WTF???"); + node->prev->next = node->next; + node->next->prev = node->prev; + node->prev = node->next = NULL; +} + +list_node_t FORCE_INLINE +list_find_in_use(list_t* list, uint8_t* addr) { + list_node_t curr = list->virtual_head->next; + list_node_t end = list->virtual_tail; + while (curr != end) { + assert(curr); + if (addr == curr->addr) { + assert(!curr->is_free && "Double free detected..."); + return curr; + } + curr = curr->next; + } + return NULL; +} + +#endif diff --git a/source/mmap_malloc/mmap_allocator.c b/source/mmap_malloc/mmap_allocator.c new file mode 100644 index 00000000..b8e7085e --- /dev/null +++ b/source/mmap_malloc/mmap_allocator.c @@ -0,0 +1,446 @@ +#define _GNU_SOURCE // To enable various non-standard GNU extensions. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "constants.h" +#include "default_config.h" +#include "heap.h" +#include "list.h" +#include "mmap_mgr.h" +#include +#include "profiling.h" + +/*---------------------------------------------------------------------------*/ +// Out of memory. +#define OUT_OF_MEMORY() do { \ + fprintf(stderr, "Out of space. Failed to allocate more memory.\n"); \ + errno = ENOMEM; \ +} while (0) + +/*---------------------------------------------------------------------------*/ +// Allocator status +enum allocator_status { + FAILED_TO_LOAD = -1, + NOT_LOADED = 0, + LOADED = 1 +}; +static enum allocator_status allocator_status = NOT_LOADED; + +/*---------------------------------------------------------------------------*/ +// Global lock. +static pthread_mutex_t glock = PTHREAD_MUTEX_INITIALIZER; + +#define GLOBAL_LOCK_ACQUIRE() do {\ + pthread_mutex_lock(&glock);\ +} while (0) + +#define GLOBAL_LOCK_RELEASE() do {\ + pthread_mutex_unlock(&glock);\ +} while (0) + +/*---------------------------------------------------------------------------*/ +// Parameters + +// Fixed +static size_t page_size = 0; +static void* mmap_region_base = NULL; +static void* mmap_region_end = NULL; + +// Configurable +static size_t mmap_heap_size = 0; +static size_t mmap_alloctor_min_bsize = 0; +static char naming_template[MAX_NAMING_TEMPLATE_SIZE + 1] = {0}; + +#define CEILING_PAGE_SIZE(size) (((size) + (page_size - 1)) & ~(page_size - 1)) + +/*---------------------------------------------------------------------------*/ +// Heap +heap_t mmap_heap; + +// Wrapper for heap check. +// Notice that heap check should acquire the global heap lock. +// Be aware of dead lock. Don't die! +#if DEBUG_HEAP +#define HEAP_CHECK() do {\ + GLOBAL_LOCK_ACQUIRE();\ + heap_check(&mmap_heap);\ + GLOBAL_LOCK_RELEASE();\ +} while (0) +#else +#define HEAP_CHECK() do {} while (0) +#endif + +bool LOCAL_HELPER +config_parameters() { + assert(page_size && "Page size should be initialized."); + + const char* env_template = getenv(env_naming_template); + if (env_template) { + if (strlen(env_template) > MAX_NAMING_TEMPLATE_SIZE) { + fprintf(stderr, "Config error: %s is too long. ", env_naming_template); + fprintf(stderr, "Max acceptable size: %d\n", MAX_NAMING_TEMPLATE_SIZE); + return false; + } + strcpy(naming_template, env_template); + } else { + strcpy(naming_template, default_naming_template); + } + + const char* evn_size = getenv(env_mmap_heap_size); + if (evn_size) { + mmap_heap_size = (size_t) strtoull(evn_size, NULL, 10); + if (mmap_heap_size % page_size != 0) { + fprintf( + stderr, + "Config error: %s is not page aligned. Page size: %ld\n", + env_mmap_heap_size, + page_size + ); + return false; + } + } else { + mmap_heap_size = default_mmap_heap_size; + } + + const char* evn_bsize = getenv(env_mmap_alloctor_min_bsize); + if (evn_bsize) { + mmap_alloctor_min_bsize = (size_t) strtoull(evn_bsize, NULL, 10); + if (mmap_alloctor_min_bsize > mmap_heap_size) { + fprintf( + stderr, + "Config error: %s is not larger than the heap size.\n", + env_mmap_alloctor_min_bsize + ); + return false; + } + } else { + mmap_alloctor_min_bsize = default_mmap_alloctor_min_bsize; + } + + const char* env_profile = getenv(env_profile_file_path); + if (env_profile) { + profile_file = fopen(env_profile, "a"); + if (!profile_file) { + fprintf( + stderr, + "Failed tp open the profile file in path: %s\n", + env_profile + ); + return false; + } + } + + const char* env_profile_freq = getenv(env_profile_frequency); + if (env_profile_freq) { + profile_frequency = (size_t) strtoull(env_profile_freq, NULL, 10); + if (profile_frequency == 0) { + fprintf( + stderr, + "Config error: %s cannot be zero.\n", + env_mmap_alloctor_min_bsize + ); + return false; + } + } + + return true; +} + +/*---------------------------------------------------------------------------*/ +// Allocator API from mmap_heap + +LOCAL_HELPER void mmap_allocator_init() { + GLOBAL_LOCK_ACQUIRE(); + if (allocator_status != NOT_LOADED) { + GLOBAL_LOCK_RELEASE(); + return; + } + + // Initialize page size. + page_size = sysconf(_SC_PAGE_SIZE); + + // Read environment variables + if (!config_parameters()) { + fprintf(stderr, "Failed to configure parameters from env.\n"); + allocator_status = FAILED_TO_LOAD; + GLOBAL_LOCK_RELEASE(); + return; + } + + // Configure profiling + if (profile_file) { + pthread_t profile_thread; + if (pthread_create(&profile_thread, NULL, profile_thread_entry, NULL)) { + fprintf(stderr, "Profiling thread init failed.\n"); + allocator_status = FAILED_TO_LOAD; + GLOBAL_LOCK_RELEASE(); + return; + } + } + + // Reserve mmap region. + mmap_region_base = mmap_reserve(mmap_heap_size); + if (!mmap_region_base) { + // Failed to reserve the mmap region. + fprintf(stderr, "Failed to reserve mmap alloctor region.\n"); + allocator_status = FAILED_TO_LOAD; + GLOBAL_LOCK_RELEASE(); + return; + } + mmap_region_end = mmap_region_base + mmap_heap_size - 1; + + // Initialize mmap heap. + if (!heap_init(&mmap_heap, mmap_region_base, mmap_heap_size)) { + allocator_status = FAILED_TO_LOAD; + GLOBAL_LOCK_RELEASE(); + return; + } + + fprintf(stderr, "MMap Allocator is successfully loaded.\n"); + allocator_status = LOADED; + GLOBAL_LOCK_RELEASE(); +} + +LOCAL_HELPER void* mmap_allocate(size_t size) { + if (size == 0) { + // Ideally will never happen. + return NULL; + } + + // Round to ensure page aligned + size = CEILING_PAGE_SIZE(size); + + GLOBAL_LOCK_ACQUIRE(); + list_node_t block = heap_allocate(&mmap_heap, size); + GLOBAL_LOCK_RELEASE(); + + if (!block) { + return NULL; + } + + assert(block->size == size); + const int retval = mmap_maptemp(block->addr, block->size, naming_template); + if (0 != retval) { + // Failed to allocate the swap file. + // We will return this node back to theheap. + GLOBAL_LOCK_ACQUIRE(); + heap_free(&mmap_heap, block); + GLOBAL_LOCK_RELEASE(); + fprintf(stderr, "Failed to create mmap region. Error code: %d\n", retval); + return NULL; + } + + return (void*) block->addr; +} + +LOCAL_HELPER bool mmap_release(void* addr) { + GLOBAL_LOCK_ACQUIRE(); + list_node_t block = list_find_in_use(&mmap_heap.node_list, addr); + if (!block) { + fprintf(stderr, "Failed to find the block with the given addr: %p\n", addr); + GLOBAL_LOCK_RELEASE(); + return false; + } + + // Unmap the region. + const int retval = mmap_unmap(addr, block->size); + if (0 != retval) { + fprintf(stderr, "Failed to unmap the region.\n"); + GLOBAL_LOCK_RELEASE(); + return false; + } + + if (!heap_free(&mmap_heap, block)) { + fprintf( + stderr, + "Failed to free as stdlib cannot allocate more memory for the heap.\n" + ); + GLOBAL_LOCK_RELEASE(); + return false; + } + + GLOBAL_LOCK_RELEASE(); + return true; +} + +LOCAL_HELPER bool mmap_release_with_copy( + list_node_t block, + void* dest, + const size_t new_size +) { + void* addr = block->addr; + // We will need to copy the data from the src to the new dest. + const size_t copy_size = (block->size < new_size) ? block->size : new_size; + memcpy(dest, addr, copy_size); + + // Unmap the region. + if (mmap_unmap(addr, block->size)) { + fprintf(stderr, "Failed to unmap the region.\n"); + GLOBAL_LOCK_RELEASE(); + return false; + } + + GLOBAL_LOCK_ACQUIRE(); + bool success = heap_free(&mmap_heap, block); + GLOBAL_LOCK_RELEASE(); + + if (!success) { + fprintf( + stderr, + "Failed to free as stdlib cannot allocate more memory for the heap.\n" + ); + return false; + } + return true; +} + +/*---------------------------------------------------------------------------*/ +// User library interface +void* mmap_malloc(size_t size); +void* mmap_calloc(size_t num_elements, size_t element_size); +void* mmap_reallocarray(void *addr, size_t size, size_t count); +void* mmap_realloc(void *addr, size_t size); + +/*---------------------------------------------------------------------------*/ +// malloc implementation +void* mmap_malloc(size_t size) { + if (allocator_status == NOT_LOADED) { + mmap_allocator_init(); + } + + if (allocator_status != LOADED || size < mmap_alloctor_min_bsize) { + return malloc(size); + } + + void* ret = mmap_allocate(size); + if (!ret) { + OUT_OF_MEMORY(); + } + HEAP_CHECK(); + return ret; +} + +/*---------------------------------------------------------------------------*/ +// calloc implementation +void* mmap_calloc(size_t num_elements, size_t element_size) { + if (allocator_status == NOT_LOADED) { + mmap_allocator_init(); + } + + const size_t total_size = num_elements * element_size; + if (allocator_status != LOADED || total_size < mmap_alloctor_min_bsize) { + return calloc(num_elements, element_size); + } + + void* ret = mmap_allocate(total_size); + if (!ret) { + OUT_OF_MEMORY(); + } + // As required of calloc, memory region must be initialized to 0. + memset(ret, 0, total_size); + return ret; +} + +/*---------------------------------------------------------------------------*/ +// reallocarray implementation +void* mmap_reallocarray(void *addr, size_t size, size_t count) { + return mmap_realloc(addr, size * count); +} + +/*---------------------------------------------------------------------------*/ +// realloc implementation +void* mmap_realloc(void *addr, size_t size) { + if (!addr) return mmap_malloc(size); // It should behave like malloc. + + if (allocator_status == NOT_LOADED) { + mmap_allocator_init(); + } + + if (allocator_status != LOADED || addr < mmap_region_base || addr > mmap_region_end) { + return realloc(addr, size); + } + + // mmap allocator is properly initialized. + if (addr >= mmap_region_base) { + + GLOBAL_LOCK_ACQUIRE(); + list_node_t block = list_find_in_use(&mmap_heap.node_list, addr); + GLOBAL_LOCK_RELEASE(); + if (!block) { + fprintf(stderr, + "Realloc failed: cannot find the given address: %p.\n", addr); + return NULL; + } + + if (block->size >= size) return addr; // Already large enough. + + // const size_t alloc_size = CEILING_PAGE_SIZE(size); + void* new_region = mmap_allocate(size); + if (!new_region) { + OUT_OF_MEMORY(); + return NULL; + } + + if (!mmap_release_with_copy(block, new_region, size)) { + fprintf(stderr, "Realloc copy failed.\n"); + mmap_release(new_region); + return NULL; + } + + HEAP_CHECK(); + return new_region; + } + + // The old address is allcoated by the std heap. We don't know the size. + void* realloc_buffer = realloc(addr, size); + if (!realloc_buffer) { + fprintf(stderr, "Failed to reallocate a buffer using std realloc.\n"); + return NULL; + } + + if (size < mmap_alloctor_min_bsize) { + return realloc_buffer; + } + + // const size_t alloc_size = CEILING_PAGE_SIZE(size); + void* new_region = mmap_allocate(size); + if (!new_region) { + OUT_OF_MEMORY(); + free(realloc_buffer); + return NULL; + } + + memcpy(new_region, realloc_buffer, size); + free(realloc_buffer); + + HEAP_CHECK(); + return new_region; +} + +/*---------------------------------------------------------------------------*/ +// free implementation +void mmap_free(void* addr) { + if (!addr) return; + + if (allocator_status == NOT_LOADED) { + mmap_allocator_init(); + } + + if (allocator_status != LOADED || addr < mmap_region_base || addr > mmap_region_end) { + /* On macOS, `addr` can be larger than `mmap_region_end` */ + free(addr); + return; + } + + if (!mmap_release(addr)) { + fprintf(stderr, "Failed to free at addr: %p\n", addr); + } + HEAP_CHECK(); +} \ No newline at end of file diff --git a/source/mmap_malloc/mmap_mgr.c b/source/mmap_malloc/mmap_mgr.c new file mode 100644 index 00000000..245960ed --- /dev/null +++ b/source/mmap_malloc/mmap_mgr.c @@ -0,0 +1,91 @@ +#define _GNU_SOURCE // To enable various non-standard GNU extensions. +#include +#include +#include +#include +#include +#include +#include + +#include "constants.h" +#include "mmap_mgr.h" +#include "profiling.h" +#include + +/* +This function will reserve a region for later mmap use. +It does not map to an existing file. +*/ +void* mmap_reserve(const size_t size) { + void* addr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (addr == MAP_FAILED) { + return NULL; + } + return addr; +} + +int mmap_maptemp(void* addr, const size_t size, char* template) { + char* filename = (char*) malloc(strlen(template) + 1); + if (!filename) { + return -1; + } + strcpy(filename, template); + int fd = mkstemp(filename); + if (fd < 0) { + free(filename); + return -2; + } + + // After unlinking the file it will not appear in the file system, + // however, since we have the fd, it can still be written. + int retval; + retval = unlink(filename); + free(filename); + if (retval) { + return -3; + } + // Resize the file to ensure we have enough space. + retval = ftruncate(fd, size); + if (retval) { + fprintf(stderr, "ftruncate failed! %s. Size: %ld\n", strerror(errno), size); + return -4; + } + + void* ret_addr = mmap( + addr, // Map to this address + size, // With the given size. + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, + fd, + 0 + ); + if (ret_addr == MAP_FAILED) { + return -5; + } + + profile_allocate(size); + + retval = close(fd); + if (retval) { + return -6; + } + + return 0; +} + +int mmap_unmap(void* addr, const size_t size) { + void* ret_addr = mmap( + addr, // Previous mapped address + size, // With the given size. + PROT_NONE, + MAP_ANONYMOUS | MAP_FIXED | MAP_PRIVATE, + -1, // Reserved + 0 + ); + if (ret_addr == MAP_FAILED) { + return -1; + } + + profile_deallocate(size); + return 0; +} \ No newline at end of file diff --git a/source/mmap_malloc/mmap_mgr.h b/source/mmap_malloc/mmap_mgr.h new file mode 100644 index 00000000..3e22bf47 --- /dev/null +++ b/source/mmap_malloc/mmap_mgr.h @@ -0,0 +1,10 @@ +#ifndef MMAP_MGR_H +#define MMAP_MGR_H + +#include + +void* mmap_reserve(const size_t size); +int mmap_maptemp(void* addr, const size_t size, char* template); +int mmap_unmap(void* addr, const size_t size); + +#endif \ No newline at end of file diff --git a/source/mmap_malloc/profiling.c b/source/mmap_malloc/profiling.c new file mode 100644 index 00000000..567ae12b --- /dev/null +++ b/source/mmap_malloc/profiling.c @@ -0,0 +1,36 @@ +#include +#include +#include + +#include "profiling.h" + +pthread_mutex_t profile_lock = PTHREAD_MUTEX_INITIALIZER; +FILE* profile_file = NULL; +size_t profile_frequency = 30; + +size_t num_mmap_file = 0; +size_t mmap_heap_total_size = 0; + +void* profile_thread_entry(void* arg) { + (void) arg; + assert(profile_file); + + while (1) { + PROFILE_LOCK_ACQUIRE(); + time_t curr = time(NULL); + size_t curr_heap_size = mmap_heap_total_size / (1024 * 1024); // In MBytes + size_t curr_num_block = num_mmap_file; + PROFILE_LOCK_RELEASE(); + + fprintf(profile_file, " %ld\n", curr); + fprintf(profile_file, "<# mmap blocks> %ld\n", curr_num_block); + fprintf( + profile_file, + " %ldMBytes\n\n", + curr_heap_size + ); + fflush(profile_file); + + sleep(profile_frequency); + } +} diff --git a/source/mmap_malloc/profiling.h b/source/mmap_malloc/profiling.h new file mode 100644 index 00000000..8e62da2b --- /dev/null +++ b/source/mmap_malloc/profiling.h @@ -0,0 +1,47 @@ +#ifndef PROFILING_H +#define PROFILING_H + +#include +#include +#include +#include + +#include "constants.h" +#include "default_config.h" + +extern pthread_mutex_t profile_lock; +extern FILE* profile_file; +extern size_t profile_frequency; + +extern size_t num_mmap_file; +extern size_t mmap_heap_total_size; + +#define PROFILE_LOCK_ACQUIRE() do {\ + pthread_mutex_lock(&profile_lock);\ +} while (0) + +#define PROFILE_LOCK_RELEASE() do {\ + pthread_mutex_unlock(&profile_lock);\ +} while (0) + +void FORCE_INLINE +profile_allocate(const size_t size) { + if (!profile_file) return; + PROFILE_LOCK_ACQUIRE(); + ++num_mmap_file; + mmap_heap_total_size += size; + PROFILE_LOCK_RELEASE(); +} + +void FORCE_INLINE +profile_deallocate(const size_t size) { + if (!profile_file) return; + PROFILE_LOCK_ACQUIRE(); + --num_mmap_file; + mmap_heap_total_size -= size; + PROFILE_LOCK_RELEASE(); +} + +void* profile_thread_entry(void* arg); + +#endif \ No newline at end of file From e6c4e487c95a52d3886e2076d6f571c2fd5b1004 Mon Sep 17 00:00:00 2001 From: Yuuki Galaxy <863605+galaxy001@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:56:37 +0800 Subject: [PATCH 4/5] vector> countCellGeneUMI modified: source/SoloFeature.h modified: source/SoloFeature_emptyDrops_CR.cpp --- source/SoloFeature.h | 3 ++- source/SoloFeature_emptyDrops_CR.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/SoloFeature.h b/source/SoloFeature.h index 185deeaf..31454782 100755 --- a/source/SoloFeature.h +++ b/source/SoloFeature.h @@ -4,6 +4,7 @@ #include #include #include +#include "mmap_allocator.hpp" #include "IncludeDefine.h" #include "ReadAlignChunk.h" @@ -53,7 +54,7 @@ class SoloFeature { vector nUMIperCBmulti; - vector countCellGeneUMI;//sparsified matrix for the counts, each entry is: geneID count1 count2 ... countNcounts + vector> countCellGeneUMI;//sparsified matrix for the counts, each entry is: geneID count1 count2 ... countNcounts vector countCellGeneUMIindex;//index of CBs in the count matrix uint32 countMatStride; //number of counts per entry in the count matrix diff --git a/source/SoloFeature_emptyDrops_CR.cpp b/source/SoloFeature_emptyDrops_CR.cpp index 9c17a976..52f18861 100644 --- a/source/SoloFeature_emptyDrops_CR.cpp +++ b/source/SoloFeature_emptyDrops_CR.cpp @@ -6,7 +6,7 @@ #include #include -double logMultinomialPDFsparse(const vector &ambProfileLogP, const vector &countCellGeneUMI, const uint32 stride, const uint32 shift, const int64 start, const uint32 nGenes, const vector &logFactorial); +double logMultinomialPDFsparse(const vector &ambProfileLogP, const vector> &countCellGeneUMI, const uint32 stride, const uint32 shift, const int64 start, const uint32 nGenes, const vector &logFactorial); void SoloFeature::emptyDrops_CR() { if (nCB<=pSolo.cellFilter.eDcr.indMin) { @@ -216,7 +216,7 @@ void SoloFeature::emptyDrops_CR() return; }; -double logMultinomialPDFsparse(const vector &ambProfileLogP, const vector &countCellGeneUMI, const uint32 stride, const uint32 shift, const int64 start, const uint32 nGenes, const vector &logFactorial) +double logMultinomialPDFsparse(const vector &ambProfileLogP, const vector> &countCellGeneUMI, const uint32 stride, const uint32 shift, const int64 start, const uint32 nGenes, const vector &logFactorial) { uint32 sumCount=0; double sumLogFac=0.0, sumCountLogP=0.0; From b386d9d12b41383bece04cb1ab70665a62b00d9f Mon Sep 17 00:00:00 2001 From: Yuuki Galaxy <863605+galaxy001@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:46:17 +0800 Subject: [PATCH 5/5] modified: mmap_malloc/mmap_allocator.c --- source/mmap_malloc/mmap_allocator.c | 61 ++++++++++++++++------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/source/mmap_malloc/mmap_allocator.c b/source/mmap_malloc/mmap_allocator.c index b8e7085e..60bfac35 100644 --- a/source/mmap_malloc/mmap_allocator.c +++ b/source/mmap_malloc/mmap_allocator.c @@ -17,6 +17,21 @@ #include #include "profiling.h" +#if defined(__APPLE__) +#include +#if TARGET_OS_OSX +#include +#define MALLOCSIZE malloc_size +#else +#error "Unsupported Apple platform" +#endif +#elif __linux__ +#include +#define MALLOCSIZE malloc_usable_size +#else +#error "Unsupported compiler" +#endif + /*---------------------------------------------------------------------------*/ // Out of memory. #define OUT_OF_MEMORY() do { \ @@ -364,11 +379,26 @@ void* mmap_realloc(void *addr, size_t size) { } if (allocator_status != LOADED || addr < mmap_region_base || addr > mmap_region_end) { - return realloc(addr, size); + if (size < mmap_alloctor_min_bsize) { + return realloc(addr, size); + } else { + size_t old_size = MALLOCSIZE(addr); + fprintf(stderr, "Realloc Heap[%zu] to Mmap[%zu].\n",old_size,size); + if (old_size > size) old_size = size; + void* new_region = mmap_allocate(size); + if (!new_region) { + OUT_OF_MEMORY(); + return NULL; + } + memcpy(new_region, addr, old_size); + free(addr); + HEAP_CHECK(); + return new_region; + } } // mmap allocator is properly initialized. - if (addr >= mmap_region_base) { + if (addr >= mmap_region_base && addr <= mmap_region_end) { GLOBAL_LOCK_ACQUIRE(); list_node_t block = list_find_in_use(&mmap_heap.node_list, addr); @@ -397,31 +427,8 @@ void* mmap_realloc(void *addr, size_t size) { HEAP_CHECK(); return new_region; } - - // The old address is allcoated by the std heap. We don't know the size. - void* realloc_buffer = realloc(addr, size); - if (!realloc_buffer) { - fprintf(stderr, "Failed to reallocate a buffer using std realloc.\n"); - return NULL; - } - - if (size < mmap_alloctor_min_bsize) { - return realloc_buffer; - } - - // const size_t alloc_size = CEILING_PAGE_SIZE(size); - void* new_region = mmap_allocate(size); - if (!new_region) { - OUT_OF_MEMORY(); - free(realloc_buffer); - return NULL; - } - - memcpy(new_region, realloc_buffer, size); - free(realloc_buffer); - - HEAP_CHECK(); - return new_region; + fprintf(stderr, "Failed in mmap_relloc.\n"); + return NULL; } /*---------------------------------------------------------------------------*/