Reduce the overhead from malloc usable

Author: sundb

deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h

@@ -70,6 +70,6 @@ void jemalloc_prefork(void);
 void jemalloc_postfork_parent(void);
 void jemalloc_postfork_child(void);
 void je_sdallocx_noflags(void *ptr, size_t size);
-void *malloc_default(size_t size);
+void *malloc_default(size_t size, size_t *usable_sz);
 
 #endif /* JEMALLOC_INTERNAL_EXTERNS_H */

deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -255,15 +255,15 @@ malloc_initialized(void) {
  * tail-call to the slowpath if they fire.
  */
 JEMALLOC_ALWAYS_INLINE void *
-imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
+imalloc_fastpath(size_t size, size_t *usable_sz, void *(fallback_alloc)(size_t, size_t *)) {
 	LOG("core.malloc.entry", "size: %zu", size);
 	if (tsd_get_allocates() && unlikely(!malloc_initialized())) {
-		return fallback_alloc(size);
+		return fallback_alloc(size, usable_sz);
 	}
 
 	tsd_t *tsd = tsd_get(false);
 	if (unlikely((size > SC_LOOKUP_MAXCLASS) || tsd == NULL)) {
-		return fallback_alloc(size);
+		return fallback_alloc(size, usable_sz);
 	}
 	/*
 	 * The code below till the branch checking the next_event threshold may
@@ -307,7 +307,7 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
 	 * 0) in a single branch.
 	 */
 	if (unlikely(allocated_after >= threshold)) {
-		return fallback_alloc(size);
+		return fallback_alloc(size, usable_sz);
 	}
 	assert(tsd_fast(tsd));
 
@@ -316,6 +316,7 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
 	cache_bin_t *bin = &tcache->bins[ind];
 	bool tcache_success;
 	void *ret;
+	if (usable_sz) *usable_sz = usize;
 
 	/*
 	 * We split up the code this way so that redundant low-water
@@ -334,7 +335,7 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
 		return ret;
 	}
 
-	return fallback_alloc(size);
+	return fallback_alloc(size, usable_sz);
 }
 
 JEMALLOC_ALWAYS_INLINE int

deps/jemalloc/src/jemalloc.c

@@ -2697,7 +2697,7 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) {
 
 JEMALLOC_NOINLINE
 void *
-malloc_default(size_t size) {
+malloc_default(size_t size, size_t *usize) {
 	void *ret;
 	static_opts_t sopts;
 	dynamic_opts_t dopts;
@@ -2731,6 +2731,7 @@ malloc_default(size_t size) {
 
 	LOG("core.malloc.exit", "result: %p", ret);
 
+	if (usize) *usize = dopts.usize;
 	return ret;
 }
 
@@ -2743,9 +2744,11 @@ JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
 void JEMALLOC_NOTHROW *
 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
 je_malloc(size_t size) {
-	return imalloc_fastpath(size, &malloc_default);
+	return imalloc_fastpath(size, NULL, &malloc_default);
 }
 
+
+
 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
 JEMALLOC_ATTR(nonnull(1))
 je_posix_memalign(void **memptr, size_t alignment, size_t size) {
@@ -2861,7 +2864,7 @@ je_calloc(size_t num, size_t size) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
+ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path, size_t *usable) {
 	if (!slow_path) {
 		tsd_assert_fast(tsd);
 	}
@@ -2894,6 +2897,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
 		    true);
 	}
 	thread_dalloc_event(tsd, usize);
+	if (usable) *usable = usize;
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -2993,7 +2997,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
 
 JEMALLOC_NOINLINE
 void
-free_default(void *ptr) {
+free_default(void *ptr, size_t *usable) {
 	UTRACE(ptr, 0, 0);
 	if (likely(ptr != NULL)) {
 		/*
@@ -3011,14 +3015,14 @@ free_default(void *ptr) {
 			tcache_t *tcache = tcache_get_from_ind(tsd,
 			    TCACHE_IND_AUTOMATIC, /* slow */ false,
 			    /* is_alloc */ false);
-			ifree(tsd, ptr, tcache, /* slow */ false);
+			ifree(tsd, ptr, tcache, /* slow */ false, usable);
 		} else {
 			tcache_t *tcache = tcache_get_from_ind(tsd,
 			    TCACHE_IND_AUTOMATIC, /* slow */ true,
 			    /* is_alloc */ false);
 			uintptr_t args_raw[3] = {(uintptr_t)ptr};
 			hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw);
-			ifree(tsd, ptr, tcache, /* slow */ true);
+			ifree(tsd, ptr, tcache, /* slow */ true, usable);
 		}
 
 		check_entry_exit_locking(tsd_tsdn(tsd));
@@ -3062,7 +3066,7 @@ free_fastpath_nonfast_aligned(void *ptr, bool check_prof) {
 
 /* Returns whether or not the free attempt was successful. */
 JEMALLOC_ALWAYS_INLINE
-bool free_fastpath(void *ptr, size_t size, bool size_hint) {
+bool free_fastpath(void *ptr, size_t size, bool size_hint, size_t *usable) {
 	tsd_t *tsd = tsd_get(false);
 	/* The branch gets optimized away unless tsd_get_allocates(). */
 	if (unlikely(tsd == NULL)) {
@@ -3116,6 +3120,7 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
 	te_free_fastpath_ctx(tsd, &deallocated, &threshold);
 
 	size_t usize = sz_index2size(alloc_ctx.szind);
+	if (usable) *usable = usize;
 	uint64_t deallocated_after = deallocated + usize;
 	/*
 	 * Check for events and tsd non-nominal (fast_threshold will be set to
@@ -3158,8 +3163,8 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW
 je_free(void *ptr) {
 	LOG("core.free.entry", "ptr: %p", ptr);
 
-	if (!free_fastpath(ptr, 0, false)) {
-		free_default(ptr);
+	if (!free_fastpath(ptr, 0, false, NULL)) {
+		free_default(ptr, NULL);
 	}
 
 	LOG("core.free.exit", "");
@@ -3601,7 +3606,7 @@ do_realloc_nonnull_zero(void *ptr) {
 		    /* is_alloc */ false);
 		uintptr_t args[3] = {(uintptr_t)ptr, 0};
 		hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
-		ifree(tsd, ptr, tcache, true);
+		ifree(tsd, ptr, tcache, true, NULL);
 
 		check_entry_exit_locking(tsd_tsdn(tsd));
 		return NULL;
@@ -3883,11 +3888,11 @@ je_dallocx(void *ptr, int flags) {
 	UTRACE(ptr, 0, 0);
 	if (likely(fast)) {
 		tsd_assert_fast(tsd);
-		ifree(tsd, ptr, tcache, false);
+		ifree(tsd, ptr, tcache, false, NULL);
 	} else {
 		uintptr_t args_raw[3] = {(uintptr_t)ptr, flags};
 		hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw);
-		ifree(tsd, ptr, tcache, true);
+		ifree(tsd, ptr, tcache, true, NULL);
 	}
 	check_entry_exit_locking(tsd_tsdn(tsd));
 
@@ -3935,7 +3940,7 @@ je_sdallocx(void *ptr, size_t size, int flags) {
 	LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
 		size, flags);
 
-	if (flags != 0 || !free_fastpath(ptr, size, true)) {
+	if (flags != 0 || !free_fastpath(ptr, size, true, NULL)) {
 		sdallocx_default(ptr, size, flags);
 	}
 
@@ -3947,7 +3952,7 @@ je_sdallocx_noflags(void *ptr, size_t size) {
 	LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr,
 		size);
 
-	if (!free_fastpath(ptr, size, true)) {
+	if (!free_fastpath(ptr, size, true, NULL)) {
 		sdallocx_default(ptr, size, 0);
 	}
 
@@ -4483,3 +4488,21 @@ get_defrag_hint(void* ptr) {
 	assert(ptr != NULL);
 	return iget_defrag_hint(TSDN_NULL, ptr);
 }
+
+JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
+void JEMALLOC_NOTHROW *
+JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
+malloc_usable(size_t size, size_t *usize) {
+	return imalloc_fastpath(size, usize, &malloc_default);
+}
+
+JEMALLOC_EXPORT void JEMALLOC_NOTHROW
+free_usable(void *ptr, size_t *usable) {
+	LOG("core.free.entry", "ptr: %p", ptr);
+
+	if (!free_fastpath(ptr, 0, false, usable)) {
+		free_default(ptr, usable);
+	}
+
+	LOG("core.free.exit", "");
+}

src/zmalloc.c

@@ -51,6 +51,9 @@ void zlibc_free(void *ptr) {
  */
 #define MALLOC_MIN_SIZE(x) ((x) > 0 ? (x) : sizeof(long))
 
+void *je_malloc_usable(size_t size, size_t *usable);
+void je_free_usable(void *ptr, size_t *usable);
+
 /* Explicitly override malloc/free etc when using tcmalloc. */
 #if defined(USE_TCMALLOC)
 #define malloc(size) tc_malloc(size)
@@ -116,13 +119,21 @@ void *extend_to_usable(void *ptr, size_t size) {
 /* Try allocating memory, and return NULL if failed.
  * '*usable' is set to the usable size if non NULL. */
 static inline void *ztrymalloc_usable_internal(size_t size, size_t *usable) {
+    size_t usize = 0;
     /* Possible overflow, return NULL, so that the caller can panic or handle a failed allocation. */
     if (size >= SIZE_MAX/2) return NULL;
+#if defined(USE_JEMALLOC)
+    void *ptr = je_malloc_usable(MALLOC_MIN_SIZE(size)+PREFIX_SIZE, &usize);
+#else
     void *ptr = malloc(MALLOC_MIN_SIZE(size)+PREFIX_SIZE);
-
+#endif
     if (!ptr) return NULL;
 #ifdef HAVE_MALLOC_SIZE
+#if defined(USE_JEMALLOC)
     size = zmalloc_size(ptr);
+#else
+    size = usize;
+#endif
     update_zmalloc_stat_alloc(size);
     if (usable) *usable = size;
     return ptr;
@@ -423,8 +434,14 @@ void zfree(void *ptr) {
 
     if (ptr == NULL) return;
 #ifdef HAVE_MALLOC_SIZE
+#if defined(USE_JEMALLOC)
+    size_t usize = 0;
+    je_free_usable(ptr, &usize);
+    update_zmalloc_stat_free(usize);
+#else
     update_zmalloc_stat_free(zmalloc_size(ptr));
     free(ptr);
+#endif
 #else
     realptr = (char*)ptr-PREFIX_SIZE;
     oldsize = *((size_t*)realptr);
@@ -442,8 +459,15 @@ void zfree_usable(void *ptr, size_t *usable) {
 
     if (ptr == NULL) return;
 #ifdef HAVE_MALLOC_SIZE
+#if defined(USE_JEMALLOC)
+    size_t usize = 0;
+    je_free_usable(ptr, &usize);
+    update_zmalloc_stat_free(usize);
+    *usable = usize;
+#else
     update_zmalloc_stat_free(*usable = zmalloc_size(ptr));
     free(ptr);
+#endif
 #else
     realptr = (char*)ptr-PREFIX_SIZE;
     *usable = oldsize = *((size_t*)realptr);