Skip to content

Add estore stats #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: redesign-expire
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions src/ebuckets.c
Original file line number Diff line number Diff line change
Expand Up @@ -2407,8 +2407,12 @@ void ebGetStats(ebuckets eb, EbucketsType *type, ebucketsStats *stats) {
}
if (type->isEbStack) {
ebStack *stack = (ebStack *) eb;
EB_STACK_EXEC_L1(type, ebGetStats(stack->l1, type, stats));
EB_STACK_EXEC_L2(type, ebGetStats(stack->l2, type, stats));
EB_STACK_EXEC_L1(type, ebGetStats(stack->l1, type, stats));
stats->totalL1Items = stats->totalItems;
EB_STACK_EXEC_L1(type, ebGetStats(stack->l2, type, stats));
stats->totalL2Items = stats->totalItems - stats->totalL1Items;
stats->totalL3Items = stack->l3 ? stack->l3->items : 0;
stats->totalItems += stats->totalL3Items;
} else if (ebIsList(eb)) {
eItem head = ebGetListPtr(type, eb);
ExpireMeta *meta = type->getExpireMeta(head);
Expand Down Expand Up @@ -2447,8 +2451,12 @@ size_t ebGetStatsMsg(char *buf, size_t bufsize, ebucketsStats *stats, int full)
"Ebuckets stats:\n"
" total items: %lu\n"
" total buckets: %lu\n"
" total segments: %lu\n",
stats->totalItems, stats->totalBuckets, stats->totalSegments);
" total segments: %lu\n"
" total items in L1: %lu\n"
" total items in L2: %lu\n"
" total items in L3: %lu\n",
stats->totalItems, stats->totalBuckets, stats->totalSegments,
stats->totalL1Items, stats->totalL2Items, stats->totalL3Items);

if (full && stats->totalBuckets > 0 && stats->totalSegments > 0) {
l += snprintf(buf + l, bufsize - l,
Expand Down
18 changes: 18 additions & 0 deletions src/ebuckets.h
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,24 @@ static inline void ebSetMetaExpTime(ExpireMeta *expMeta, uint64_t t) {
expMeta->expireTimeHi = (uint16_t)((t) >> 32);
}

/* Statistics API */

#define EBUCKETS_STATS_VECTLEN 50

typedef struct ebucketsStats {
uint64_t totalItems; /* Total number of items */
uint64_t totalBuckets; /* Total number of buckets */
uint64_t totalSegments; /* Total number of segments */
uint64_t totalL1Items; /* Total number of items in L1 */
uint64_t totalL2Items; /* Total number of items in L2 */
uint64_t totalL3Items; /* Total number of items in L3 */
uint64_t avgItemsPerBucket; /* Average number of items per bucket */
uint64_t avgItemsPerSegment; /* Average number of items per segment */
uint64_t avgSegPerBucket; /* Average number of segments per bucket */
} ebucketsStats;
void ebGetStats(ebuckets eb, EbucketsType *type, ebucketsStats *stats);
size_t ebGetStatsMsg(char *buf, size_t bufsize, ebucketsStats *stats, int full);

/* Debug API */

void ebValidate(ebuckets eb, EbucketsType *type);
Expand Down
196 changes: 194 additions & 2 deletions src/expire.c
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,35 @@ void estoreIncrementalCascade(estore *es, uint64_t now, uint64_t maxCascade) {
}
}

void estoreCombineStats(ebucketsStats *from, ebucketsStats *into) {
into->totalItems += from->totalItems;
into->totalBuckets += from->totalBuckets;
into->totalSegments += from->totalSegments;
into->totalL1Items += from->totalL1Items;
into->totalL2Items += from->totalL2Items;
into->totalL3Items += from->totalL3Items;

if (into->totalBuckets == 0) return;

into->avgItemsPerBucket = into->totalItems / into->totalBuckets;
into->avgItemsPerSegment = into->totalItems / into->totalSegments;
into->avgSegPerBucket = into->totalSegments / into->totalBuckets;
}

void estoreGetStats(estore *es, char *buf, size_t bufsize, int full) {
if (!server.cluster_enabled) {
ebucketsStats stats = {0}; /* must be zeroed */
ebGetStats(es->buckets[0], es->bucket_type, &stats);
ebGetStatsMsg(buf, bufsize, &stats, full);
} else {
assert(0); // TODO_MOTI: Support cluster mode (See:kvstoreGetStats())
}
ebucketsStats clusterStats = {0}; /* must be zeroed */
for (int i = 0; i < es->num_buckets; i++) {
ebucketsStats stats = {0}; /* must be zeroed */
ebGetStats(es->buckets[i], es->bucket_type, &stats);
estoreCombineStats(&stats, &clusterStats);
}
ebGetStatsMsg(buf, bufsize, &clusterStats, full);
}
}

/* Get the total number of items in the expiration store */
Expand Down Expand Up @@ -1108,3 +1129,174 @@ void touchCommand(client *c) {
if (lookupKeyRead(c->db,c->argv[j]) != NULL) touched++;
addReplyLongLong(c,touched);
}

#ifdef REDIS_TEST
#include <stdio.h>
#include "testhelp.h"

#define TEST(name) printf("test — %s\n", name);
typedef struct TestKVObj {
kvobj obj;
ExpireMeta mexpire;
} TestKVObj;

ExpireMeta *getTestKVObjExpireMeta(const void *item) {
return &((TestKVObj *)item)->mexpire;
}

void deleteTestKVObjCallback(eItem item, void *ctx) {
UNUSED(ctx);
UNUSED(item);
}

EbucketsType testEbType = {
.getExpireMeta = getTestKVObjExpireMeta,
.onDeleteItem = deleteTestKVObjCallback,
.itemsAddrAreOdd = 0,
.ebp.precision = 0,
.ebp.keySize = EB_PRECISION2KEYSIZE(0 /*.precision*/),
};

/* ./redis-server test expire */
int expireTest(int argc, char **argv, int flags) {
UNUSED(argc);
UNUSED(argv);
UNUSED(flags);

TEST("estoreCombineStats correctly combines statistics") {
ebucketsStats from = {
.totalItems = 20,
.totalBuckets = 4,
.totalSegments = 2,
.avgItemsPerBucket = 0, /* will be recalculated */
.avgItemsPerSegment = 0,
.avgSegPerBucket = 0
};

ebucketsStats into = {
.totalItems = 10,
.totalBuckets = 2,
.totalSegments = 1,
.avgItemsPerBucket = 0,
.avgItemsPerSegment = 0,
.avgSegPerBucket = 0
};

estoreCombineStats(&from, &into);

/* Validate combined totals */
assert(into.totalItems == 30);
assert(into.totalBuckets == 6);
assert(into.totalSegments == 3);

/* Validate recomputed averages */
assert(into.avgItemsPerBucket == 5); /* 30 / 6 */
assert(into.avgItemsPerSegment == 10); /* 30 / 3 */
assert(into.avgSegPerBucket == 0); /* 3 / 6 = 0 (integer division) */

/* Edge case: totalBuckets = 0 */
ebucketsStats zeroBucketsFrom = {
.totalItems = 5,
.totalBuckets = 0,
.totalSegments = 2
};

ebucketsStats zeroBucketsInto = {
.totalItems = 0,
.totalBuckets = 0,
.totalSegments = 0
};

estoreCombineStats(&zeroBucketsFrom, &zeroBucketsInto);

/* Should add values, but skip average calculations */
assert(zeroBucketsInto.totalItems == 5);
assert(zeroBucketsInto.totalBuckets == 0);
assert(zeroBucketsInto.totalSegments == 2);
assert(zeroBucketsInto.avgItemsPerBucket == 0);
assert(zeroBucketsInto.avgItemsPerSegment == 0);
assert(zeroBucketsInto.avgSegPerBucket == 0);
}

TEST("estoreGetStats returns correct stats in non-clustered mode") {
server.cluster_enabled = 0;
EbucketsType stackType = testEbType;
stackType.isEbStack = 1;

estore *es = estoreCreate(&stackType, 0); /* only one bucket */

TestKVObj *kv1 = zmalloc(sizeof(TestKVObj));
TestKVObj *kv2 = zmalloc(sizeof(TestKVObj));
TestKVObj *kv3 = zmalloc(sizeof(TestKVObj));
TestKVObj *kv4 = zmalloc(sizeof(TestKVObj));

estoreAdd(es, (kvobj*)kv1, 0, 1);
estoreAdd(es, (kvobj*)kv2, 0, 1);
estoreAdd(es, (kvobj*)kv3, 0, EB_EXPIRE_TIME_MAX -1); /* Level 2 */
estoreAdd(es, (kvobj*)kv4, 0, EB_EXPIRE_TIME_MAX + 1); /* Level 3 */

char buf[1024];
memset(buf, 0, sizeof(buf));

estoreGetStats(es, buf, sizeof(buf), 1);

/* Validate that output includes expected stats */
assert(strstr(buf, " total items: 4") != NULL);
assert(strstr(buf, " total buckets: 2") != NULL);
assert(strstr(buf, " total segments: 2") != NULL);

assert(strstr(buf, " total items in L1: 2") != NULL);
assert(strstr(buf, " total items in L2: 1") != NULL);
assert(strstr(buf, " total items in L3: 1") != NULL);

assert(strstr(buf, " avg items per bucket: 2") != NULL);
assert(strstr(buf, " avg items per segment: 2") != NULL);
assert(strstr(buf, " avg segments per bucket: 1") != NULL);

zfree(kv1);
zfree(kv2);
zfree(kv3);
zfree(kv4);
estoreRelease(es);
}

TEST("estoreGetStats returns combined stats in clustered mode") {
server.cluster_enabled = 1;

estore *es = estoreCreate(&testEbType, 1); /* two buckets */

/* Add objects to different buckets */
TestKVObj *kv1 = zmalloc(sizeof(TestKVObj));
TestKVObj *kv2 = zmalloc(sizeof(TestKVObj));
TestKVObj *kv3 = zmalloc(sizeof(TestKVObj));
TestKVObj *kv4 = zmalloc(sizeof(TestKVObj));

estoreAdd(es, (kvobj*)kv1, 0, 1); /* bucket 0 */
estoreAdd(es, (kvobj*)kv2, 1, 1); /* bucket 1 */
estoreAdd(es, (kvobj*)kv3, 1, 2); /* bucket 1 */
estoreAdd(es, (kvobj*)kv4, 0, 2); /* bucket 0 */

char buf[1024];
memset(buf, 0, sizeof(buf));

estoreGetStats(es, buf, sizeof(buf), 1); /* full = 1 for verbose output */

/* Validate that combined values are present */
assert(strstr(buf, " total items: 4") != NULL);
assert(strstr(buf, " total buckets: 2") != NULL);
assert(strstr(buf, " total segments: 2") != NULL);

assert(strstr(buf, " avg items per bucket: 2") != NULL);
assert(strstr(buf, " avg items per segment: 2") != NULL);
assert(strstr(buf, " avg segments per bucket: 1") != NULL);

zfree(kv1);
zfree(kv2);
zfree(kv3);
zfree(kv4);
estoreRelease(es);
}

return 0;
}
#endif
4 changes: 4 additions & 0 deletions src/expire.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,8 @@ ebuckets *estoreGetBucket(estore *es, int slot);

size_t estoreMemUsage(estore *es);

#ifdef REDIS_TEST
int expireTest(int argc, char *argv[], int flags);
#endif

#endif
1 change: 1 addition & 0 deletions src/server.c
Original file line number Diff line number Diff line change
Expand Up @@ -7296,6 +7296,7 @@ struct redisTest {
{"listpack", listpackTest},
{"kvstore", kvstoreTest},
{"ebuckets", ebucketsTest},
{"expire", expireTest}
};
redisTestProc *getTestProcByName(const char *name) {
int numtests = sizeof(redisTests)/sizeof(struct redisTest);
Expand Down