Skip to content

FEATURE: Add cmdlog optional filtering #935

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

FEATURE: Add cmdlog optional filtering #935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 161 additions & 4 deletions cmdlog.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <fcntl.h>
#include <assert.h>
#include <errno.h>
#include <ctype.h>

#include "memcached/util.h"

Expand All @@ -38,6 +39,10 @@
#define CMDLOG_FILENAME_LENGTH CMDLOG_DIRPATH_LENGTH + 128
#define CMDLOG_FILENAME_FORMAT "%s/command_%d_%d_%d_%d.log"

#define CMDLOG_FILTER_MAXNUM 10
#define CMDLOG_FILTER_CMD_MAXLEN 15
#define CMDLOG_FILTER_KEY_MAXLEN 16000

/* cmdlog state */
#define CMDLOG_NOT_STARTED 0 /* not started */
#define CMDLOG_OVERFLOW_STOP 1 /* stop by command log overflow */
Expand All @@ -59,6 +64,12 @@ struct cmd_log_buffer {
uint32_t last;
};

/* command log filter structure */
struct cmd_log_filter {
char command[CMDLOG_FILTER_CMD_MAXLEN + 1];
char key[CMDLOG_FILTER_KEY_MAXLEN + 1];
};

/*command log flush structure */
struct cmd_log_flush {
pthread_t tid; /* flush thread id */
Expand All @@ -85,6 +96,8 @@ struct cmd_log_global {
struct cmd_log_buffer buffer;
struct cmd_log_flush flush;
struct cmd_log_stats stats;
struct cmd_log_filter filters[CMDLOG_FILTER_MAXNUM];
int nfilters;
volatile bool reqstop;
};
struct cmd_log_global cmdlog;
Expand Down Expand Up @@ -397,25 +410,42 @@ char *cmdlog_stats(void)
return str;
}

void cmdlog_write(char *client_ip, char *command)
void cmdlog_write(char *client_ip, char *command, int cmdlen)
{
struct tm *ptm;
struct timeval val;
struct cmd_log_buffer *buffer = &cmdlog.buffer;
char inputstr[CMDLOG_INPUT_SIZE];
char *ptr = command;
int inputlen;
int nwritten;
int seglen;

gettimeofday(&val, NULL);
ptm = localtime(&val.tv_sec);

nwritten = snprintf(inputstr, CMDLOG_INPUT_SIZE, "%02d:%02d:%02d.%06ld %s %s\n",
ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (long)val.tv_usec, client_ip, command);
nwritten = snprintf(inputstr, CMDLOG_INPUT_SIZE, "%02d:%02d:%02d.%06ld %s ",
ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (long)val.tv_usec, client_ip);

while (ptr < command + cmdlen && nwritten < CMDLOG_INPUT_SIZE) {
seglen = snprintf(inputstr + nwritten, CMDLOG_INPUT_SIZE - nwritten, "%s", ptr);
nwritten += seglen;
ptr += seglen;

if (ptr < command + cmdlen && nwritten < CMDLOG_INPUT_SIZE) {
inputstr[nwritten++] = ' ';
ptr++;
}
}

/* truncated ? */
if (nwritten > CMDLOG_INPUT_SIZE) {
if (nwritten >= CMDLOG_INPUT_SIZE) {
inputstr[CMDLOG_INPUT_SIZE-4] = '.';
inputstr[CMDLOG_INPUT_SIZE-3] = '.';
inputstr[CMDLOG_INPUT_SIZE-2] = '\n';
} else {
inputstr[nwritten++] = '\n';
inputstr[nwritten] = '\0';
}
inputlen = strlen(inputstr);

Expand Down Expand Up @@ -447,3 +477,130 @@ void cmdlog_write(char *client_ip, char *command)
}
pthread_mutex_unlock(&buffer->lock);
}

static int get_key_idx_by_command(const char *command)
{
assert(command);
if (strstr(command, "mget") ||
strcmp(command, "scan") == 0 ||
strcmp(command, "flush") == 0 ||
strcmp(command, "scrub") == 0 ||
strcmp(command, "stats") == 0 ||
strcmp(command, "config") == 0 ||
strcmp(command, "cmdlog") == 0 ||
strcmp(command, "lqdetect") == 0 ||
strcmp(command, "dump") == 0 ||
strcmp(command, "zkensemble") == 0 ||
strcmp(command, "help") == 0 ||
strcmp(command, "shutdown") == 0 ||
strcmp(command, "reload") == 0) {
return -1;
} else if ((command[0] == 'l' ||
command[0] == 's' ||
command[0] == 'm' ||
command[0] == 'b') &&
strcmp(command + 1, "op") == 0) {
return 2;
}

return 1;
}

bool is_cmdlog_filter_match(token_t *tokens, size_t ntokens)
{
assert(ntokens >= 2);
char command[CMDLOG_FILTER_CMD_MAXLEN + 1] = "";
char key[CMDLOG_FILTER_KEY_MAXLEN + 1] = "";
int key_idx = -1;

if (cmdlog.nfilters == 0) {
return true;
}

if (ntokens >= 3 && strcmp(tokens[0].value + 1, "op") == 0) {
snprintf(command, CMDLOG_FILTER_CMD_MAXLEN + 1, "%3s_%s", tokens[0].value, tokens[1].value);
} else {
snprintf(command, CMDLOG_FILTER_CMD_MAXLEN + 1, "%s", tokens[0].value);
}

key_idx = get_key_idx_by_command(command);
if (key_idx > 0) {
snprintf(key, CMDLOG_FILTER_KEY_MAXLEN + 1, "%s", tokens[key_idx].value);
}

for (int i = 0; i < cmdlog.nfilters; ++i) {
if (cmdlog.filters[i].command[0] == '\0' || strcmp(command, cmdlog.filters[i].command) == 0) {
if (cmdlog.filters[i].key[0] == '\0' || strcmp(key, cmdlog.filters[i].key) == 0) {
return true;
}
}
}

return false;
}

int cmdlog_filter_add(const char *command, int command_len, const char *key, int key_len)
{
if (cmdlog.nfilters >= CMDLOG_FILTER_MAXNUM) {
return 0;
}

if (command_len > CMDLOG_FILTER_CMD_MAXLEN || key_len > CMDLOG_FILTER_KEY_MAXLEN) {
return -1;
}

if (key_len > 0) {
for (int i = 0; key[i] != '\0' && i < CMDLOG_FILTER_KEY_MAXLEN; ++i) {
if (!isgraph(key[i])) {
return -1;
}
}
}

pthread_mutex_lock(&cmdlog.lock);
strcpy(cmdlog.filters[cmdlog.nfilters].command, command);
strcpy(cmdlog.filters[cmdlog.nfilters].key, key);
cmdlog.nfilters++;
pthread_mutex_unlock(&cmdlog.lock);
return 1;
}

int cmdlog_filter_remove(int idx, bool remove_all)
{
int nremove = 0;

if (idx < 0 || idx >= cmdlog.nfilters) {
return -1;
}

pthread_mutex_lock(&cmdlog.lock);
if (remove_all) {
nremove += cmdlog.nfilters;
cmdlog.nfilters = 0;
} else {
for (int i = idx; i < cmdlog.nfilters - 1; ++i) {
cmdlog.filters[i] = cmdlog.filters[i + 1];
}
nremove++;
cmdlog.nfilters--;
}
pthread_mutex_unlock(&cmdlog.lock);
return nremove;
}

char *cmdlog_filter_list(void)
{
char *buf = (char *)malloc((cmdlog.nfilters + 1) * CMDLOG_INPUT_SIZE);
int nwritten = 0;

pthread_mutex_lock(&cmdlog.lock);
if (buf) {
nwritten += snprintf(buf, CMDLOG_INPUT_SIZE, "\t(%d / %d)\n", cmdlog.nfilters, CMDLOG_FILTER_MAXNUM);
for (int i = 0; i < cmdlog.nfilters; ++i) {
nwritten += snprintf(buf + nwritten, CMDLOG_INPUT_SIZE, "\t%d. command = %s, key = %s\n",
i, cmdlog.filters[i].command, cmdlog.filters[i].key);
}
}
pthread_mutex_unlock(&cmdlog.lock);
return buf;
}
6 changes: 5 additions & 1 deletion cmdlog.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,9 @@ void cmdlog_final(void);
int cmdlog_start(char *file_path, bool *already_started);
void cmdlog_stop(bool *already_stopped);
char *cmdlog_stats(void);
void cmdlog_write(char *client_ip, char *command);
void cmdlog_write(char *client_ip, char *command, int cmdlen);
bool is_cmdlog_filter_match(token_t *tokens, size_t ntokens);
int cmdlog_filter_add(const char *command, int command_len, const char *key, int key_len);
int cmdlog_filter_remove(int idx, bool remove_all);
char *cmdlog_filter_list(void);
#endif
95 changes: 90 additions & 5 deletions memcached.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10567,6 +10567,89 @@ static void process_scan_command(conn *c, token_t *tokens, const size_t ntokens)
#endif

#ifdef COMMAND_LOGGING
static void process_cmdlog_filter(conn *c, token_t *tokens, const size_t ntokens)
{
assert(ntokens > 3);
const char *subcommand = tokens[2].value;
int ret = 0;

if (strcmp(subcommand, "add") == 0) {
char *command = "";
char *key = "";
int command_len = 0;
int key_len = 0;

if (ntokens == 6 && strcmp(tokens[3].value, "command") == 0) {
command = tokens[4].value;
command_len = tokens[4].length;
} else if (ntokens == 6 && strcmp(tokens[3].value, "key") == 0) {
key = tokens[4].value;
key_len = tokens[4].length;
} else if (ntokens == 8 && strcmp(tokens[3].value, "command") == 0 && strcmp(tokens[5].value, "key") == 0) {
command = tokens[4].value;
command_len = tokens[4].length;
key = tokens[6].value;
key_len = tokens[6].length;
} else {
out_string(c, "CLIENT_ERROR bad command line format");
return;
}

ret = cmdlog_filter_add(command, command_len, key, key_len);
if (ret > 0) {
int len = command_len + key_len + 26;
char *buf = (char *)malloc(len);
if (buf) {
snprintf(buf, len, "\tFilter: command = %s, key = %s", command, key);
out_string(c, buf);
free(buf);
} else {
out_string(c, "SERVER_ERROR out of memory");
}
} else if (ret == 0) {
out_string(c, "CLIENT_ERROR filter list is full");
} else {
out_string(c, "CLIENT_ERROR invalid parameters");
}
} else if (strcmp(subcommand, "remove") == 0) {
int idx = 0;
bool remove_all = false;

if (ntokens == 4) {
remove_all = true;
} else if (ntokens == 5) {
if (!safe_strtol(tokens[3].value, &idx)) idx = -1;
} else {
out_string(c, "CLIENT_ERROR bad command line format");
return;
}

ret = cmdlog_filter_remove(idx, remove_all);
if (ret > 0) {
int len = 32;
char buf[len];
snprintf(buf, len, "\t%d filters removed", ret);
out_string(c, buf);
} else {
out_string(c, "CLIENT_ERROR invalid parameters");
}
} else if (strcmp(subcommand, "list") == 0) {
if (ntokens != 4) {
out_string(c, "CLIENT_ERROR bad command line format");
return;
}

char *ret_str = cmdlog_filter_list();
if (ret_str) {
write_and_free(c, ret_str, strlen(ret_str));
} else {
out_string(c, "SERVER_ERROR out of memory");
}
} else {
out_string(c, "CLIENT_ERROR bad command line format");
}
}

static void process_cmdlog_command(conn *c, token_t *tokens, const size_t ntokens)
{
char *type = tokens[SUBCOMMAND_TOKEN].value;
Expand Down Expand Up @@ -10608,8 +10691,10 @@ static void process_cmdlog_command(conn *c, token_t *tokens, const size_t ntoken
} else {
out_string(c, "\tcommand logging failed to get stats memory.\n");
}
} else if (ntokens > 3 && strcmp(type, "filter") == 0) {
process_cmdlog_filter(c, tokens, ntokens);
} else {
out_string(c, "\t* Usage: cmdlog [start [path] | stop | stats]\n");
out_string(c, "\t* Usage: cmdlog [start [path] | stop | stats | filter [add|remove|list]]\n");
}
}
#endif
Expand Down Expand Up @@ -13861,14 +13946,14 @@ static void process_command_ascii(conn *c, char *command, int cmdlen)
return;
}

ntokens = tokenize_command(command, cmdlen, tokens, MAX_TOKENS);

#ifdef COMMAND_LOGGING
if (cmdlog_in_use) {
cmdlog_write(c->client_ip, command);
if (cmdlog_in_use && is_cmdlog_filter_match(tokens, ntokens)) {
cmdlog_write(c->client_ip, command, cmdlen);
}
#endif

ntokens = tokenize_command(command, cmdlen, tokens, MAX_TOKENS);

if ((ntokens >= 3) && (strcmp(tokens[COMMAND_TOKEN].value, "get") == 0))
{
process_get_command(c, tokens, ntokens, false, false);
Expand Down