diff --git a/src/aof.c b/src/aof.c index 8ccd8d8f887..f3f6d782fc4 100644 --- a/src/aof.c +++ b/src/aof.c @@ -30,6 +30,13 @@ aofManifest *aofLoadManifestFromFile(sds am_filepath); void aofManifestFreeAndUpdate(aofManifest *am); void aof_background_fsync_and_close(int fd); +/* When we call 'startAppendOnly', we will create a temp INCR AOF, and rename + * it to the real INCR AOF name when the AOFRW is done, so if want to know the + * accurate start offset of the INCR AOF, we need to record it when we create + * the temp INCR AOF. This variable is used to record the start offset, and + * set the start offset of the real INCR AOF when the AOFRW is done. */ +static long long tempIncAofStartReplOffset = 0; + /* ---------------------------------------------------------------------------- * AOF Manifest file implementation. * @@ -73,10 +80,15 @@ void aof_background_fsync_and_close(int fd); #define AOF_MANIFEST_KEY_FILE_NAME "file" #define AOF_MANIFEST_KEY_FILE_SEQ "seq" #define AOF_MANIFEST_KEY_FILE_TYPE "type" +#define AOF_MANIFEST_KEY_FILE_STARTOFFSET "startoffset" +#define AOF_MANIFEST_KEY_FILE_ENDOFFSET "endoffset" /* Create an empty aofInfo. */ aofInfo *aofInfoCreate(void) { - return zcalloc(sizeof(aofInfo)); + aofInfo *ai = zcalloc(sizeof(aofInfo)); + ai->start_offset = -1; + ai->end_offset = -1; + return ai; } /* Free the aofInfo structure (pointed to by ai) and its embedded file_name. */ @@ -93,6 +105,8 @@ aofInfo *aofInfoDup(aofInfo *orig) { ai->file_name = sdsdup(orig->file_name); ai->file_seq = orig->file_seq; ai->file_type = orig->file_type; + ai->start_offset = orig->start_offset; + ai->end_offset = orig->end_offset; return ai; } @@ -105,10 +119,19 @@ sds aofInfoFormat(sds buf, aofInfo *ai) { if (sdsneedsrepr(ai->file_name)) filename_repr = sdscatrepr(sdsempty(), ai->file_name, sdslen(ai->file_name)); - sds ret = sdscatprintf(buf, "%s %s %s %lld %s %c\n", + sds ret = sdscatprintf(buf, "%s %s %s %lld %s %c", AOF_MANIFEST_KEY_FILE_NAME, filename_repr ? filename_repr : ai->file_name, AOF_MANIFEST_KEY_FILE_SEQ, ai->file_seq, AOF_MANIFEST_KEY_FILE_TYPE, ai->file_type); + + if (ai->start_offset != -1) { + ret = sdscatprintf(ret, " %s %lld", AOF_MANIFEST_KEY_FILE_STARTOFFSET, ai->start_offset); + if (ai->end_offset != -1) { + ret = sdscatprintf(ret, " %s %lld", AOF_MANIFEST_KEY_FILE_ENDOFFSET, ai->end_offset); + } + } + + ret = sdscatlen(ret, "\n", 1); sdsfree(filename_repr); return ret; @@ -304,6 +327,10 @@ aofManifest *aofLoadManifestFromFile(sds am_filepath) { ai->file_seq = atoll(argv[i+1]); } else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_FILE_TYPE)) { ai->file_type = (argv[i+1])[0]; + } else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_FILE_STARTOFFSET)) { + ai->start_offset = atoll(argv[i+1]); + } else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_FILE_ENDOFFSET)) { + ai->end_offset = atoll(argv[i+1]); } /* else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_OTHER)) {} */ } @@ -433,12 +460,13 @@ sds getNewBaseFileNameAndMarkPreAsHistory(aofManifest *am) { * for example: * appendonly.aof.1.incr.aof */ -sds getNewIncrAofName(aofManifest *am) { +sds getNewIncrAofName(aofManifest *am, long long start_reploff) { aofInfo *ai = aofInfoCreate(); ai->file_type = AOF_FILE_TYPE_INCR; ai->file_name = sdscatprintf(sdsempty(), "%s.%lld%s%s", server.aof_filename, ++am->curr_incr_file_seq, INCR_FILE_SUFFIX, AOF_FORMAT_SUFFIX); ai->file_seq = am->curr_incr_file_seq; + ai->start_offset = start_reploff; listAddNodeTail(am->incr_aof_list, ai); am->dirty = 1; return ai->file_name; @@ -456,7 +484,7 @@ sds getLastIncrAofName(aofManifest *am) { /* If 'incr_aof_list' is empty, just create a new one. */ if (!listLength(am->incr_aof_list)) { - return getNewIncrAofName(am); + return getNewIncrAofName(am, server.master_repl_offset); } /* Or return the last one. */ @@ -781,10 +809,11 @@ int openNewIncrAofForAppend(void) { if (server.aof_state == AOF_WAIT_REWRITE) { /* Use a temporary INCR AOF file to accumulate data during AOF_WAIT_REWRITE. */ new_aof_name = getTempIncrAofName(); + tempIncAofStartReplOffset = server.master_repl_offset; } else { /* Dup a temp aof_manifest to modify. */ temp_am = aofManifestDup(server.aof_manifest); - new_aof_name = sdsdup(getNewIncrAofName(temp_am)); + new_aof_name = sdsdup(getNewIncrAofName(temp_am, server.master_repl_offset)); } sds new_aof_filepath = makePath(server.aof_dirname, new_aof_name); newfd = open(new_aof_filepath, O_WRONLY|O_TRUNC|O_CREAT, 0644); @@ -833,6 +862,50 @@ int openNewIncrAofForAppend(void) { return C_ERR; } +/* When we close gracefully the AOF file, we have the chance to persist the + * end replication offset of current INCR AOF. */ +void updateCurIncrAofEndOffset(void) { + if (server.aof_state != AOF_ON) return; + serverAssert(server.aof_manifest != NULL); + + if (listLength(server.aof_manifest->incr_aof_list) == 0) return; + aofInfo *ai = listNodeValue(listLast(server.aof_manifest->incr_aof_list)); + ai->end_offset = server.master_repl_offset; + server.aof_manifest->dirty = 1; + /* It doesn't matter if the persistence fails since this information is not + * critical, we can get an approximate value by start offset plus file size. */ + persistAofManifest(server.aof_manifest); +} + +/* After loading AOF data, we need to update the `server.master_repl_offset` + * based on the information of the last INCR AOF, to avoid the rollback of + * the start offset of new INCR AOF. */ +void updateReplOffsetAndResetEndOffset(void) { + if (server.aof_state != AOF_ON) return; + serverAssert(server.aof_manifest != NULL); + + /* If the INCR file has an end offset, we directly use it, and clear it + * to avoid the next time we load the manifest file, we will use the same + * offset, but the real offset may have advanced. */ + if (listLength(server.aof_manifest->incr_aof_list) == 0) return; + aofInfo *ai = listNodeValue(listLast(server.aof_manifest->incr_aof_list)); + if (ai->end_offset != -1) { + server.master_repl_offset = ai->end_offset; + ai->end_offset = -1; + server.aof_manifest->dirty = 1; + /* We must update the end offset of INCR file correctly, otherwise we + * may keep wrong information in the manifest file, since we continue + * to append data to the same INCR file. */ + if (persistAofManifest(server.aof_manifest) != AOF_OK) + exit(1); + } else { + /* If the INCR file doesn't have an end offset, we need to calculate + * the replication offset by the start offset plus the file size. */ + server.master_repl_offset = (ai->start_offset == -1 ? 0 : ai->start_offset) + + getAppendOnlyFileSize(ai->file_name, NULL); + } +} + /* Whether to limit the execution of Background AOF rewrite. * * At present, if AOFRW fails, redis will automatically retry. If it continues @@ -938,6 +1011,7 @@ void stopAppendOnly(void) { server.aof_last_fsync = server.mstime; } close(server.aof_fd); + updateCurIncrAofEndOffset(); server.aof_fd = -1; server.aof_selected_db = -1; @@ -1071,35 +1145,34 @@ void flushAppendOnlyFile(int force) { mstime_t latency; if (sdslen(server.aof_buf) == 0) { - /* Check if we need to do fsync even the aof buffer is empty, - * because previously in AOF_FSYNC_EVERYSEC mode, fsync is - * called only when aof buffer is not empty, so if users - * stop write commands before fsync called in one second, - * the data in page cache cannot be flushed in time. */ - if (server.aof_fsync == AOF_FSYNC_EVERYSEC && - server.aof_last_incr_fsync_offset != server.aof_last_incr_size && - server.mstime - server.aof_last_fsync >= 1000 && - !(sync_in_progress = aofFsyncInProgress())) { - goto try_fsync; - - /* Check if we need to do fsync even the aof buffer is empty, - * the reason is described in the previous AOF_FSYNC_EVERYSEC block, - * and AOF_FSYNC_ALWAYS is also checked here to handle a case where - * aof_fsync is changed from everysec to always. */ - } else if (server.aof_fsync == AOF_FSYNC_ALWAYS && - server.aof_last_incr_fsync_offset != server.aof_last_incr_size) - { - goto try_fsync; - } else { + if (server.aof_last_incr_fsync_offset == server.aof_last_incr_size) { /* All data is fsync'd already: Update fsynced_reploff_pending just in case. - * This is needed to avoid a WAITAOF hang in case a module used RM_Call with the NO_AOF flag, - * in which case master_repl_offset will increase but fsynced_reploff_pending won't be updated - * (because there's no reason, from the AOF POV, to call fsync) and then WAITAOF may wait on - * the higher offset (which contains data that was only propagated to replicas, and not to AOF) */ - if (!sync_in_progress && server.aof_fsync != AOF_FSYNC_NO) + * This is needed to avoid a WAITAOF hang in case a module used RM_Call + * with the NO_AOF flag, in which case master_repl_offset will increase but + * fsynced_reploff_pending won't be updated (because there's no reason, from + * the AOF POV, to call fsync) and then WAITAOF may wait on the higher offset + * (which contains data that was only propagated to replicas, and not to AOF) */ + if (!aofFsyncInProgress()) atomicSet(server.fsynced_reploff_pending, server.master_repl_offset); - return; + } else { + /* Check if we need to do fsync even the aof buffer is empty, + * because previously in AOF_FSYNC_EVERYSEC mode, fsync is + * called only when aof buffer is not empty, so if users + * stop write commands before fsync called in one second, + * the data in page cache cannot be flushed in time. */ + if (server.aof_fsync == AOF_FSYNC_EVERYSEC && + server.mstime - server.aof_last_fsync >= 1000 && + !(sync_in_progress = aofFsyncInProgress())) + goto try_fsync; + + /* Check if we need to do fsync even the aof buffer is empty, + * the reason is described in the previous AOF_FSYNC_EVERYSEC block, + * and AOF_FSYNC_ALWAYS is also checked here to handle a case where + * aof_fsync is changed from everysec to always. */ + if (server.aof_fsync == AOF_FSYNC_ALWAYS) + goto try_fsync; } + return; } if (server.aof_fsync == AOF_FSYNC_EVERYSEC) @@ -2665,7 +2738,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { sds temp_incr_aof_name = getTempIncrAofName(); sds temp_incr_filepath = makePath(server.aof_dirname, temp_incr_aof_name); /* Get next new incr aof name. */ - sds new_incr_filename = getNewIncrAofName(temp_am); + sds new_incr_filename = getNewIncrAofName(temp_am, tempIncAofStartReplOffset); new_incr_filepath = makePath(server.aof_dirname, new_incr_filename); latencyStartMonitor(latency); if (rename(temp_incr_filepath, new_incr_filepath) == -1) { diff --git a/src/commands.def b/src/commands.def index d8f49634217..9d70326e6a6 100644 --- a/src/commands.def +++ b/src/commands.def @@ -3472,6 +3472,78 @@ struct COMMAND_ARG HGETALL_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/********** HGETDEL ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HGETDEL history */ +#define HGETDEL_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HGETDEL tips */ +#define HGETDEL_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HGETDEL key specs */ +keySpec HGETDEL_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HGETDEL fields argument table */ +struct COMMAND_ARG HGETDEL_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HGETDEL argument table */ +struct COMMAND_ARG HGETDEL_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HGETDEL_fields_Subargs}, +}; + +/********** HGETEX ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HGETEX history */ +#define HGETEX_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HGETEX tips */ +#define HGETEX_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HGETEX key specs */ +keySpec HGETEX_Keyspecs[1] = { +{"RW and UPDATE because it changes the TTL",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HGETEX expiration argument table */ +struct COMMAND_ARG HGETEX_expiration_Subargs[] = { +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("persist",ARG_TYPE_PURE_TOKEN,-1,"PERSIST",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HGETEX fields argument table */ +struct COMMAND_ARG HGETEX_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HGETEX argument table */ +struct COMMAND_ARG HGETEX_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HGETEX_expiration_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HGETEX_fields_Subargs}, +}; + /********** HINCRBY ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -3903,6 +3975,60 @@ struct COMMAND_ARG HSET_Args[] = { {MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSET_data_Subargs}, }; +/********** HSETEX ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HSETEX history */ +#define HSETEX_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HSETEX tips */ +#define HSETEX_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HSETEX key specs */ +keySpec HSETEX_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HSETEX condition argument table */ +struct COMMAND_ARG HSETEX_condition_Subargs[] = { +{MAKE_ARG("fnx",ARG_TYPE_PURE_TOKEN,-1,"FNX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fxx",ARG_TYPE_PURE_TOKEN,-1,"FXX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETEX expiration argument table */ +struct COMMAND_ARG HSETEX_expiration_Subargs[] = { +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("keepttl",ARG_TYPE_PURE_TOKEN,-1,"KEEPTTL",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETEX fields data argument table */ +struct COMMAND_ARG HSETEX_fields_data_Subargs[] = { +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETEX fields argument table */ +struct COMMAND_ARG HSETEX_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSETEX_fields_data_Subargs}, +}; + +/* HSETEX argument table */ +struct COMMAND_ARG HSETEX_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETEX_condition_Subargs}, +{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HSETEX_expiration_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HSETEX_fields_Subargs}, +}; + /********** HSETNX ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -11038,11 +11164,13 @@ struct COMMAND_STRUCT redisCommandTable[] = { /* hash */ {MAKE_CMD("hdel","Deletes one or more fields and their values from a hash. Deletes the hash if no fields remain.","O(N) where N is the number of fields to be removed.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HDEL_History,1,HDEL_Tips,0,hdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HDEL_Keyspecs,1,NULL,2),.args=HDEL_Args}, {MAKE_CMD("hexists","Determines whether a field exists in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXISTS_History,0,HEXISTS_Tips,0,hexistsCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXISTS_Keyspecs,1,NULL,2),.args=HEXISTS_Args}, -{MAKE_CMD("hexpire","Set expiry for hash field using relative time to expire (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args}, -{MAKE_CMD("hexpireat","Set expiry for hash field using an absolute Unix timestamp (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireatCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIREAT_Keyspecs,1,NULL,4),.args=HEXPIREAT_Args}, +{MAKE_CMD("hexpire","Set expiry for hash field using relative time to expire (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args}, +{MAKE_CMD("hexpireat","Set expiry for hash field using an absolute Unix timestamp (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireatCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HEXPIREAT_Keyspecs,1,NULL,4),.args=HEXPIREAT_Args}, {MAKE_CMD("hexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in seconds.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,2),.args=HEXPIRETIME_Args}, {MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args}, {MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args}, +{MAKE_CMD("hgetdel","Returns the value of a field and deletes it from the hash.","O(N) where N is the number of specified fields","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETDEL_History,0,HGETDEL_Tips,0,hgetdelCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HGETDEL_Keyspecs,1,NULL,2),.args=HGETDEL_Args}, +{MAKE_CMD("hgetex","Get the value of one or more fields of a given hash key, and optionally set their expiration.","O(N) where N is the number of specified fields","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETEX_History,0,HGETEX_Tips,0,hgetexCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HGETEX_Keyspecs,1,NULL,3),.args=HGETEX_Args}, {MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args}, {MAKE_CMD("hincrbyfloat","Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBYFLOAT_History,0,HINCRBYFLOAT_Tips,0,hincrbyfloatCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBYFLOAT_Keyspecs,1,NULL,3),.args=HINCRBYFLOAT_Args}, {MAKE_CMD("hkeys","Returns all fields in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HKEYS_History,0,HKEYS_Tips,1,hkeysCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HKEYS_Keyspecs,1,NULL,1),.args=HKEYS_Args}, @@ -11050,13 +11178,14 @@ struct COMMAND_STRUCT redisCommandTable[] = { {MAKE_CMD("hmget","Returns the values of all fields in a hash.","O(N) where N is the number of fields being requested.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMGET_History,0,HMGET_Tips,0,hmgetCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HMGET_Keyspecs,1,NULL,2),.args=HMGET_Args}, {MAKE_CMD("hmset","Sets the values of multiple fields.","O(N) where N is the number of fields being set.","2.0.0",CMD_DOC_DEPRECATED,"`HSET` with multiple field-value pairs","4.0.0","hash",COMMAND_GROUP_HASH,HMSET_History,0,HMSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HMSET_Keyspecs,1,NULL,2),.args=HMSET_Args}, {MAKE_CMD("hpersist","Removes the expiration time for each specified field","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPERSIST_History,0,HPERSIST_Tips,0,hpersistCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HPERSIST_Keyspecs,1,NULL,2),.args=HPERSIST_Args}, -{MAKE_CMD("hpexpire","Set expiry for hash field using relative time to expire (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRE_History,0,HPEXPIRE_Tips,0,hpexpireCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRE_Keyspecs,1,NULL,4),.args=HPEXPIRE_Args}, -{MAKE_CMD("hpexpireat","Set expiry for hash field using an absolute Unix timestamp (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIREAT_History,0,HPEXPIREAT_Tips,0,hpexpireatCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIREAT_Keyspecs,1,NULL,4),.args=HPEXPIREAT_Args}, +{MAKE_CMD("hpexpire","Set expiry for hash field using relative time to expire (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRE_History,0,HPEXPIRE_Tips,0,hpexpireCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRE_Keyspecs,1,NULL,4),.args=HPEXPIRE_Args}, +{MAKE_CMD("hpexpireat","Set expiry for hash field using an absolute Unix timestamp (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIREAT_History,0,HPEXPIREAT_Tips,0,hpexpireatCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIREAT_Keyspecs,1,NULL,4),.args=HPEXPIREAT_Args}, {MAKE_CMD("hpexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in msec.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRETIME_History,0,HPEXPIRETIME_Tips,0,hpexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRETIME_Keyspecs,1,NULL,2),.args=HPEXPIRETIME_Args}, {MAKE_CMD("hpttl","Returns the TTL in milliseconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPTTL_History,0,HPTTL_Tips,1,hpttlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPTTL_Keyspecs,1,NULL,2),.args=HPTTL_Args}, {MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args}, {MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args}, {MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSET_Keyspecs,1,NULL,2),.args=HSET_Args}, +{MAKE_CMD("hsetex","Set the value of one or more fields of a given hash key, and optionally set their expiration.","O(N) where N is the number of fields being set.","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETEX_History,0,HSETEX_Tips,0,hsetexCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETEX_Keyspecs,1,NULL,4),.args=HSETEX_Args}, {MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args}, {MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args}, {MAKE_CMD("httl","Returns the TTL in seconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,1,httlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,2),.args=HTTL_Args}, diff --git a/src/commands/hexpire.json b/src/commands/hexpire.json index 832c182aea2..a08a1d1f5dc 100644 --- a/src/commands/hexpire.json +++ b/src/commands/hexpire.json @@ -9,7 +9,6 @@ "history": [], "command_flags": [ "WRITE", - "DENYOOM", "FAST" ], "acl_categories": [ diff --git a/src/commands/hexpireat.json b/src/commands/hexpireat.json index 4a7c0c71886..7e8ea33cad9 100644 --- a/src/commands/hexpireat.json +++ b/src/commands/hexpireat.json @@ -9,7 +9,6 @@ "history": [], "command_flags": [ "WRITE", - "DENYOOM", "FAST" ], "acl_categories": [ diff --git a/src/commands/hgetdel.json b/src/commands/hgetdel.json new file mode 100644 index 00000000000..af748fb52a9 --- /dev/null +++ b/src/commands/hgetdel.json @@ -0,0 +1,78 @@ +{ + "HGETDEL": { + "summary": "Returns the value of a field and deletes it from the hash.", + "complexity": "O(N) where N is the number of specified fields", + "group": "hash", + "since": "8.0.0", + "arity": -5, + "function": "hgetdelCommand", + "history": [], + "command_flags": [ + "WRITE", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "ACCESS", + "DELETE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the given fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer" + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} + diff --git a/src/commands/hgetex.json b/src/commands/hgetex.json new file mode 100644 index 00000000000..02889ca8392 --- /dev/null +++ b/src/commands/hgetex.json @@ -0,0 +1,111 @@ +{ + "HGETEX": { + "summary": "Get the value of one or more fields of a given hash key, and optionally set their expiration.", + "complexity": "O(N) where N is the number of specified fields", + "group": "hash", + "since": "8.0.0", + "arity": -5, + "function": "hgetexCommand", + "history": [], + "command_flags": [ + "WRITE", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "notes": "RW and UPDATE because it changes the TTL", + "flags": [ + "RW", + "ACCESS", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the given fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "expiration", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "seconds", + "type": "integer", + "token": "EX" + }, + { + "name": "milliseconds", + "type": "integer", + "token": "PX" + }, + { + "name": "unix-time-seconds", + "type": "unix-time", + "token": "EXAT" + }, + { + "name": "unix-time-milliseconds", + "type": "unix-time", + "token": "PXAT" + }, + { + "name": "persist", + "type": "pure-token", + "token": "PERSIST" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer" + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} + diff --git a/src/commands/hpexpire.json b/src/commands/hpexpire.json index 02c68e61634..269eb4eace0 100644 --- a/src/commands/hpexpire.json +++ b/src/commands/hpexpire.json @@ -9,7 +9,6 @@ "history": [], "command_flags": [ "WRITE", - "DENYOOM", "FAST" ], "acl_categories": [ diff --git a/src/commands/hpexpireat.json b/src/commands/hpexpireat.json index 58e5555fb5f..f5f1319270b 100644 --- a/src/commands/hpexpireat.json +++ b/src/commands/hpexpireat.json @@ -9,7 +9,6 @@ "history": [], "command_flags": [ "WRITE", - "DENYOOM", "FAST" ], "acl_categories": [ diff --git a/src/commands/hsetex.json b/src/commands/hsetex.json new file mode 100644 index 00000000000..6f6a6c60079 --- /dev/null +++ b/src/commands/hsetex.json @@ -0,0 +1,132 @@ +{ + "HSETEX": { + "summary": "Set the value of one or more fields of a given hash key, and optionally set their expiration.", + "complexity": "O(N) where N is the number of fields being set.", + "group": "hash", + "since": "8.0.0", + "arity": -6, + "function": "hsetexCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "oneOf": [ + { + "description": "No field was set (due to FXX or FNX flags).", + "const": 0 + }, + { + "description": "All the fields were set.", + "const": 1 + } + ] + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "fnx", + "type": "pure-token", + "token": "FNX" + }, + { + "name": "fxx", + "type": "pure-token", + "token": "FXX" + } + ] + }, + { + "name": "expiration", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "seconds", + "type": "integer", + "token": "EX" + }, + { + "name": "milliseconds", + "type": "integer", + "token": "PX" + }, + { + "name": "unix-time-seconds", + "type": "unix-time", + "token": "EXAT" + }, + { + "name": "unix-time-milliseconds", + "type": "unix-time", + "token": "PXAT" + }, + { + "name": "keepttl", + "type": "pure-token", + "token": "KEEPTTL" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer" + }, + { + "name": "data", + "type": "block", + "multiple": true, + "arguments": [ + { + "name": "field", + "type": "string" + }, + { + "name": "value", + "type": "string" + } + ] + } + ] + } + ] + } +} diff --git a/src/defrag.c b/src/defrag.c index 35a55b92a90..3668da2b775 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -23,7 +23,7 @@ #ifdef HAVE_DEFRAG -#define DEFRAG_CYCLE_US 500 /* The time spent (in microseconds) of the periodic active defrag process */ +#define DEFRAG_CYCLE_US 500 /* Standard duration of defrag cycle (in microseconds) */ typedef enum { DEFRAG_NOT_DONE = 0, DEFRAG_DONE = 1 } doneStatus; @@ -124,6 +124,12 @@ typedef struct { } defragPubSubCtx; static_assert(offsetof(defragPubSubCtx, kvstate) == 0, "defragStageKvstoreHelper requires this"); +typedef struct { + sds module_name; + RedisModuleDefragCtx *module_ctx; + unsigned long cursor; +} defragModuleCtx; + /* this method was added to jemalloc in order to help us understand which * pointers are worthwhile moving and which aren't */ int je_get_defrag_hint(void* ptr); @@ -1211,11 +1217,33 @@ static doneStatus defragLuaScripts(void *ctx, monotime endtime) { return DEFRAG_DONE; } +/* Handles defragmentation of module global data. This is a stage function + * that gets called periodically during the active defragmentation process. */ static doneStatus defragModuleGlobals(void *ctx, monotime endtime) { - UNUSED(endtime); - UNUSED(ctx); - moduleDefragGlobals(); - return DEFRAG_DONE; + defragModuleCtx *defrag_module_ctx = ctx; + + RedisModule *module = moduleGetHandleByName(defrag_module_ctx->module_name); + if (!module) { + /* Module has been unloaded, nothing to defrag. */ + return DEFRAG_DONE; + } + + /* Set up context for the module's defrag callback. */ + defrag_module_ctx->module_ctx->endtime = endtime; + defrag_module_ctx->module_ctx->cursor = &defrag_module_ctx->cursor; + + /* Call appropriate version of module's defrag callback: + * 1. Version 2 (defrag_cb_2): Supports incremental defrag and returns whether more work is needed + * 2. Version 1 (defrag_cb): Legacy version, performs all work in one call. + * Note: V1 doesn't support incremental defragmentation, may block for longer periods. */ + if (module->defrag_cb_2) { + return module->defrag_cb_2(defrag_module_ctx->module_ctx) ? DEFRAG_NOT_DONE : DEFRAG_DONE; + } else if (module->defrag_cb) { + module->defrag_cb(defrag_module_ctx->module_ctx); + return DEFRAG_DONE; + } else { + redis_unreachable(); + } } static void freeDefragKeysContext(void *ctx) { @@ -1226,6 +1254,13 @@ static void freeDefragKeysContext(void *ctx) { zfree(defrag_keys_ctx); } +static void freeDefragModelContext(void *ctx) { + defragModuleCtx *defrag_model_ctx = ctx; + sdsfree(defrag_model_ctx->module_name); + zfree(defrag_model_ctx->module_ctx); + zfree(defrag_model_ctx); +} + static void freeDefragContext(void *ptr) { StageDescriptor *stage = ptr; if (stage->ctx_free_fn) @@ -1508,7 +1543,21 @@ static void beginDefragCycle(void) { addDefragStage(defragStagePubsubKvstore, zfree, defrag_pubsubshard_ctx); addDefragStage(defragLuaScripts, NULL, NULL); - addDefragStage(defragModuleGlobals, NULL, NULL); + + /* Add stages for modules. */ + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (module->defrag_cb || module->defrag_cb_2) { + defragModuleCtx *ctx = zmalloc(sizeof(defragModuleCtx)); + ctx->cursor = 0; + ctx->module_name = sdsnew(module->name); + ctx->module_ctx = zcalloc(sizeof(RedisModuleDefragCtx)); + addDefragStage(defragModuleGlobals, freeDefragModelContext, ctx); + } + } + dictReleaseIterator(di); defrag.current_stage = NULL; defrag.start_cycle = getMonotonicUs(); diff --git a/src/evict.c b/src/evict.c index 059e82fe372..fe19ca17791 100644 --- a/src/evict.c +++ b/src/evict.c @@ -162,7 +162,7 @@ int evictionPoolPopulate(redisDb *db, kvstore *samplekvs, struct evictionPoolEnt idle = 255-LFUDecrAndReturn(o); } else if (server.maxmemory_policy == MAXMEMORY_VOLATILE_TTL) { /* In this case the sooner the expire the better. */ - idle = ULLONG_MAX - (long)dictGetVal(de); + idle = ULLONG_MAX - dictGetSignedIntegerVal(de); } else { serverPanic("Unknown eviction policy in evictionPoolPopulate()"); } diff --git a/src/module.c b/src/module.c index 1f4deb969c4..4fb0eba58bc 100644 --- a/src/module.c +++ b/src/module.c @@ -2309,6 +2309,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->options = 0; module->info_cb = 0; module->defrag_cb = 0; + module->defrag_cb_2 = 0; module->defrag_start_cb = 0; module->defrag_end_cb = 0; module->loadmod = NULL; @@ -11263,7 +11264,7 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * The way it should be used: * * RedisModuleScanCursor *c = RedisModule_ScanCursorCreate(); - * RedisModuleKey *key = RedisModule_OpenKey(...) + * RedisModuleKey *key = RedisModule_OpenKey(...); * while(RedisModule_ScanKey(key, c, callback, privateData)); * RedisModule_CloseKey(key); * RedisModule_ScanCursorDestroy(c); @@ -11273,13 +11274,13 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * * RedisModuleScanCursor *c = RedisModule_ScanCursorCreate(); * RedisModule_ThreadSafeContextLock(ctx); - * RedisModuleKey *key = RedisModule_OpenKey(...) + * RedisModuleKey *key = RedisModule_OpenKey(...); * while(RedisModule_ScanKey(ctx, c, callback, privateData)){ * RedisModule_CloseKey(key); * RedisModule_ThreadSafeContextUnlock(ctx); * // do some background job * RedisModule_ThreadSafeContextLock(ctx); - * RedisModuleKey *key = RedisModule_OpenKey(...) + * key = RedisModule_OpenKey(...); * } * RedisModule_CloseKey(key); * RedisModule_ScanCursorDestroy(c); @@ -13783,16 +13784,6 @@ const char *RM_GetCurrentCommandName(RedisModuleCtx *ctx) { * ## Defrag API * -------------------------------------------------------------------------- */ -/* The defrag context, used to manage state during calls to the data type - * defrag callback. - */ -struct RedisModuleDefragCtx { - monotime endtime; - unsigned long *cursor; - struct redisObject *key; /* Optional name of key processed, NULL when unknown. */ - int dbid; /* The dbid of the key being processed, -1 when unknown. */ -}; - /* Register a defrag callback for global data, i.e. anything that the module * may allocate that is not tied to a specific data type. */ @@ -13801,6 +13792,17 @@ int RM_RegisterDefragFunc(RedisModuleCtx *ctx, RedisModuleDefragFunc cb) { return REDISMODULE_OK; } +/* Register a defrag callback for global data, i.e. anything that the module + * may allocate that is not tied to a specific data type. + * This is a more advanced version of RM_RegisterDefragFunc, in that it takes + * a callbacks that has a return value, and can use RM_DefragShouldStop + * in and indicate that it should be called again later, or is it done (returned 0). + */ +int RM_RegisterDefragFunc2(RedisModuleCtx *ctx, RedisModuleDefragFunc2 cb) { + ctx->module->defrag_cb_2 = cb; + return REDISMODULE_OK; +} + /* Register a defrag callbacks that will be called when defrag operation starts and ends. * * The callbacks are the same as `RM_RegisterDefragFunc` but the user @@ -13992,16 +13994,6 @@ int moduleDefragValue(robj *key, robj *value, int dbid) { return 1; } -/* Call registered module API defrag functions */ -void moduleDefragGlobals(void) { - dictForEach(modules, struct RedisModule, module, - if (module->defrag_cb) { - RedisModuleDefragCtx defrag_ctx = { 0, NULL, NULL, -1}; - module->defrag_cb(&defrag_ctx); - } - ); -} - /* Call registered module API defrag start functions */ void moduleDefragStart(void) { dictForEach(modules, struct RedisModule, module, @@ -14381,6 +14373,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(GetCurrentCommandName); REGISTER_API(GetTypeMethodVersion); REGISTER_API(RegisterDefragFunc); + REGISTER_API(RegisterDefragFunc2); REGISTER_API(RegisterDefragCallbacks); REGISTER_API(DefragAlloc); REGISTER_API(DefragAllocRaw); diff --git a/src/object.c b/src/object.c index 6a03f26a741..45acd2e8360 100644 --- a/src/object.c +++ b/src/object.c @@ -988,7 +988,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { dict *d; dictIterator *di; struct dictEntry *de; - size_t asize = 0, elesize = 0, samples = 0; + size_t asize = 0, elesize = 0, elecount = 0, samples = 0; if (o->type == OBJ_STRING) { if(o->encoding == OBJ_ENCODING_INT) { @@ -1007,9 +1007,10 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { asize = sizeof(*o)+sizeof(quicklist); do { elesize += sizeof(quicklistNode)+zmalloc_size(node->entry); + elecount += node->count; samples++; } while ((node = node->next) && samples < sample_size); - asize += (double)elesize/samples*ql->len; + asize += (double)elesize/elecount*ql->count; } else if (o->encoding == OBJ_ENCODING_LISTPACK) { asize = sizeof(*o)+zmalloc_size(o->ptr); } else { diff --git a/src/redis-cli.c b/src/redis-cli.c index 36f3d24ff9f..3e447d6dcab 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -3118,7 +3118,7 @@ static void usage(int err) { " -i When -r is used, waits seconds per command.\n" " It is possible to specify sub-second times like -i 0.1.\n" " This interval is also used in --scan and --stat per cycle.\n" -" and in --bigkeys, --memkeys, and --hotkeys per 100 cycles.\n" +" and in --bigkeys, --memkeys, --keystats, and --hotkeys per 100 cycles.\n" " -n Database number.\n" " -2 Start session in RESP2 protocol mode.\n" " -3 Start session in RESP3 protocol mode.\n" @@ -3181,9 +3181,10 @@ version,tls_usage); " --hotkeys Sample Redis keys looking for hot keys.\n" " only works when maxmemory-policy is *lfu.\n" " --scan List all keys using the SCAN command.\n" -" --pattern Keys pattern when using the --scan, --bigkeys or --hotkeys\n" -" options (default: *).\n" -" --count Count option when using the --scan, --bigkeys or --hotkeys (default: 10).\n" +" --pattern Keys pattern when using the --scan, --bigkeys, --memkeys,\n" +" --keystats or --hotkeys options (default: *).\n" +" --count Count option when using the --scan, --bigkeys, --memkeys,\n" +" --keystats or --hotkeys (default: 10).\n" " --quoted-pattern Same as --pattern, but the specified string can be\n" " quoted, in order to pass an otherwise non binary-safe string.\n" " --intrinsic-latency Run a test to measure intrinsic system latency.\n" diff --git a/src/redismodule.h b/src/redismodule.h index 54f778315a1..f7d8cc76af5 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -840,6 +840,7 @@ typedef struct RedisModuleDefragCtx RedisModuleDefragCtx; * exposed since you can't cast a function pointer to (void *). */ typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); typedef void (*RedisModuleDefragFunc)(RedisModuleDefragCtx *ctx); +typedef int (*RedisModuleDefragFunc2)(RedisModuleDefragCtx *ctx); typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata); /* ------------------------- End of common defines ------------------------ */ @@ -1305,6 +1306,7 @@ REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, RedisMod REDISMODULE_API int *(*RedisModule_GetCommandKeysWithFlags)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys, int **out_flags) REDISMODULE_ATTR; REDISMODULE_API const char *(*RedisModule_GetCurrentCommandName)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_RegisterDefragFunc)(RedisModuleCtx *ctx, RedisModuleDefragFunc func) REDISMODULE_ATTR; +REDISMODULE_API int (*RedisModule_RegisterDefragFunc2)(RedisModuleCtx *ctx, RedisModuleDefragFunc2 func) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_RegisterDefragCallbacks)(RedisModuleCtx *ctx, RedisModuleDefragFunc start, RedisModuleDefragFunc end) REDISMODULE_ATTR; REDISMODULE_API void *(*RedisModule_DefragAlloc)(RedisModuleDefragCtx *ctx, void *ptr) REDISMODULE_ATTR; REDISMODULE_API void *(*RedisModule_DefragAllocRaw)(RedisModuleDefragCtx *ctx, size_t size) REDISMODULE_ATTR; @@ -1678,6 +1680,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(GetCommandKeysWithFlags); REDISMODULE_GET_API(GetCurrentCommandName); REDISMODULE_GET_API(RegisterDefragFunc); + REDISMODULE_GET_API(RegisterDefragFunc2); REDISMODULE_GET_API(RegisterDefragCallbacks); REDISMODULE_GET_API(DefragAlloc); REDISMODULE_GET_API(DefragAllocRaw); diff --git a/src/replication.c b/src/replication.c index c2a49951a3c..611913c9215 100644 --- a/src/replication.c +++ b/src/replication.c @@ -519,6 +519,9 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { /* We can't have slaves attached and no backlog. */ serverAssert(!(listLength(slaves) != 0 && server.repl_backlog == NULL)); + /* Update the time of sending replication stream to replicas. */ + server.repl_stream_lastio = server.unixtime; + /* Must install write handler for all replicas first before feeding * replication stream. */ prepareReplicasToWrite(); @@ -4479,8 +4482,6 @@ long long replicationGetSlaveOffset(void) { /* Replication cron function, called 1 time per second. */ void replicationCron(void) { - static long long replication_cron_loops = 0; - /* Check failover status first, to see if we need to start * handling the failover. */ updateFailoverStatus(); @@ -4533,9 +4534,12 @@ void replicationCron(void) { listNode *ln; robj *ping_argv[1]; - /* First, send PING according to ping_slave_period. */ - if ((replication_cron_loops % server.repl_ping_slave_period) == 0 && - listLength(server.slaves)) + /* First, send PING according to ping_slave_period. The reason why master + * sends PING is to keep the connection with replica active, so master need + * not send PING to replicas if already sent replication stream in the past + * repl_ping_slave_period time. */ + if (server.masterhost == NULL && listLength(server.slaves) && + server.unixtime >= server.repl_stream_lastio + server.repl_ping_slave_period) { /* Note that we don't send the PING if the clients are paused during * a Redis Cluster manual failover: the PING we send will otherwise @@ -4577,7 +4581,7 @@ void replicationCron(void) { (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END && server.rdb_child_type != RDB_CHILD_TYPE_SOCKET)); - if (is_presync) { + if (is_presync && !(slave->flags & CLIENT_CLOSE_ASAP)) { connWrite(slave->conn, "\n", 1); } } @@ -4672,7 +4676,6 @@ void replicationCron(void) { /* Refresh the number of slaves with lag <= min-slaves-max-lag. */ refreshGoodSlavesCount(); - replication_cron_loops++; /* Incremented with frequency 1 HZ. */ } int shouldStartChildReplication(int *mincapa_out, int *req_out) { diff --git a/src/server.c b/src/server.c index fd819d5b3b8..9e1250ca04b 100644 --- a/src/server.c +++ b/src/server.c @@ -2016,6 +2016,7 @@ void createSharedObjects(void) { shared.set = createStringObject("SET",3); shared.eval = createStringObject("EVAL",4); shared.hpexpireat = createStringObject("HPEXPIREAT",10); + shared.hpersist = createStringObject("HPERSIST",8); shared.hdel = createStringObject("HDEL",4); /* Shared command argument */ @@ -2174,6 +2175,7 @@ void initServerConfig(void) { server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ server.master_repl_offset = 0; server.fsynced_reploff_pending = 0; + server.repl_stream_lastio = server.unixtime; /* Replication partial resync backlog */ server.repl_backlog = NULL; @@ -4579,6 +4581,9 @@ int finishShutdown(void) { } } + /* Update the end offset of current INCR AOF if possible. */ + updateCurIncrAofEndOffset(); + /* Free the AOF manifest. */ if (server.aof_manifest) aofManifestFree(server.aof_manifest); @@ -6843,6 +6848,7 @@ void loadDataFromDisk(void) { exit(1); if (ret != AOF_NOT_EXIST) serverLog(LL_NOTICE, "DB loaded from append only file: %.3f seconds", (float)(ustime()-start)/1000000); + updateReplOffsetAndResetEndOffset(); } else { rdbSaveInfo rsi = RDB_SAVE_INFO_INIT; int rsi_is_valid = 0; diff --git a/src/server.h b/src/server.h index 052de222e74..d1339072205 100644 --- a/src/server.h +++ b/src/server.h @@ -891,6 +891,7 @@ struct RedisModule { int blocked_clients; /* Count of RedisModuleBlockedClient in this module. */ RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */ RedisModuleDefragFunc defrag_cb; /* Callback for global data defrag. */ + RedisModuleDefragFunc2 defrag_cb_2; /* Version 2 callback for global data defrag. */ RedisModuleDefragFunc defrag_start_cb; /* Callback indicating defrag started. */ RedisModuleDefragFunc defrag_end_cb; /* Callback indicating defrag ended. */ struct moduleLoadQueueEntry *loadmod; /* Module load arguments for config rewrite. */ @@ -900,6 +901,16 @@ struct RedisModule { }; typedef struct RedisModule RedisModule; +/* The defrag context, used to manage state during calls to the data type + * defrag callback. + */ +struct RedisModuleDefragCtx { + monotime endtime; + unsigned long *cursor; + struct redisObject *key; /* Optional name of key processed, NULL when unknown. */ + int dbid; /* The dbid of the key being processed, -1 when unknown. */ +}; + /* This is a wrapper for the 'rio' streams used inside rdb.c in Redis, so that * the user does not have to take the total count of the written bytes nor * to care about error conditions. */ @@ -1433,7 +1444,7 @@ struct sharedObjectsStruct { *rpop, *lpop, *lpush, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *srem, *xgroup, *xclaim, *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, - *hdel, *hpexpireat, + *hdel, *hpexpireat, *hpersist, *time, *pxat, *absttl, *retrycount, *force, *justid, *entriesread, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, *getack, *special_asterick, *special_equals, *default_username, *redacted, @@ -1619,6 +1630,8 @@ typedef struct { sds file_name; /* file name */ long long file_seq; /* file sequence */ aof_file_type file_type; /* file type */ + long long start_offset; /* the start replication offset of the file */ + long long end_offset; /* the end replication offset of the file */ } aofInfo; typedef struct { @@ -2011,6 +2024,7 @@ struct redisServer { size_t repl_buffer_mem; /* The memory of replication buffer. */ list *repl_buffer_blocks; /* Replication buffers blocks list * (serving replica clients and repl backlog) */ + time_t repl_stream_lastio; /* Unix time of the latest sending replication stream. */ /* Replication (slave) */ char *masteruser; /* AUTH with this user and masterauth with master */ sds masterauth; /* AUTH with this password with master */ @@ -2672,7 +2686,6 @@ size_t moduleGetMemUsage(robj *key, robj *val, size_t sample_size, int dbid); robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, int todb, robj *value); int moduleDefragValue(robj *key, robj *obj, int dbid); int moduleLateDefrag(robj *key, robj *value, unsigned long *cursor, monotime endtime, int dbid); -void moduleDefragGlobals(void); void moduleDefragStart(void); void moduleDefragEnd(void); void *moduleGetHandleByName(char *modulename); @@ -3057,6 +3070,8 @@ void aofOpenIfNeededOnServerStart(void); void aofManifestFree(aofManifest *am); int aofDelHistoryFiles(void); int aofRewriteLimited(void); +void updateCurIncrAofEndOffset(void); +void updateReplOffsetAndResetEndOffset(void); /* Child info */ void openChildInfoPipe(void); @@ -3356,7 +3371,9 @@ typedef struct dictExpireMetadata { #define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */ #define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields * may expire and only one notification is desired. */ -#define HFE_LAZY_ACCESS_EXPIRED (1<<3) /* Avoid lazy expire and allow access to expired fields */ +#define HFE_LAZY_NO_SIGNAL (1<<3) /* Do not send signal, used when multiple fields + * may expire and only one signal is desired. */ +#define HFE_LAZY_ACCESS_EXPIRED (1<<4) /* Avoid lazy expire and allow access to expired fields */ void hashTypeConvert(robj *o, int enc, ebuckets *hexpires); void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end); @@ -3876,6 +3893,7 @@ void strlenCommand(client *c); void zrankCommand(client *c); void zrevrankCommand(client *c); void hsetCommand(client *c); +void hsetexCommand(client *c); void hpexpireCommand(client *c); void hexpireCommand(client *c); void hpexpireatCommand(client *c); @@ -3888,6 +3906,8 @@ void hpersistCommand(client *c); void hsetnxCommand(client *c); void hgetCommand(client *c); void hmgetCommand(client *c); +void hgetexCommand(client *c); +void hgetdelCommand(client *c); void hdelCommand(client *c); void hlenCommand(client *c); void hstrlenCommand(client *c); diff --git a/src/t_hash.c b/src/t_hash.c index c6e48b77abc..b4d31bd0e62 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -48,7 +48,7 @@ typedef listpackEntry CommonEntry; /* extend usage beyond lp */ static ExpireAction onFieldExpire(eItem item, void *ctx); static ExpireMeta* hfieldGetExpireMeta(const eItem field); static ExpireMeta *hashGetExpireMeta(const eItem hash); -static void hexpireGenericCommand(client *c, const char *cmd, long long basetime, int unit); +static void hexpireGenericCommand(client *c, long long basetime, int unit); static ExpireAction hashTypeActiveExpire(eItem hashObj, void *ctx); static uint64_t hashTypeExpire(robj *o, ExpireCtx *expireCtx, int updateGlobalHFE); static void hfieldPersist(robj *hashObj, hfield field); @@ -214,15 +214,13 @@ typedef struct HashTypeSetEx { * minimum expiration time. If minimum recorded * is above minExpire of the hash, then we don't * have to update global HFE DS */ - int fieldDeleted; /* Number of fields deleted */ - int fieldUpdated; /* Number of fields updated */ /* Optionally provide client for notification */ client *c; const char *cmd; } HashTypeSetEx; -int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cmd, +int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, ExpireSetCond expireSetCond, HashTypeSetEx *ex); SetExRes hashTypeSetEx(robj *o, sds field, uint64_t expireAt, HashTypeSetEx *exInfo); @@ -531,6 +529,15 @@ SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field, prevExpire = (uint64_t) expireTime; } + /* Special value of EXPIRE_TIME_INVALID indicates field should be persisted.*/ + if (expireAt == EB_EXPIRE_TIME_INVALID) { + /* Return error if already there is no ttl. */ + if (prevExpire == EB_EXPIRE_TIME_INVALID) + return HSETEX_NO_CONDITION_MET; + listpackExUpdateExpiry(ex->hashObj, field, fptr, vptr, HASH_LP_NO_TTL); + return HSETEX_OK; + } + if (prevExpire == EB_EXPIRE_TIME_INVALID) { /* For fields without expiry, LT condition is considered valid */ if (ex->expireSetCond & (HFE_XX | HFE_GT)) @@ -551,13 +558,7 @@ SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field, if (unlikely(checkAlreadyExpired(expireAt))) { propagateHashFieldDeletion(ex->db, ex->key->ptr, field, sdslen(field)); hashTypeDelete(ex->hashObj, field, 1); - - /* get listpack length */ - listpackEx *lpt = ((listpackEx *) ex->hashObj->ptr); - unsigned long length = lpLength(lpt->lp) / 3; - updateKeysizesHist(ex->db, getKeySlot(ex->key->ptr), OBJ_HASH, length+1, length); server.stat_expired_subkeys++; - ex->fieldDeleted++; return HSETEX_DELETED; } @@ -565,7 +566,6 @@ SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field, ex->minExpireFields = expireAt; listpackExUpdateExpiry(ex->hashObj, field, fptr, vptr, expireAt); - ex->fieldUpdated++; return HSETEX_OK; } @@ -788,7 +788,8 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs dbDelete(db,keyObj); res = GETF_EXPIRED_HASH; } - signalModifiedKey(NULL, db, keyObj); + if (!(hfeFlags & HFE_LAZY_NO_SIGNAL)) + signalModifiedKey(NULL, db, keyObj); decrRefCount(keyObj); return res; } @@ -1010,34 +1011,33 @@ int hashTypeSet(redisDb *db, robj *o, sds field, sds value, int flags) { SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt) { dict *ht = exInfo->hashObj->ptr; dictEntry *existingEntry = NULL; + hfield hfNew = NULL; - /* New field with expiration metadata */ - hfield hfNew = hfieldNew(field, sdslen(field), 1 /*withExpireMeta*/); - - if ((existingEntry = dictFind(ht, field)) == NULL) { - hfieldFree(hfNew); + if ((existingEntry = dictFind(ht, field)) == NULL) return HSETEX_NO_FIELD; - } hfield hfOld = dictGetKey(existingEntry); + /* Special value of EXPIRE_TIME_INVALID indicates field should be persisted.*/ + if (expireAt == EB_EXPIRE_TIME_INVALID) { + /* Return error if already there is no ttl. */ + if (hfieldGetExpireTime(hfOld) == EB_EXPIRE_TIME_INVALID) + return HSETEX_NO_CONDITION_MET; + + hfieldPersist(exInfo->hashObj, hfOld); + return HSETEX_OK; + } /* If field doesn't have expiry metadata attached */ if (!hfieldIsExpireAttached(hfOld)) { - /* For fields without expiry, LT condition is considered valid */ - if (exInfo->expireSetCond & (HFE_XX | HFE_GT)) { - hfieldFree(hfNew); + if (exInfo->expireSetCond & (HFE_XX | HFE_GT)) return HSETEX_NO_CONDITION_MET; - } /* Delete old field. Below goanna dictSetKey(..,hfNew) */ hfieldFree(hfOld); - + /* New field with expiration metadata */ + hfNew = hfieldNew(field, sdslen(field), 1); } else { /* field has ExpireMeta struct attached */ - - /* No need for hfNew (Just modify expire-time of existing field) */ - hfieldFree(hfNew); - uint64_t prevExpire = hfieldGetExpireTime(hfOld); /* If field has valid expiration time, then check GT|LT|NX */ @@ -1073,13 +1073,10 @@ SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt /* If expired, then delete the field and propagate the deletion. * If replica, continue like the field is valid */ if (unlikely(checkAlreadyExpired(expireAt))) { - unsigned long length = dictSize(ht); - updateKeysizesHist(exInfo->db, getKeySlot(exInfo->key->ptr), OBJ_HASH, length, length-1); /* replicas should not initiate deletion of fields */ propagateHashFieldDeletion(exInfo->db, exInfo->key->ptr, field, sdslen(field)); hashTypeDelete(exInfo->hashObj, field, 1); server.stat_expired_subkeys++; - exInfo->fieldDeleted++; return HSETEX_DELETED; } @@ -1088,7 +1085,6 @@ SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt dictExpireMetadata *dm = (dictExpireMetadata *) dictMetadata(ht); ebAdd(&dm->hfe, &hashFieldExpireBucketsType, hfNew, expireAt); - exInfo->fieldUpdated++; return HSETEX_OK; } @@ -1097,20 +1093,18 @@ SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt * * Take care to call first hashTypeSetExInit() and then call this function. * Finally, call hashTypeSetExDone() to notify and update global HFE DS. + * + * Special value of EB_EXPIRE_TIME_INVALID for 'expireAt' argument will persist + * the field. */ -SetExRes hashTypeSetEx(robj *o, sds field, uint64_t expireAt, HashTypeSetEx *exInfo) -{ - if (o->encoding == OBJ_ENCODING_LISTPACK_EX) - { +SetExRes hashTypeSetEx(robj *o, sds field, uint64_t expireAt, HashTypeSetEx *exInfo) { + if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { unsigned char *fptr = NULL, *vptr = NULL, *tptr = NULL; - listpackEx *lpt = o->ptr; - long long expireTime = HASH_LP_NO_TTL; - - if ((fptr = lpFirst(lpt->lp)) == NULL) - return HSETEX_NO_FIELD; - fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2); + fptr = lpFirst(lpt->lp); + if (fptr) + fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2); if (!fptr) return HSETEX_NO_FIELD; @@ -1120,7 +1114,7 @@ SetExRes hashTypeSetEx(robj *o, sds field, uint64_t expireAt, HashTypeSetEx *exI serverAssert(vptr != NULL); tptr = lpNext(lpt->lp, vptr); - serverAssert(tptr && lpGetIntegerValue(tptr, &expireTime)); + serverAssert(tptr); /* update TTL */ return hashTypeSetExpiryListpack(exInfo, field, fptr, vptr, tptr, expireAt); @@ -1144,19 +1138,16 @@ void initDictExpireMetadata(sds key, robj *o) { } /* Init HashTypeSetEx struct before calling hashTypeSetEx() */ -int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cmd, +int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, ExpireSetCond expireSetCond, HashTypeSetEx *ex) { dict *ht = o->ptr; ex->expireSetCond = expireSetCond; ex->minExpire = EB_EXPIRE_TIME_INVALID; ex->c = c; - ex->cmd = cmd; ex->db = db; ex->key = key; ex->hashObj = o; - ex->fieldDeleted = 0; - ex->fieldUpdated = 0; ex->minExpireFields = EB_EXPIRE_TIME_INVALID; /* Take care that HASH support expiration */ @@ -1220,50 +1211,38 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm /* * After calling hashTypeSetEx() for setting fields or their expiry, call this - * function to notify and update global HFE DS. + * function to update global HFE DS. */ void hashTypeSetExDone(HashTypeSetEx *ex) { - /* Notify keyspace event, update dirty count and update global HFE DS */ - if (ex->fieldDeleted + ex->fieldUpdated > 0) { - - server.dirty += ex->fieldDeleted + ex->fieldUpdated; - if (ex->fieldDeleted && hashTypeLength(ex->hashObj, 0) == 0) { - dbDelete(ex->db,ex->key); - signalModifiedKey(ex->c, ex->db, ex->key); - notifyKeyspaceEvent(NOTIFY_HASH, "hdel", ex->key, ex->db->id); - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",ex->key, ex->db->id); - } else { - signalModifiedKey(ex->c, ex->db, ex->key); - notifyKeyspaceEvent(NOTIFY_HASH, ex->fieldDeleted ? "hdel" : "hexpire", - ex->key, ex->db->id); - - /* If minimum HFE of the hash is smaller than expiration time of the - * specified fields in the command as well as it is smaller or equal - * than expiration time provided in the command, then the minimum - * HFE of the hash won't change following this command. */ - if ((ex->minExpire < ex->minExpireFields)) - return; - /* Retrieve new expired time. It might have changed. */ - uint64_t newMinExpire = hashTypeGetMinExpire(ex->hashObj, 1 /*accurate*/); - - /* Calculate the diff between old minExpire and newMinExpire. If it is - * only few seconds, then don't have to update global HFE DS. At the worst - * case fields of hash will be active-expired up to few seconds later. - * - * In any case, active-expire operation will know to update global - * HFE DS more efficiently than here for a single item. - */ - uint64_t diff = (ex->minExpire > newMinExpire) ? - (ex->minExpire - newMinExpire) : (newMinExpire - ex->minExpire); - if (diff < HASH_NEW_EXPIRE_DIFF_THRESHOLD) return; - - if (ex->minExpire != EB_EXPIRE_TIME_INVALID) - ebRemove(&ex->db->hexpires, &hashExpireBucketsType, ex->hashObj); - if (newMinExpire != EB_EXPIRE_TIME_INVALID) - ebAdd(&ex->db->hexpires, &hashExpireBucketsType, ex->hashObj, newMinExpire); - } - } + if (hashTypeLength(ex->hashObj, 0) == 0) + return; + + /* If minimum HFE of the hash is smaller than expiration time of the + * specified fields in the command as well as it is smaller or equal + * than expiration time provided in the command, then the minimum + * HFE of the hash won't change following this command. */ + if ((ex->minExpire < ex->minExpireFields)) + return; + + /* Retrieve new expired time. It might have changed. */ + uint64_t newMinExpire = hashTypeGetMinExpire(ex->hashObj, 1 /*accurate*/); + + /* Calculate the diff between old minExpire and newMinExpire. If it is + * only few seconds, then don't have to update global HFE DS. At the worst + * case fields of hash will be active-expired up to few seconds later. + * + * In any case, active-expire operation will know to update global + * HFE DS more efficiently than here for a single item. + */ + uint64_t diff = (ex->minExpire > newMinExpire) ? + (ex->minExpire - newMinExpire) : (newMinExpire - ex->minExpire); + if (diff < HASH_NEW_EXPIRE_DIFF_THRESHOLD) return; + + if (ex->minExpire != EB_EXPIRE_TIME_INVALID) + ebRemove(&ex->db->hexpires, &hashExpireBucketsType, ex->hashObj); + if (newMinExpire != EB_EXPIRE_TIME_INVALID) + ebAdd(&ex->db->hexpires, &hashExpireBucketsType, ex->hashObj, newMinExpire); } /* Delete an element from a hash. @@ -2222,6 +2201,303 @@ void hsetCommand(client *c) { server.dirty += (c->argc - 2)/2; } +/* Parse expire time from argument and do boundary checks. */ +static int parseExpireTime(client *c, robj *o, int unit, long long basetime, + long long *expire) +{ + long long val; + + /* Read the expiry time from command */ + if (getLongLongFromObjectOrReply(c, o, &val, NULL) != C_OK) + return C_ERR; + + if (val < 0) { + addReplyError(c,"invalid expire time, must be >= 0"); + return C_ERR; + } + + if (unit == UNIT_SECONDS) { + if (val > (long long) HFE_MAX_ABS_TIME_MSEC / 1000) { + addReplyErrorExpireTime(c); + return C_ERR; + } + val *= 1000; + } + + if (val > (long long) HFE_MAX_ABS_TIME_MSEC - basetime) { + addReplyErrorExpireTime(c); + return C_ERR; + } + val += basetime; + *expire = val; + return C_OK; +} + +/* Flags that are used as part of HGETEX and HSETEX commands. */ +#define HFE_EX (1<<0) /* Expiration time in seconds */ +#define HFE_PX (1<<1) /* Expiration time in milliseconds */ +#define HFE_EXAT (1<<2) /* Expiration time in unix seconds */ +#define HFE_PXAT (1<<3) /* Expiration time in unix milliseconds */ +#define HFE_PERSIST (1<<4) /* Persist fields */ +#define HFE_KEEPTTL (1<<5) /* Do not discard field ttl on set op */ +#define HFE_FXX (1<<6) /* Set fields if all the fields already exist */ +#define HFE_FNX (1<<7) /* Set fields if none of the fields exist */ + +/* Parse hsetex command arguments. + * HSETEX + * [FNX|FXX] + * [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL] + * FIELDS field value [field value ...] +*/ +static int hsetexParseArgs(client *c, int *flags, + long long *expire_time, int *expire_time_pos, + int *first_field_pos, int *field_count) { + *flags = 0; + *first_field_pos = -1; + *field_count = -1; + *expire_time_pos = -1; + + for (int i = 2; i < c->argc; i++) { + if (!strcasecmp(c->argv[i]->ptr, "fields")) { + long val; + + if (i >= c->argc - 3) { + addReplyErrorArity(c); + return C_ERR; + } + + if (getRangeLongFromObjectOrReply(c, c->argv[i + 1], 1, INT_MAX, &val, + "invalid number of fields") != C_OK) + return C_ERR; + + int remaining = (c->argc - i - 2); + if (remaining % 2 != 0 || val != remaining / 2) { + addReplyErrorArity(c); + return C_ERR; + } + + *first_field_pos = i + 2; + *field_count = (int) val; + return C_OK; + } else if (!strcasecmp(c->argv[i]->ptr, "EX")) { + if (*flags & (HFE_EX | HFE_EXAT | HFE_PX | HFE_PXAT | HFE_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_EX; + i++; + + if (parseExpireTime(c, c->argv[i], UNIT_SECONDS, + commandTimeSnapshot(), expire_time) != C_OK) + return C_ERR; + + *expire_time_pos = i; + } else if (!strcasecmp(c->argv[i]->ptr, "PX")) { + if (*flags & (HFE_EX | HFE_EXAT | HFE_PX | HFE_PXAT | HFE_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_PX; + i++; + if (parseExpireTime(c, c->argv[i], UNIT_MILLISECONDS, + commandTimeSnapshot(), expire_time) != C_OK) + return C_ERR; + + *expire_time_pos = i; + } else if (!strcasecmp(c->argv[i]->ptr, "EXAT")) { + if (*flags & (HFE_EX | HFE_EXAT | HFE_PX | HFE_PXAT | HFE_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_EXAT; + i++; + if (parseExpireTime(c, c->argv[i], UNIT_SECONDS, 0, expire_time) != C_OK) + return C_ERR; + + *expire_time_pos = i; + } else if (!strcasecmp(c->argv[i]->ptr, "PXAT")) { + if (*flags & (HFE_EX | HFE_EXAT | HFE_PX | HFE_PXAT | HFE_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_PXAT; + i++; + if (parseExpireTime(c, c->argv[i], UNIT_MILLISECONDS, 0, + expire_time) != C_OK) + return C_ERR; + + *expire_time_pos = i; + } else if (!strcasecmp(c->argv[i]->ptr, "KEEPTTL")) { + if (*flags & (HFE_EX | HFE_EXAT | HFE_PX | HFE_PXAT | HFE_KEEPTTL)) + goto err_expiration; + *flags |= HFE_KEEPTTL; + } else if (!strcasecmp(c->argv[i]->ptr, "FXX")) { + if (*flags & (HFE_FXX | HFE_FNX)) + goto err_condition; + *flags |= HFE_FXX; + } else if (!strcasecmp(c->argv[i]->ptr, "FNX")) { + if (*flags & (HFE_FXX | HFE_FNX)) + goto err_condition; + *flags |= HFE_FNX; + } else { + addReplyErrorFormat(c, "unknown argument: %s", (char*) c->argv[i]->ptr); + return C_ERR; + } + } + + serverAssert(0); + +err_missing_expire: + addReplyError(c, "missing expire time"); + return C_ERR; +err_condition: + addReplyError(c, "Only one of FXX or FNX arguments can be specified"); + return C_ERR; +err_expiration: + addReplyError(c, "Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments can be specified"); + return C_ERR; +} + +/* Set the value of one or more fields of a given hash key, and optionally set + * their expiration. + * + * HSETEX key + * [FNX | FXX] + * [EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT unix-time-milliseconds | KEEPTTL] + * FIELDS field value [field value...] + * + * Reply: + * Integer reply: 0 if no fields were set (due to FXX/FNX args) + * Integer reply: 1 if all the fields were set + */ +void hsetexCommand(client *c) { + int flags = 0, first_field_pos = 0, field_count = 0, expire_time_pos = -1; + int updated = 0, deleted = 0, set_expiry; + long long expire_time = EB_EXPIRE_TIME_INVALID; + unsigned long oldlen, newlen; + robj *o; + HashTypeSetEx setex; + + if (hsetexParseArgs(c, &flags, &expire_time, &expire_time_pos, + &first_field_pos, &field_count) != C_OK) + return; + + o = lookupKeyWrite(c->db, c->argv[1]); + if (checkType(c, o, OBJ_HASH)) + return; + + if (!o) { + if (flags & HFE_FXX) { + addReplyLongLong(c, 0); + return; + } + o = createHashObject(); + dbAdd(c->db, c->argv[1], o); + } + oldlen = hashTypeLength(o, 0); + + if (flags & (HFE_FXX | HFE_FNX)) { + int found = 0; + for (int i = 0; i < field_count; i++) { + sds field = c->argv[first_field_pos + (i * 2)]->ptr; + const int opt = HFE_LAZY_NO_NOTIFICATION | + HFE_LAZY_NO_SIGNAL | + HFE_LAZY_AVOID_HASH_DEL; + int exists = hashTypeExists(c->db, o, field, opt, NULL); + found += (exists != 0); + + /* Check for early exit if the condition is already invalid. */ + if (((flags & HFE_FXX) && !exists) || + ((flags & HFE_FNX) && exists)) + break; + } + + int all_exists = (found == field_count); + int non_exists = (found == 0); + + if (((flags & HFE_FNX) && !non_exists) || + ((flags & HFE_FXX) && !all_exists)) + { + addReplyLongLong(c, 0); + goto out; + } + } + hashTypeTryConversion(c->db, o,c->argv, first_field_pos, c->argc - 1); + + /* Check if we will set the expiration time. */ + set_expiry = flags & (HFE_EX | HFE_PX | HFE_EXAT | HFE_PXAT); + if (set_expiry) + hashTypeSetExInit(c->argv[1], o, c, c->db, 0, &setex); + + + for (int i = 0; i < field_count; i++) { + sds field = c->argv[first_field_pos + (i * 2)]->ptr; + sds value = c->argv[first_field_pos + (i * 2) + 1]->ptr; + + int opt = HASH_SET_COPY; + /* If we are going to set the expiration time later, no need to discard + * it as part of set operation now. */ + if (flags & (HFE_EX | HFE_PX | HFE_EXAT | HFE_PXAT | HFE_KEEPTTL)) + opt |= HASH_SET_KEEP_TTL; + + hashTypeSet(c->db, o, field, value, opt); + + /* Update the expiration time. */ + if (set_expiry) { + int ret = hashTypeSetEx(o, field, expire_time, &setex); + updated += (ret == HSETEX_OK); + deleted += (ret == HSETEX_DELETED); + } + } + + if (set_expiry) + hashTypeSetExDone(&setex); + + server.dirty += field_count; + signalModifiedKey(c, c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); + if (deleted || updated) + notifyKeyspaceEvent(NOTIFY_HASH, deleted ? "hdel": "hexpire", + c->argv[1], c->db->id); + + if (deleted) { + /* If fields are deleted due to timestamp is being in the past, hdel's + * are already propagated. No need to propagate the command itself. */ + preventCommandPropagation(c); + } else if (set_expiry && !(flags & HFE_PXAT)) { + /* Propagate as 'HSETEX PXAT ..' if there is EX/EXAT/PX flag*/ + + /* Replace EX/EXAT/PX with PXAT */ + rewriteClientCommandArgument(c, expire_time_pos - 1, shared.pxat); + /* Replace timestamp with unix timestamp milliseconds. */ + robj *expire = createStringObjectFromLongLong(expire_time); + rewriteClientCommandArgument(c, expire_time_pos, expire); + decrRefCount(expire); + } + + addReplyLongLong(c, 1); + +out: + /* Key may become empty due to lazy expiry in hashTypeExists() + * or the new expiration time is in the past.*/ + newlen = hashTypeLength(o, 0); + if (newlen == 0) { + dbDelete(c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id); + } + if (oldlen != newlen) + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, + oldlen, newlen); +} + void hincrbyCommand(client *c) { long long value, incr, oldvalue; robj *o; @@ -2393,6 +2669,254 @@ void hmgetCommand(client *c) { } } +/* Get and delete the value of one or more fields of a given hash key. + * HGETDEL FIELDS field1 field2 ... + * Reply: list of the value associated with each field or nil if the field + * doesn’t exist. + */ +void hgetdelCommand(client *c) { + int res = 0, hfe = 0, deleted = 0, expired = 0; + unsigned long oldlen = 0, newlen= 0; + long num_fields = 0; + robj *o; + + o = lookupKeyWrite(c->db, c->argv[1]); + if (checkType(c, o, OBJ_HASH)) + return; + + if (strcasecmp(c->argv[2]->ptr, "FIELDS") != 0) { + addReplyError(c, "Mandatory argument FIELDS is missing or not at the right position"); + return; + } + + /* Read number of fields */ + if (getRangeLongFromObjectOrReply(c, c->argv[3], 1, LONG_MAX, &num_fields, + "Number of fields must be a positive integer") != C_OK) + return; + + /* Verify `numFields` is consistent with number of arguments */ + if (num_fields != c->argc - 4) { + addReplyError(c, "The `numfields` parameter must match the number of arguments"); + return; + } + + /* Hash field expiration is optimized to avoid frequent update global HFE DS + * for each field deletion. Eventually active-expiration will run and update + * or remove the hash from global HFE DS gracefully. Nevertheless, statistic + * "subexpiry" might reflect wrong number of hashes with HFE to the user if + * it is the last field with expiration. The following logic checks if this + * is the last field with expiration and removes it from global HFE DS. */ + if (o) { + hfe = hashTypeIsFieldsWithExpire(o); + oldlen = hashTypeLength(o, 0); + } + + addReplyArrayLen(c, num_fields); + for (int i = 4; i < c->argc; i++) { + const int flags = HFE_LAZY_NO_NOTIFICATION | + HFE_LAZY_NO_SIGNAL | + HFE_LAZY_AVOID_HASH_DEL; + res = addHashFieldToReply(c, o, c->argv[i]->ptr, flags); + expired += (res == GETF_EXPIRED); + /* Try to delete only if it's found and not expired lazily. */ + if (res == GETF_OK) { + deleted++; + serverAssert(hashTypeDelete(o, c->argv[i]->ptr, 1) == 1); + } + } + + /* Return if no modification has been made. */ + if (expired == 0 && deleted == 0) + return; + + signalModifiedKey(c, c->db, c->argv[1]); + + if (expired) + notifyKeyspaceEvent(NOTIFY_HASH, "hexpired", c->argv[1], c->db->id); + if (deleted) { + notifyKeyspaceEvent(NOTIFY_HASH, "hdel", c->argv[1], c->db->id); + server.dirty += deleted; + + /* Propagate as HDEL command. + * Orig: HGETDEL FIELDS field1 field2 ... + * Repl: HDEL field1 field2 ... */ + rewriteClientCommandArgument(c, 0, shared.hdel); + rewriteClientCommandArgument(c, 2, NULL); /* Delete FIELDS arg */ + rewriteClientCommandArgument(c, 2, NULL); /* Delete arg */ + } + + /* Key may have become empty because of deleting fields or lazy expire. */ + newlen = hashTypeLength(o, 0); + if (newlen == 0) { + dbDelete(c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id); + } else if (hfe && (hashTypeIsFieldsWithExpire(o) == 0)) { /*is it last HFE*/ + ebRemove(&c->db->hexpires, &hashExpireBucketsType, o); + } + + if (oldlen != newlen) + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, + oldlen, newlen); +} + +/* Get and delete the value of one or more fields of a given hash key. + * + * HGETEX + * [EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT unix-time-milliseconds | PERSIST] + * FIELDS field1 field2 ... + * + * Reply: list of the value associated with each field or nil if the field + * doesn’t exist. + */ +void hgetexCommand(client *c) { + int expired = 0, deleted = 0, updated = 0; + int num_fields_pos = 3, cond = 0; + long num_fields; + unsigned long oldlen = 0, newlen = 0; + long long expire_time = 0; + robj *o; + HashTypeSetEx setex; + + o = lookupKeyWrite(c->db, c->argv[1]); + if (checkType(c, o, OBJ_HASH)) + return; + + /* Read optional arg */ + if (!strcasecmp(c->argv[2]->ptr, "ex")) + cond = HFE_EX; + else if (!strcasecmp(c->argv[2]->ptr, "px")) + cond = HFE_PX; + else if (!strcasecmp(c->argv[2]->ptr, "exat")) + cond = HFE_EXAT; + else if (!strcasecmp(c->argv[2]->ptr, "pxat")) + cond = HFE_PXAT; + else if (!strcasecmp(c->argv[2]->ptr, "persist")) + cond = HFE_PERSIST; + + /* Parse expiration time */ + if (cond & (HFE_EX | HFE_PX | HFE_EXAT | HFE_PXAT)) { + num_fields_pos += 2; + int unit = (cond & (HFE_EX | HFE_EXAT)) ? UNIT_SECONDS : UNIT_MILLISECONDS; + long long basetime = cond & (HFE_EX | HFE_PX) ? commandTimeSnapshot() : 0; + if (parseExpireTime(c, c->argv[3], unit, basetime, &expire_time) != C_OK) + return; + } else if (cond & HFE_PERSIST) { + num_fields_pos += 1; + } + + if (strcasecmp(c->argv[num_fields_pos - 1]->ptr, "FIELDS") != 0) { + addReplyError(c, "Mandatory argument FIELDS is missing or not at the right position"); + return; + } + + /* Read number of fields */ + if (getRangeLongFromObjectOrReply(c, c->argv[num_fields_pos], 1, LONG_MAX, &num_fields, + "Number of fields must be a positive integer") != C_OK) + return; + + /* Check number of fields is consistent with number of arguments */ + if (num_fields != c->argc - num_fields_pos - 1) { + addReplyError(c, "The `numfields` parameter must match the number of arguments"); + return; + } + + /* Non-existing keys and empty hashes are the same thing. Reply null if the + * key does not exist.*/ + if (!o) { + addReplyArrayLen(c, num_fields); + for (int i = 0; i < num_fields; i++) + addReplyNull(c); + return; + } + + oldlen = hashTypeLength(o, 0); + if (cond) + hashTypeSetExInit(c->argv[1], o, c, c->db, 0, &setex); + + addReplyArrayLen(c, num_fields); + for (int i = num_fields_pos + 1; i < c->argc; i++) { + const int flags = HFE_LAZY_NO_NOTIFICATION | + HFE_LAZY_NO_SIGNAL | + HFE_LAZY_AVOID_HASH_DEL; + sds field = c->argv[i]->ptr; + int res = addHashFieldToReply(c, o, field, flags); + expired += (res == GETF_EXPIRED); + + /* Set expiration only if the field exists and not expired lazily. */ + if (res == GETF_OK && cond) { + if (cond & HFE_PERSIST) + expire_time = EB_EXPIRE_TIME_INVALID; + + res = hashTypeSetEx(o, field, expire_time, &setex); + deleted += (res == HSETEX_DELETED); + updated += (res == HSETEX_OK); + } + } + + if (cond) + hashTypeSetExDone(&setex); + + /* Exit early if no modification has been made. */ + if (expired == 0 && deleted == 0 && updated == 0) + return; + + server.dirty += deleted + updated; + signalModifiedKey(c, c->db, c->argv[1]); + + /* Key may become empty due to lazy expiry in addHashFieldToReply() + * or the new expiration time is in the past.*/ + newlen = hashTypeLength(o, 0); + if (newlen == 0) { + dbDelete(c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id); + } + if (oldlen != newlen) + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, + oldlen, newlen); + + /* This command will never be propagated as it is. It will be propagated as + * HDELs when fields are lazily expired or deleted, if the new timestamp is + * in the past. HDEL's will be emitted as part of addHashFieldToReply() + * or hashTypeSetEx() in this case. + * + * If PERSIST flags is used, it will be propagated as HPERSIST command. + * IF EX/EXAT/PX/PXAT flags are used, it will be replicated as HPEXPRITEAT. + */ + if (expired) + notifyKeyspaceEvent(NOTIFY_HASH, "hexpired", c->argv[1], c->db->id); + if (updated) { + if (cond & HFE_PERSIST) { + notifyKeyspaceEvent(NOTIFY_HASH, "hpersist", c->argv[1], c->db->id); + + /* Propagate as HPERSIST command. + * Orig: HGETEX PERSIST FIELDS field1 field2 ... + * Repl: HPERSIST FIELDS field1 field2 ... */ + rewriteClientCommandArgument(c, 0, shared.hpersist); + rewriteClientCommandArgument(c, 2, NULL); /* Delete PERSIST arg */ + } else { + notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", c->argv[1], c->db->id); + + /* Propagate as HPEXPIREAT command. + * Orig: HGETEX [EX|PX|EXAT|PXAT] ttl FIELDS field1 field2 ... + * Repl: HPEXPIREAT ttl FIELDS field1 field2 ... */ + rewriteClientCommandArgument(c, 0, shared.hpexpireat); + rewriteClientCommandArgument(c, 2, NULL); /* Del [EX|PX|EXAT|PXAT]*/ + + /* Rewrite TTL if it is not unix time milliseconds already. */ + if (!(cond & HFE_PXAT)) { + robj *expire = createStringObjectFromLongLong(expire_time); + rewriteClientCommandArgument(c, 2, expire); + decrRefCount(expire); + } + } + } else if (deleted) { + /* If we are here, fields are deleted because new timestamp was in the + * past. HDELs are already propagated as part of hashTypeSetEx(). */ + notifyKeyspaceEvent(NOTIFY_HASH, "hdel", c->argv[1], c->db->id); + preventCommandPropagation(c); + } +} + void hdelCommand(client *c) { robj *o; int j, deleted = 0, keyremoved = 0; @@ -3174,10 +3698,11 @@ static void httlGenericCommand(client *c, const char *cmd, long long basetime, i * not met, then command will be rejected. Otherwise, EXPIRE command will be * propagated for given key. */ -static void hexpireGenericCommand(client *c, const char *cmd, long long basetime, int unit) { +static void hexpireGenericCommand(client *c, long long basetime, int unit) { long numFields = 0, numFieldsAt = 4; long long expire; /* unix time in msec */ - int fieldAt, fieldsNotSet = 0, expireSetCond = 0; + int fieldAt, fieldsNotSet = 0, expireSetCond = 0, updated = 0, deleted = 0; + unsigned long oldlen, newlen; robj *hashObj, *keyArg = c->argv[1], *expireArg = c->argv[2]; /* Read the hash object */ @@ -3186,28 +3711,8 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime return; /* Read the expiry time from command */ - if (getLongLongFromObjectOrReply(c, expireArg, &expire, NULL) != C_OK) - return; - - if (expire < 0) { - addReplyError(c,"invalid expire time, must be >= 0"); - return; - } - - if (unit == UNIT_SECONDS) { - if (expire > (long long) HFE_MAX_ABS_TIME_MSEC / 1000) { - addReplyErrorExpireTime(c); - return; - } - expire *= 1000; - } - - /* Ensure that the final absolute Unix timestamp does not exceed EB_EXPIRE_TIME_MAX. */ - if (expire > (long long) HFE_MAX_ABS_TIME_MSEC - basetime) { - addReplyErrorExpireTime(c); + if (parseExpireTime(c, expireArg, unit, basetime, &expire) != C_OK) return; - } - expire += basetime; /* Read optional expireSetCond [NX|XX|GT|LT] */ char *optArg = c->argv[3]->ptr; @@ -3247,14 +3752,18 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime return; } + oldlen = hashTypeLength(hashObj, 0); + HashTypeSetEx exCtx; - hashTypeSetExInit(keyArg, hashObj, c, c->db, cmd, expireSetCond, &exCtx); + hashTypeSetExInit(keyArg, hashObj, c, c->db, expireSetCond, &exCtx); addReplyArrayLen(c, numFields); fieldAt = numFieldsAt + 1; while (fieldAt < c->argc) { sds field = c->argv[fieldAt]->ptr; SetExRes res = hashTypeSetEx(hashObj, field, expire, &exCtx); + updated += (res == HSETEX_OK); + deleted += (res == HSETEX_DELETED); if (unlikely(res != HSETEX_OK)) { /* If the field was not set, prevent field propagation */ @@ -3269,17 +3778,34 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime hashTypeSetExDone(&exCtx); + if (deleted + updated > 0) { + server.dirty += deleted + updated; + signalModifiedKey(c, c->db, keyArg); + notifyKeyspaceEvent(NOTIFY_HASH, deleted ? "hdel" : "hexpire", + keyArg, c->db->id); + } + + newlen = hashTypeLength(hashObj, 0); + if (newlen == 0) { + dbDelete(c->db, keyArg); + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyArg, c->db->id); + } + + if (oldlen != newlen) + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, + oldlen, newlen); + /* Avoid propagating command if not even one field was updated (Either because * the time is in the past, and corresponding HDELs were sent, or conditions * not met) then it is useless and invalid to propagate command with no fields */ - if (exCtx.fieldUpdated == 0) { + if (updated == 0) { preventCommandPropagation(c); return; } /* If some fields were dropped, rewrite the number of fields */ if (fieldsNotSet) { - robj *numFieldsObj = createStringObjectFromLongLong(exCtx.fieldUpdated); + robj *numFieldsObj = createStringObjectFromLongLong(updated); rewriteClientCommandArgument(c, numFieldsAt, numFieldsObj); decrRefCount(numFieldsObj); } @@ -3297,48 +3823,48 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime } } -/* HPEXPIRE key milliseconds [ NX | XX | GT | LT] numfields */ +/* HPEXPIRE key milliseconds [ NX | XX | GT | LT] FIELDS numfields */ void hpexpireCommand(client *c) { - hexpireGenericCommand(c,"hpexpire", commandTimeSnapshot(),UNIT_MILLISECONDS); + hexpireGenericCommand(c,commandTimeSnapshot(),UNIT_MILLISECONDS); } -/* HEXPIRE key seconds [NX | XX | GT | LT] numfields */ +/* HEXPIRE key seconds [NX | XX | GT | LT] FIELDS numfields */ void hexpireCommand(client *c) { - hexpireGenericCommand(c,"hexpire", commandTimeSnapshot(),UNIT_SECONDS); + hexpireGenericCommand(c,commandTimeSnapshot(),UNIT_SECONDS); } -/* HEXPIREAT key unix-time-seconds [NX | XX | GT | LT] numfields */ +/* HEXPIREAT key unix-time-seconds [NX | XX | GT | LT] FIELDS numfields */ void hexpireatCommand(client *c) { - hexpireGenericCommand(c,"hexpireat", 0,UNIT_SECONDS); + hexpireGenericCommand(c,0,UNIT_SECONDS); } -/* HPEXPIREAT key unix-time-milliseconds [NX | XX | GT | LT] numfields */ +/* HPEXPIREAT key unix-time-milliseconds [NX | XX | GT | LT] FIELDS numfields */ void hpexpireatCommand(client *c) { - hexpireGenericCommand(c,"hpexpireat", 0,UNIT_MILLISECONDS); + hexpireGenericCommand(c,0,UNIT_MILLISECONDS); } /* for each specified field: get the remaining time to live in seconds*/ -/* HTTL key numfields */ +/* HTTL key FIELDS numfields */ void httlCommand(client *c) { httlGenericCommand(c, "httl", commandTimeSnapshot(), UNIT_SECONDS); } -/* HPTTL key numfields */ +/* HPTTL key FIELDS numfields */ void hpttlCommand(client *c) { httlGenericCommand(c, "hpttl", commandTimeSnapshot(), UNIT_MILLISECONDS); } -/* HEXPIRETIME key numFields */ +/* HEXPIRETIME key FIELDS numfields */ void hexpiretimeCommand(client *c) { httlGenericCommand(c, "hexpiretime", 0, UNIT_SECONDS); } -/* HPEXPIRETIME key numFields */ +/* HPEXPIRETIME key FIELDS numfields */ void hpexpiretimeCommand(client *c) { httlGenericCommand(c, "hexpiretime", 0, UNIT_MILLISECONDS); } -/* HPERSIST key */ +/* HPERSIST key FIELDS numfields */ void hpersistCommand(client *c) { robj *hashObj; long numFields = 0, numFieldsAt = 3; diff --git a/tests/integration/aof-multi-part.tcl b/tests/integration/aof-multi-part.tcl index bdd03823398..5a0025070a5 100644 --- a/tests/integration/aof-multi-part.tcl +++ b/tests/integration/aof-multi-part.tcl @@ -1329,4 +1329,210 @@ tags {"external:skip"} { } } } + + # Test Part 3 + # + # Test if INCR AOF offset information is as expected + test {Multi Part AOF writes start offset in the manifest} { + set aof_dirpath "$server_path/$aof_dirname" + set aof_manifest_file "$server_path/$aof_dirname/${aof_basename}$::manifest_suffix" + + start_server_aof [list dir $server_path] { + set client [redis [srv host] [srv port] 0 $::tls] + wait_done_loading $client + + # The manifest file has startoffset now + assert_aof_manifest_content $aof_manifest_file { + {file appendonly.aof.1.base.rdb seq 1 type b} + {file appendonly.aof.1.incr.aof seq 1 type i startoffset 0} + } + } + + clean_aof_persistence $aof_dirpath + } + + test {Multi Part AOF won't add the offset of incr AOF from old version} { + create_aof $aof_dirpath $aof_base1_file { + append_to_aof [formatCommand set k1 v1] + } + + create_aof $aof_dirpath $aof_incr1_file { + append_to_aof [formatCommand set k2 v2] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + start_server_aof [list dir $server_path] { + assert_equal 1 [is_alive [srv pid]] + set client [redis [srv host] [srv port] 0 $::tls] + wait_done_loading $client + + assert_equal v1 [$client get k1] + assert_equal v2 [$client get k2] + + $client set k3 v3 + catch {$client shutdown} + + # Should not add offset to the manifest since we also don't know the right + # starting replication of them. + set fp [open $aof_manifest_file r] + set content [read $fp] + close $fp + assert ![regexp {startoffset} $content] + + # The manifest file still have information from the old version + assert_aof_manifest_content $aof_manifest_file { + {file appendonly.aof.1.base.aof seq 1 type b} + {file appendonly.aof.1.incr.aof seq 1 type i} + } + } + + clean_aof_persistence $aof_dirpath + } + + test {Multi Part AOF can update master_repl_offset with only startoffset info} { + create_aof $aof_dirpath $aof_base1_file { + append_to_aof [formatCommand set k1 v1] + } + + create_aof $aof_dirpath $aof_incr1_file { + append_to_aof [formatCommand set k2 v2] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i startoffset 100\n" + } + + start_server [list overrides [list dir $server_path appendonly yes ]] { + wait_done_loading r + r select 0 + assert_equal v1 [r get k1] + assert_equal v2 [r get k2] + + # After loading AOF, redis will update the replication offset based on + # the information of the last INCR AOF, to avoid the rollback of the + # start offset of new INCR AOF. If the INCR file doesn't have an end offset + # info, redis will calculate the replication offset by the start offset + # plus the file size. + set file_size [file size $aof_incr1_file] + set offset [expr $file_size + 100] + assert_equal $offset [s master_repl_offset] + } + + clean_aof_persistence $aof_dirpath + } + + test {Multi Part AOF can update master_repl_offset with endoffset info} { + create_aof $aof_dirpath $aof_base1_file { + append_to_aof [formatCommand set k1 v1] + } + + create_aof $aof_dirpath $aof_incr1_file { + append_to_aof [formatCommand set k2 v2] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i startoffset 100 endoffset 200\n" + } + + start_server [list overrides [list dir $server_path appendonly yes ]] { + wait_done_loading r + r select 0 + assert_equal v1 [r get k1] + assert_equal v2 [r get k2] + + # If the INCR file has an end offset, redis directly uses it as replication offset + assert_equal 200 [s master_repl_offset] + + # We should reset endoffset in manifest file + set fp [open $aof_manifest_file r] + set content [read $fp] + close $fp + assert ![regexp {endoffset} $content] + } + + clean_aof_persistence $aof_dirpath + } + + test {Multi Part AOF will add the end offset if we close gracefully the AOF} { + start_server_aof [list dir $server_path] { + set client [redis [srv host] [srv port] 0 $::tls] + wait_done_loading $client + + assert_aof_manifest_content $aof_manifest_file { + {file appendonly.aof.1.base.rdb seq 1 type b} + {file appendonly.aof.1.incr.aof seq 1 type i startoffset 0} + } + + $client set k1 v1 + $client set k2 v2 + # Close AOF gracefully when stopping appendonly, we should add endoffset + # in the manifest file, 'endoffset' should be 2 since writing 2 commands + r config set appendonly no + assert_aof_manifest_content $aof_manifest_file { + {file appendonly.aof.1.base.rdb seq 1 type b} + {file appendonly.aof.1.incr.aof seq 1 type i startoffset 0 endoffset 2} + } + r config set appendonly yes + waitForBgrewriteaof $client + + $client set k3 v3 + # Close AOF gracefully when shutting down server, we should add endoffset + # in the manifest file, 'endoffset' should be 3 since writing 3 commands + catch {$client shutdown} + assert_aof_manifest_content $aof_manifest_file { + {file appendonly.aof.2.base.rdb seq 2 type b} + {file appendonly.aof.2.incr.aof seq 2 type i startoffset 2 endoffset 3} + } + } + + clean_aof_persistence $aof_dirpath + } + + test {INCR AOF has accurate start offset when AOFRW} { + start_server [list overrides [list dir $server_path appendonly yes ]] { + r config set auto-aof-rewrite-percentage 0 + + # Start write load to let the master_repl_offset continue increasing + # since appendonly is enabled + set load_handle0 [start_write_load [srv 0 host] [srv 0 port] 10] + wait_for_condition 50 100 { + [r dbsize] > 0 + } else { + fail "No write load detected." + } + + # We obtain the master_repl_offset at the time of bgrewriteaof by pausing + # the redis process, sending pipeline commands, and then resuming the process + set rd [redis_deferring_client] + pause_process [srv 0 pid] + set buf "info replication\r\n" + append buf "bgrewriteaof\r\n" + $rd write $buf + $rd flush + resume_process [srv 0 pid] + # Read the replication offset and the start of the bgrewriteaof + regexp {master_repl_offset:(\d+)} [$rd read] -> offset1 + assert_match {*rewriting started*} [$rd read] + $rd close + + # Get the start offset from the manifest file after bgrewriteaof + waitForBgrewriteaof r + set fp [open $aof_manifest_file r] + set content [read $fp] + close $fp + set offset2 [lindex [regexp -inline {startoffset (\d+)} $content] 1] + + # The start offset of INCR AOF should be the same as master_repl_offset + # when we trigger bgrewriteaof + assert {$offset1 == $offset2} + stop_write_load $load_handle0 + wait_load_handlers_disconnected + } + } } diff --git a/tests/modules/defragtest.c b/tests/modules/defragtest.c index 597b5aa79f3..d436e56b07f 100644 --- a/tests/modules/defragtest.c +++ b/tests/modules/defragtest.c @@ -21,34 +21,45 @@ unsigned long int datatype_defragged = 0; unsigned long int datatype_raw_defragged = 0; unsigned long int datatype_resumes = 0; unsigned long int datatype_wrong_cursor = 0; -unsigned long int global_attempts = 0; unsigned long int defrag_started = 0; unsigned long int defrag_ended = 0; -unsigned long int global_defragged = 0; +unsigned long int global_strings_attempts = 0; +unsigned long int global_strings_defragged = 0; +unsigned long int global_strings_pauses = 0; -int global_strings_len = 0; +unsigned long global_strings_len = 0; RedisModuleString **global_strings = NULL; -static void createGlobalStrings(RedisModuleCtx *ctx, int count) +static void createGlobalStrings(RedisModuleCtx *ctx, unsigned long count) { global_strings_len = count; global_strings = RedisModule_Alloc(sizeof(RedisModuleString *) * count); - for (int i = 0; i < count; i++) { + for (unsigned long i = 0; i < count; i++) { global_strings[i] = RedisModule_CreateStringFromLongLong(ctx, i); } } -static void defragGlobalStrings(RedisModuleDefragCtx *ctx) +static int defragGlobalStrings(RedisModuleDefragCtx *ctx) { - for (int i = 0; i < global_strings_len; i++) { - RedisModuleString *new = RedisModule_DefragRedisModuleString(ctx, global_strings[i]); - global_attempts++; + unsigned long cursor = 0; + RedisModule_DefragCursorGet(ctx, &cursor); + RedisModule_Assert(cursor < global_strings_len); + for (; cursor < global_strings_len; cursor++) { + RedisModuleString *new = RedisModule_DefragRedisModuleString(ctx, global_strings[cursor]); + global_strings_attempts++; if (new != NULL) { - global_strings[i] = new; - global_defragged++; + global_strings[cursor] = new; + global_strings_defragged++; + } + + if (RedisModule_DefragShouldStop(ctx)) { + global_strings_pauses++; + RedisModule_DefragCursorSet(ctx, cursor); + return 1; } } + return 0; } static void defragStart(RedisModuleDefragCtx *ctx) { @@ -70,8 +81,9 @@ static void FragInfo(RedisModuleInfoCtx *ctx, int for_crash_report) { RedisModule_InfoAddFieldLongLong(ctx, "datatype_raw_defragged", datatype_raw_defragged); RedisModule_InfoAddFieldLongLong(ctx, "datatype_resumes", datatype_resumes); RedisModule_InfoAddFieldLongLong(ctx, "datatype_wrong_cursor", datatype_wrong_cursor); - RedisModule_InfoAddFieldLongLong(ctx, "global_attempts", global_attempts); - RedisModule_InfoAddFieldLongLong(ctx, "global_defragged", global_defragged); + RedisModule_InfoAddFieldLongLong(ctx, "global_strings_attempts", global_strings_attempts); + RedisModule_InfoAddFieldLongLong(ctx, "global_strings_defragged", global_strings_defragged); + RedisModule_InfoAddFieldLongLong(ctx, "global_strings_pauses", global_strings_pauses); RedisModule_InfoAddFieldLongLong(ctx, "defrag_started", defrag_started); RedisModule_InfoAddFieldLongLong(ctx, "defrag_ended", defrag_ended); } @@ -99,8 +111,9 @@ static int fragResetStatsCommand(RedisModuleCtx *ctx, RedisModuleString **argv, datatype_raw_defragged = 0; datatype_resumes = 0; datatype_wrong_cursor = 0; - global_attempts = 0; - global_defragged = 0; + global_strings_attempts = 0; + global_strings_defragged = 0; + global_strings_pauses = 0; defrag_started = 0; defrag_ended = 0; @@ -258,7 +271,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_ERR; RedisModule_RegisterInfoFunc(ctx, FragInfo); - RedisModule_RegisterDefragFunc(ctx, defragGlobalStrings); + RedisModule_RegisterDefragFunc2(ctx, defragGlobalStrings); RedisModule_RegisterDefragCallbacks(ctx, defragStart, defragEnd); return REDISMODULE_OK; diff --git a/tests/support/aofmanifest.tcl b/tests/support/aofmanifest.tcl index 151626294fd..68eed037b5a 100644 --- a/tests/support/aofmanifest.tcl +++ b/tests/support/aofmanifest.tcl @@ -122,7 +122,7 @@ proc assert_aof_manifest_content {manifest_path content} { assert_equal [llength $lines] [llength $content] for { set i 0 } { $i < [llength $lines] } {incr i} { - assert_equal [lindex $lines $i] [lindex $content $i] + assert {[string first [lindex $content $i] [lindex $lines $i]] != -1} } } diff --git a/tests/unit/info-keysizes.tcl b/tests/unit/info-keysizes.tcl index 98d6d4e6fb0..a866efcfd67 100644 --- a/tests/unit/info-keysizes.tcl +++ b/tests/unit/info-keysizes.tcl @@ -334,6 +334,31 @@ proc test_all_keysizes { {replMode 0} } { run_cmd_verify_hist {$server HSET h2 2 2} {db0_HASH:2=1} run_cmd_verify_hist {$server HDEL h2 1} {db0_HASH:1=1} run_cmd_verify_hist {$server HDEL h2 2} {} + # HGETDEL + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h2 FIELDS 1 1 1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETEX h2 FIELDS 1 2 2} {db0_HASH:2=1} + run_cmd_verify_hist {$server HGETDEL h2 FIELDS 1 1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HGETDEL h2 FIELDS 1 3} {db0_HASH:1=1} + run_cmd_verify_hist {$server HGETDEL h2 FIELDS 1 2} {} + # HGETEX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 FIELDS 2 f1 1 f2 1} {db0_HASH:2=1} + run_cmd_verify_hist {$server HGETEX h1 PXAT 1 FIELDS 1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETEX h1 FIELDS 1 f3 1} {db0_HASH:2=1} + run_cmd_verify_hist {$server HGETEX h1 PX 50 FIELDS 1 f2} {db0_HASH:2=1} + run_cmd_verify_hist {} {db0_HASH:1=1} 1 + run_cmd_verify_hist {$server HGETEX h1 PX 50 FIELDS 1 f3} {db0_HASH:1=1} + run_cmd_verify_hist {} {} 1 + # HSETEX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 FIELDS 2 f1 1 f2 1} {db0_HASH:2=1} + run_cmd_verify_hist {$server HSETEX h1 PXAT 1 FIELDS 1 f1 v1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETEX h1 FIELDS 1 f3 1} {db0_HASH:2=1} + run_cmd_verify_hist {$server HSETEX h1 PX 50 FIELDS 1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {} {db0_HASH:1=1} 1 + run_cmd_verify_hist {$server HSETEX h1 PX 50 FIELDS 1 f3 v3} {db0_HASH:1=1} + run_cmd_verify_hist {} {} 1 # HMSET run_cmd_verify_hist {$server FLUSHALL} {} run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3} {db0_HASH:2=1} diff --git a/tests/unit/moduleapi/defrag.tcl b/tests/unit/moduleapi/defrag.tcl index 3f36ee1912b..9e7efb673cb 100644 --- a/tests/unit/moduleapi/defrag.tcl +++ b/tests/unit/moduleapi/defrag.tcl @@ -1,7 +1,7 @@ set testmodule [file normalize tests/modules/defragtest.so] start_server {tags {"modules"} overrides {{save ""}}} { - r module load $testmodule 10000 + r module load $testmodule 50000 r config set hz 100 r config set active-defrag-ignore-bytes 1 r config set active-defrag-threshold-lower 0 @@ -46,7 +46,8 @@ start_server {tags {"modules"} overrides {{save ""}}} { after 2000 set info [r info defragtest_stats] - assert {[getInfoProperty $info defragtest_global_attempts] > 0} + assert {[getInfoProperty $info defragtest_global_strings_attempts] > 0} + assert {[getInfoProperty $info defragtest_global_strings_pauses] > 0} assert_morethan [getInfoProperty $info defragtest_defrag_started] 0 assert_morethan [getInfoProperty $info defragtest_defrag_ended] 0 } diff --git a/tests/unit/pubsub.tcl b/tests/unit/pubsub.tcl index 9a4f1196b90..def27190810 100644 --- a/tests/unit/pubsub.tcl +++ b/tests/unit/pubsub.tcl @@ -414,6 +414,58 @@ start_server {tags {"pubsub network"}} { assert_equal "pmessage * __keyspace@${db}__:myhash del" [$rd1 read] r debug set-active-expire 1 + + # Test HSETEX, HGETEX and HGETDEL notifications + r hsetex myhash FIELDS 3 f4 v4 f5 v5 f6 v6 + assert_equal "pmessage * __keyspace@${db}__:myhash hset" [$rd1 read] + + # hgetex sets ttl in past + r hgetex myhash PX 0 FIELDS 1 f4 + assert_equal "pmessage * __keyspace@${db}__:myhash hdel" [$rd1 read] + + # hgetex sets ttl + r hgetex myhash EXAT [expr {[clock seconds] + 999999}] FIELDS 1 f5 + assert_equal "pmessage * __keyspace@${db}__:myhash hexpire" [$rd1 read] + + # hgetex persists field + r hgetex myhash PERSIST FIELDS 1 f5 + assert_equal "pmessage * __keyspace@${db}__:myhash hpersist" [$rd1 read] + + # hgetdel deletes a field + r hgetdel myhash FIELDS 1 f5 + assert_equal "pmessage * __keyspace@${db}__:myhash hdel" [$rd1 read] + + # hsetex sets field and expiry time + r hsetex myhash EXAT [expr {[clock seconds] + 999999}] FIELDS 1 f6 v6 + assert_equal "pmessage * __keyspace@${db}__:myhash hset" [$rd1 read] + assert_equal "pmessage * __keyspace@${db}__:myhash hexpire" [$rd1 read] + + # hsetex sets field and ttl in the past + r hsetex myhash PX 0 FIELDS 1 f6 v6 + assert_equal "pmessage * __keyspace@${db}__:myhash hset" [$rd1 read] + assert_equal "pmessage * __keyspace@${db}__:myhash hdel" [$rd1 read] + assert_equal "pmessage * __keyspace@${db}__:myhash del" [$rd1 read] + + # Test that we will get `hexpired` notification when a hash field is + # removed by lazy expire using hgetdel command + r debug set-active-expire 0 + r hsetex myhash PX 10 FIELDS 1 f1 v1 + assert_equal "pmessage * __keyspace@${db}__:myhash hset" [$rd1 read] + assert_equal "pmessage * __keyspace@${db}__:myhash hexpire" [$rd1 read] + + # Set another field + r hsetex myhash FIELDS 1 f2 v2 + assert_equal "pmessage * __keyspace@${db}__:myhash hset" [$rd1 read] + # Wait until field expires + after 20 + r hgetdel myhash FIELDS 1 f1 + assert_equal "pmessage * __keyspace@${db}__:myhash hexpired" [$rd1 read] + # Get and delete the only field + r hgetdel myhash FIELDS 1 f2 + assert_equal "pmessage * __keyspace@${db}__:myhash hdel" [$rd1 read] + assert_equal "pmessage * __keyspace@${db}__:myhash del" [$rd1 read] + r debug set-active-expire 1 + $rd1 close } {0} {needs:debug} } ;# foreach diff --git a/tests/unit/type/hash-field-expire.tcl b/tests/unit/type/hash-field-expire.tcl index 0d1840091c6..02cac6008b0 100644 --- a/tests/unit/type/hash-field-expire.tcl +++ b/tests/unit/type/hash-field-expire.tcl @@ -855,6 +855,430 @@ start_server {tags {"external:skip needs:debug"}} { assert_equal [r HINCRBYFLOAT h1 f1 2.5] 12.5 assert_range [r HPTTL h1 FIELDS 1 f1] 1 20 } + + test "HGETDEL - delete field with ttl ($type)" { + r debug set-active-expire 0 + r del h1 + + # Test deleting only field in a hash. Due to lazy expiry, + # reply will be null and the field and the key will be deleted. + r hsetex h1 PX 5 FIELDS 1 f1 10 + after 15 + assert_equal [r hgetdel h1 fields 1 f1] "{}" + assert_equal [r exists h1] 0 + + # Test deleting one field among many. f2 will lazily expire + r hsetex h1 FIELDS 3 f1 10 f2 20 f3 value3 + r hpexpire h1 5 FIELDS 1 f2 + after 15 + assert_equal [r hgetdel h1 fields 2 f2 f3] "{} value3" + assert_equal [lsort [r hgetall h1]] [lsort "f1 10"] + + # Try to delete the last field, along with non-existing fields + assert_equal [r hgetdel h1 fields 4 f1 f2 f3 f4] "10 {} {} {}" + r debug set-active-expire 1 + } + + test "HGETEX - input validation ($type)" { + r del h1 + assert_error "*wrong number of arguments*" {r HGETEX} + assert_error "*wrong number of arguments*" {r HGETEX h1} + assert_error "*wrong number of arguments*" {r HGETEX h1 FIELDS} + assert_error "*wrong number of arguments*" {r HGETEX h1 FIELDS 0} + assert_error "*wrong number of arguments*" {r HGETEX h1 FIELDS 1} + assert_error "*argument FIELDS is missing*" {r HGETEX h1 XFIELDX 1 a} + assert_error "*argument FIELDS is missing*" {r HGETEX h1 PXAT 1 1} + assert_error "*argument FIELDS is missing*" {r HGETEX h1 PERSIST 1 FIELDS 1 a} + assert_error "*must match the number of arguments*" {r HGETEX h1 FIELDS 2 a} + assert_error "*Number of fields must be a positive integer*" {r HGETEX h1 FIELDS 0 a} + assert_error "*Number of fields must be a positive integer*" {r HGETEX h1 FIELDS -1 a} + assert_error "*Number of fields must be a positive integer*" {r HGETEX h1 FIELDS 9223372036854775808 a} + } + + test "HGETEX - input validation (expire time) ($type)" { + assert_error "*value is not an integer or out of range*" {r HGETEX h1 EX bla FIELDS 1 a} + assert_error "*value is not an integer or out of range*" {r HGETEX h1 EX 9223372036854775808 FIELDS 1 a} + assert_error "*value is not an integer or out of range*" {r HGETEX h1 EXAT 9223372036854775808 FIELDS 1 a} + assert_error "*invalid expire time, must be >= 0*" {r HGETEX h1 PX -1 FIELDS 1 a} + assert_error "*invalid expire time, must be >= 0*" {r HGETEX h1 PXAT -1 FIELDS 1 a} + assert_error "*invalid expire time*" {r HGETEX h1 EX -1 FIELDS 1 a} + assert_error "*invalid expire time*" {r HGETEX h1 EX [expr (1<<48)] FIELDS 1 a} + assert_error "*invalid expire time*" {r HGETEX h1 EX [expr (1<<46) - [clock seconds] + 100 ] FIELDS 1 a} + assert_error "*invalid expire time*" {r HGETEX h1 EXAT [expr (1<<46) + 100 ] FIELDS 1 a} + assert_error "*invalid expire time*" {r HGETEX h1 PX [expr (1<<46) - [clock milliseconds] + 100 ] FIELDS 1 a} + assert_error "*invalid expire time*" {r HGETEX h1 PXAT [expr (1<<46) + 100 ] FIELDS 1 a} + } + + test "HGETEX - get without setting ttl ($type)" { + r del h1 + r hset h1 a 1 b 2 c strval + assert_equal [r hgetex h1 fields 1 a] "1" + assert_equal [r hgetex h1 fields 2 a b] "1 2" + assert_equal [r hgetex h1 fields 3 a b c] "1 2 strval" + assert_equal [r HTTL h1 FIELDS 3 a b c] "$T_NO_EXPIRY $T_NO_EXPIRY $T_NO_EXPIRY" + } + + test "HGETEX - get and set the ttl ($type)" { + r del h1 + r hset h1 a 1 b 2 c strval + assert_equal [r hgetex h1 EX 10000 fields 1 a] "1" + assert_range [r HTTL h1 FIELDS 1 a] 9000 10000 + assert_equal [r hgetex h1 EX 10000 fields 1 c] "strval" + assert_range [r HTTL h1 FIELDS 1 c] 9000 10000 + } + + test "HGETEX - Test 'EX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetex myhash EX 1000 FIELDS 1 field1] [list "value1"] + assert_range [r httl myhash FIELDS 1 field1] 1 1000 + } + + test "HGETEX - Test 'EXAT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetex myhash EXAT 4000000000 FIELDS 1 field2] [list "value2"] + assert_range [expr [r httl myhash FIELDS 1 field2] + [clock seconds]] 3900000000 4000000000 + } + + test "HGETEX - Test 'PX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetex myhash PX 1000000 FIELDS 1 field3] [list "value3"] + assert_range [r httl myhash FIELDS 1 field3] 900 1000 + } + + test "HGETEX - Test 'PXAT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetex myhash PXAT 4000000000000 FIELDS 1 field3] [list "value3"] + assert_range [expr [r httl myhash FIELDS 1 field3] + [clock seconds]] 3900000000 4000000000 + } + + test "HGETEX - Test 'PERSIST' flag ($type)" { + r del myhash + r debug set-active-expire 0 + + r hsetex myhash PX 5000 FIELDS 3 f1 v1 f2 v2 f3 v3 + assert_not_equal [r httl myhash FIELDS 1 f1] "$T_NO_EXPIRY" + assert_not_equal [r httl myhash FIELDS 1 f2] "$T_NO_EXPIRY" + assert_not_equal [r httl myhash FIELDS 1 f3] "$T_NO_EXPIRY" + + # Persist f1 and verify it does not have TTL anymore + assert_equal [r hgetex myhash PERSIST FIELDS 1 f1] "v1" + assert_equal [r httl myhash FIELDS 1 f1] "$T_NO_EXPIRY" + + # Persist rest of the fields + assert_equal [r hgetex myhash PERSIST FIELDS 2 f2 f3] "v2 v3" + assert_equal [r httl myhash FIELDS 2 f2 f3] "$T_NO_EXPIRY $T_NO_EXPIRY" + + # Redo the operation. It should be noop as fields are persisted already. + assert_equal [r hgetex myhash PERSIST FIELDS 2 f2 f3] "v2 v3" + assert_equal [r httl myhash FIELDS 2 f2 f3] "$T_NO_EXPIRY $T_NO_EXPIRY" + + # Final sanity, fields exist and have no attached ttl. + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2 f3 v3"] + assert_equal [r httl myhash FIELDS 3 f1 f2 f3] "$T_NO_EXPIRY $T_NO_EXPIRY $T_NO_EXPIRY" + r debug set-active-expire 1 + } + + test "HGETEX - Test setting ttl in the past will delete the key ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + + # hgetex without setting ttl + assert_equal [lsort [r hgetex myhash fields 3 f1 f2 f3]] [lsort "v1 v2 v3"] + assert_equal [r httl myhash FIELDS 3 f1 f2 f3] "$T_NO_EXPIRY $T_NO_EXPIRY $T_NO_EXPIRY" + + # set an expired ttl and verify the key is deleted + r hgetex myhash PXAT 1 fields 3 f1 f2 f3 + assert_equal [r exists myhash] 0 + } + + test "HGETEX - Test active expiry ($type)" { + r del myhash + r debug set-active-expire 0 + + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + assert_equal [lsort [r hgetex myhash PXAT 1 FIELDS 5 f1 f2 f3 f4 f5]] [lsort "v1 v2 v3 v4 v5"] + + r debug set-active-expire 1 + wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" } + } + + test "HGETEX - A field with TTL overridden with another value (TTL discarded) ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + r hgetex myhash PX 10000 FIELDS 1 f1 + r hgetex myhash EX 100 FIELDS 1 f2 + + # f2 ttl will be discarded + r hset myhash f2 v22 + assert_equal [r hget myhash f2] "v22" + assert_equal [r httl myhash FIELDS 2 f2 f3] "$T_NO_EXPIRY $T_NO_EXPIRY" + + # Other field is not affected (still has TTL) + assert_not_equal [r httl myhash FIELDS 1 f1] "$T_NO_EXPIRY" + } + + test "HGETEX - Test with lazy expiry ($type)" { + r del myhash + r debug set-active-expire 0 + + r hsetex myhash PX 1 FIELDS 2 f1 v1 f2 v2 + after 5 + assert_equal [r hgetex myhash FIELDS 2 f1 f2] "{} {}" + assert_equal [r exists myhash] 0 + + r debug set-active-expire 1 + } + + test "HSETEX - input validation ($type)" { + assert_error {*wrong number of arguments*} {r hsetex myhash} + assert_error {*wrong number of arguments*} {r hsetex myhash fields} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 1} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 2 a b} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 2 a b c} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 2 a b c d e} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 3 a b c d} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 3 a b c d e} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 3 a b c d e f g} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 3 a b} + assert_error {*wrong number of arguments*} {r hsetex myhash fields 1 a b unknown} + assert_error {*unknown argument*} {r hsetex myhash nx fields 1 a b} + assert_error {*unknown argument*} {r hsetex myhash 1 fields 1 a b} + + # Only one of FNX or FXX + assert_error {*Only one of FXX or FNX arguments *} {r hsetex myhash fxx fxx EX 100 fields 1 a b} + assert_error {*Only one of FXX or FNX arguments *} {r hsetex myhash fxx fnx EX 100 fields 1 a b} + assert_error {*Only one of FXX or FNX arguments *} {r hsetex myhash fnx fxx EX 100 fields 1 a b} + assert_error {*Only one of FXX or FNX arguments *} {r hsetex myhash fnx fnx EX 100 fields 1 a b} + assert_error {*Only one of FXX or FNX arguments *} {r hsetex myhash fxx fnx fxx EX 100 fields 1 a b} + assert_error {*Only one of FXX or FNX arguments *} {r hsetex myhash fnx fxx fnx EX 100 fields 1 a b} + + # Only one of EX, PX, EXAT, PXAT or KEEPTTL can be specified + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash EX 100 PX 1000 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash EX 100 EXAT 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash EXAT 100 EX 1000 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash EXAT 100 PX 1000 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash PX 100 EXAT 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash PX 100 PXAT 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash PXAT 100 EX 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash PXAT 100 EXAT 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash EX 100 KEEPTTL fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash KEEPTTL EX 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash EX 100 EX 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash EXAT 100 EXAT 100 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash PX 10 PX 10 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash PXAT 10 PXAT 10 fields 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetex myhash KEEPTTL KEEPTTL fields 1 a b} + + # missing expire time + assert_error {*not an integer or out of range*} {r hsetex myhash ex fields 1 a b} + assert_error {*not an integer or out of range*} {r hsetex myhash px fields 1 a b} + assert_error {*not an integer or out of range*} {r hsetex myhash exat fields 1 a b} + assert_error {*not an integer or out of range*} {r hsetex myhash pxat fields 1 a b} + + # expire time more than 2 ^ 48 + assert_error {*invalid expire time*} {r hsetex myhash EXAT [expr (1<<48)] 1 a b} + assert_error {*invalid expire time*} {r hsetex myhash PXAT [expr (1<<48)] 1 a b} + assert_error {*invalid expire time*} {r hsetex myhash EX [expr (1<<48) - [clock seconds] + 1000 ] 1 a b} + assert_error {*invalid expire time*} {r hsetex myhash PX [expr (1<<48) - [clock milliseconds] + 1000 ] 1 a b} + + # invalid expire time + assert_error {*invalid expire time*} {r hsetex myhash EXAT -1 1 a b} + assert_error {*not an integer or out of range*} {r hsetex myhash EXAT 9223372036854775808 1 a b} + assert_error {*not an integer or out of range*} {r hsetex myhash EXAT x 1 a b} + + # invalid numfields arg + assert_error {*invalid number of fields*} {r hsetex myhash fields x a b} + assert_error {*invalid number of fields*} {r hsetex myhash fields 9223372036854775808 a b} + assert_error {*invalid number of fields*} {r hsetex myhash fields 0 a b} + assert_error {*invalid number of fields*} {r hsetex myhash fields -1 a b} + } + + test "HSETEX - Basic test ($type)" { + r del myhash + + # set field + assert_equal [r hsetex myhash FIELDS 1 f1 v1] 1 + assert_equal [r hget myhash f1] "v1" + + # override + assert_equal [r hsetex myhash FIELDS 1 f1 v11] 1 + assert_equal [r hget myhash f1] "v11" + + # set multiple + assert_equal [r hsetex myhash FIELDS 2 f1 v1 f2 v2] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2"] + assert_equal [r hsetex myhash FIELDS 3 f1 v111 f2 v222 f3 v333] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v111 f2 v222 f3 v333"] + } + + test "HSETEX - Test FXX flag ($type)" { + r del myhash + + # Key is empty, command fails due to FXX + assert_equal [r hsetex myhash FXX FIELDS 2 f1 v1 f2 v2] 0 + # Verify it did not leave the key empty + assert_equal [r exists myhash] 0 + + # Command fails and no change on fields + r hset myhash f1 v1 + assert_equal [r hsetex myhash FXX FIELDS 2 f1 v1 f2 v2] 0 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1"] + + # Command executed successfully + assert_equal [r hsetex myhash FXX FIELDS 1 f1 v11] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v11"] + + # Try with multiple fields + r hset myhash f2 v2 + assert_equal [r hsetex myhash FXX FIELDS 2 f1 v111 f2 v222] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v111 f2 v222"] + + # Try with expiry + assert_equal [r hsetex myhash FXX EX 100 FIELDS 2 f1 v1 f2 v2] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2"] + assert_range [r httl myhash FIELDS 1 f1] 80 100 + assert_range [r httl myhash FIELDS 1 f2] 80 100 + + # Try with expiry, FXX arg comes after TTL + assert_equal [r hsetex myhash PX 5000 FXX FIELDS 2 f1 v1 f2 v2] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2"] + assert_range [r hpttl myhash FIELDS 1 f1] 4500 5000 + assert_range [r hpttl myhash FIELDS 1 f2] 4500 5000 + } + + test "HSETEX - Test FXX flag with lazy expire ($type)" { + r del myhash + r debug set-active-expire 0 + + r hsetex myhash PX 10 FIELDS 1 f1 v1 + after 15 + assert_equal [r hsetex myhash FXX FIELDS 1 f1 v11] 0 + assert_equal [r exists myhash] 0 + r debug set-active-expire 1 + } + + test "HSETEX - Test FNX flag ($type)" { + r del myhash + + # Command successful on an empty key + assert_equal [r hsetex myhash FNX FIELDS 1 f1 v1] 1 + + # Command fails and no change on fields + assert_equal [r hsetex myhash FNX FIELDS 2 f1 v1 f2 v2] 0 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1"] + + # Command executed successfully + assert_equal [r hsetex myhash FNX FIELDS 2 f2 v2 f3 v3] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2 f3 v3"] + assert_equal [r hsetex myhash FXX FIELDS 3 f1 v11 f2 v22 f3 v33] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v11 f2 v22 f3 v33"] + + # Try with expiry + r del myhash + assert_equal [r hsetex myhash FNX EX 100 FIELDS 2 f1 v1 f2 v2] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2"] + assert_range [r httl myhash FIELDS 1 f1] 80 100 + assert_range [r httl myhash FIELDS 1 f2] 80 100 + + # Try with expiry, FNX arg comes after TTL + assert_equal [r hsetex myhash PX 5000 FNX FIELDS 1 f3 v3] 1 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2 f3 v3"] + assert_range [r hpttl myhash FIELDS 1 f3] 4500 5000 + } + + test "HSETEX - Test 'EX' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 + assert_equal [r hsetex myhash EX 1000 FIELDS 1 f3 v3 ] 1 + assert_range [r httl myhash FIELDS 1 f3] 900 1000 + } + + test "HSETEX - Test 'EXAT' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 + assert_equal [r hsetex myhash EXAT 4000000000 FIELDS 1 f3 v3] 1 + assert_range [expr [r httl myhash FIELDS 1 f3] + [clock seconds]] 3900000000 4000000000 + } + + test "HSETEX - Test 'PX' flag ($type)" { + r del myhash + assert_equal [r hsetex myhash PX 1000000 FIELDS 1 f3 v3] 1 + assert_range [r httl myhash FIELDS 1 f3] 990 1000 + } + + test "HSETEX - Test 'PXAT' flag ($type)" { + r del myhash + r hset myhash f1 v2 f2 v2 f3 v3 + assert_equal [r hsetex myhash PXAT 4000000000000 FIELDS 1 f2 v2] 1 + assert_range [expr [r httl myhash FIELDS 1 f2] + [clock seconds]] 3900000000 4000000000 + } + + test "HSETEX - Test 'KEEPTTL' flag ($type)" { + r del myhash + + r hsetex myhash FIELDS 2 f1 v1 f2 v2 + r hsetex myhash PX 20000 FIELDS 1 f2 v2 + + # f1 does not have ttl + assert_equal [r httl myhash FIELDS 1 f1] "$T_NO_EXPIRY" + + # f2 has ttl + assert_not_equal [r httl myhash FIELDS 1 f2] "$T_NO_EXPIRY" + + # Validate KEEPTTL preserves the TTL + assert_equal [r hsetex myhash KEEPTTL FIELDS 1 f2 v22] 1 + assert_equal [r hget myhash f2] "v22" + assert_not_equal [r httl myhash FIELDS 1 f2] "$T_NO_EXPIRY" + + # Try with multiple fields. First, set fields and TTL + r hsetex myhash EX 10000 FIELDS 3 f1 v1 f2 v2 f3 v3 + + # Update fields with KEEPTTL flag + r hsetex myhash KEEPTTL FIELDS 3 f1 v111 f2 v222 f3 v333 + + # Verify values are set, ttls are untouched + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v111 f2 v222 f3 v333"] + assert_range [r httl myhash FIELDS 1 f1] 9000 10000 + assert_range [r httl myhash FIELDS 1 f2] 9000 10000 + assert_range [r httl myhash FIELDS 1 f3] 9000 10000 + } + + test "HSETEX - Test no expiry flag discards TTL ($type)" { + r del myhash + + r hsetex myhash FIELDS 1 f1 v1 + r hsetex myhash PX 100000 FIELDS 1 f2 v2 + assert_range [r hpttl myhash FIELDS 1 f2] 1 100000 + + assert_equal [r hsetex myhash FIELDS 2 f1 v1 f2 v2] 1 + assert_equal [r httl myhash FIELDS 2 f1 f2] "$T_NO_EXPIRY $T_NO_EXPIRY" + } + + test "HSETEX - Test with active expiry" { + r del myhash + r debug set-active-expire 0 + + r hsetex myhash PX 10 FIELDS 5 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r debug set-active-expire 1 + wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" } + } + + test "HSETEX - Set time in the past ($type)" { + r del myhash + + # Try on an empty key + assert_equal [r hsetex myhash EXAT [expr {[clock seconds] - 1}] FIELDS 2 f1 v1 f2 v2] 1 + assert_equal [r hexists myhash field1] 0 + + # Try with existing fields + r hset myhash fields 2 f1 v1 f2 v2 + assert_equal [r hsetex myhash EXAT [expr {[clock seconds] - 1}] FIELDS 2 f1 v1 f2 v2] 1 + assert_equal [r hexists myhash field1] 0 + } } test "Statistics - Hashes with HFEs ($type)" { @@ -879,6 +1303,13 @@ start_server {tags {"external:skip needs:debug"}} { r hdel myhash3 f2 assert_match [get_stat_subexpiry r] 2 + # hash4: 2 fields, 1 with TTL. HGETDEL field with TTL. subexpiry decr -1 + r hset myhash4 f1 v1 f2 v2 + r hpexpire myhash4 100 FIELDS 1 f2 + assert_match [get_stat_subexpiry r] 3 + r hgetdel myhash4 FIELDS 1 f2 + assert_match [get_stat_subexpiry r] 2 + # Expired fields of hash1 and hash2. subexpiry decr -2 wait_for_condition 50 50 { [get_stat_subexpiry r] == 0 @@ -887,6 +1318,21 @@ start_server {tags {"external:skip needs:debug"}} { } } + test "HFE commands against wrong type" { + r set wrongtype somevalue + assert_error "WRONGTYPE Operation against a key*" {r hexpire wrongtype 10 fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hexpireat wrongtype 10 fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hpexpire wrongtype 10 fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hpexpireat wrongtype 10 fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hexpiretime wrongtype fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hpexpiretime wrongtype fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r httl wrongtype fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hpttl wrongtype fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hpersist wrongtype fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hgetex wrongtype fields 1 f1} + assert_error "WRONGTYPE Operation against a key*" {r hsetex wrongtype fields 1 f1 v1} + } + r config set hash-max-listpack-entries 512 } @@ -1048,6 +1494,54 @@ start_server {tags {"external:skip needs:debug"}} { fail "Field f2 of hash h2 wasn't deleted" } + # HSETEX + r hsetex h3 FIELDS 1 f1 v1 + r hsetex h3 FXX FIELDS 1 f1 v11 + r hsetex h3 FNX FIELDS 1 f2 v22 + r hsetex h3 KEEPTTL FIELDS 1 f2 v22 + + # Next one will fail due to FNX arg and it won't be replicated + r hsetex h3 FNX FIELDS 2 f1 v1 f2 v2 + + # Commands with EX/PX/PXAT/EXAT will be replicated as PXAT + r hsetex h3 EX 10000 FIELDS 1 f1 v111 + r hsetex h3 PX 10000 FIELDS 1 f1 v111 + r hsetex h3 PXAT [expr [clock milliseconds]+100000] FIELDS 1 f1 v111 + r hsetex h3 EXAT [expr [clock seconds]+100000] FIELDS 1 f1 v111 + + # Following commands will set and then delete the fields because + # of TTL in the past. HDELs will be propagated. + r hsetex h3 PX 0 FIELDS 1 f1 v111 + r hsetex h3 PX 0 FIELDS 3 f1 v2 f2 v2 f3 v3 + + # HGETEX + r hsetex h4 FIELDS 3 f1 v1 f2 v2 f3 v3 + # No change on expiry, it won't be replicated. + r hgetex h4 FIELDS 1 f1 + + # Commands with EX/PX/PXAT/EXAT will be replicated as + # HPEXPIREAT command. + r hgetex h4 EX 10000 FIELDS 1 f1 + r hgetex h4 PX 10000 FIELDS 1 f1 + r hgetex h4 PXAT [expr [clock milliseconds]+100000] FIELDS 1 f1 + r hgetex h4 EXAT [expr [clock seconds]+100000] FIELDS 1 f1 + + # Following commands will delete the fields because of TTL in + # the past. HDELs will be propagated. + r hgetex h4 PX 0 FIELDS 1 f1 + # HDELs will be propagated for f2 and f3 as only those exist. + r hgetex h4 PX 0 FIELDS 3 f1 f2 f3 + + # HGETEX with PERSIST flag will be replicated as HPERSIST + r hsetex h4 EX 1000 FIELDS 1 f4 v4 + r hgetex h4 PERSIST FIELDS 1 f4 + + # Nothing will be replicated as f4 is persisted already. + r hgetex h4 PERSIST FIELDS 1 f4 + + # Replicated as hdel + r hgetdel h4 FIELDS 1 f4 + # Assert that each TTL-related command are persisted with absolute timestamps in AOF assert_aof_content $aof { {select *} @@ -1068,6 +1562,33 @@ start_server {tags {"external:skip needs:debug"}} { {hdel h1 f2} {hdel h2 f1} {hdel h2 f2} + {hsetex h3 FIELDS 1 f1 v1} + {hsetex h3 FXX FIELDS 1 f1 v11} + {hsetex h3 FNX FIELDS 1 f2 v22} + {hsetex h3 KEEPTTL FIELDS 1 f2 v22} + {hsetex h3 PXAT * 1 f1 v111} + {hsetex h3 PXAT * 1 f1 v111} + {hsetex h3 PXAT * 1 f1 v111} + {hsetex h3 PXAT * 1 f1 v111} + {hdel h3 f1} + {multi} + {hdel h3 f1} + {hdel h3 f2} + {hdel h3 f3} + {exec} + {hsetex h4 FIELDS 3 f1 v1 f2 v2 f3 v3} + {hpexpireat h4 * FIELDS 1 f1} + {hpexpireat h4 * FIELDS 1 f1} + {hpexpireat h4 * FIELDS 1 f1} + {hpexpireat h4 * FIELDS 1 f1} + {hdel h4 f1} + {multi} + {hdel h4 f2} + {hdel h4 f3} + {exec} + {hsetex h4 PXAT * FIELDS 1 f4 v4} + {hpersist h4 FIELDS 1 f4} + {hdel h4 f4} } } } {} {needs:debug} @@ -1135,6 +1656,16 @@ start_server {tags {"external:skip needs:debug"}} { r hpexpire h2 1 FIELDS 2 f1 f2 after 200 + r hsetex h3 EX 100000 FIELDS 2 f1 v1 f2 v2 + r hsetex h3 EXAT [expr [clock seconds] + 1000] FIELDS 2 f1 v1 f2 v2 + r hsetex h3 PX 100000 FIELDS 2 f1 v1 f2 v2 + r hsetex h3 PXAT [expr [clock milliseconds]+100000] FIELDS 2 f1 v1 f2 v2 + + r hgetex h3 EX 100000 FIELDS 2 f1 f2 + r hgetex h3 EXAT [expr [clock seconds] + 1000] FIELDS 2 f1 f2 + r hgetex h3 PX 100000 FIELDS 2 f1 f2 + r hgetex h3 PXAT [expr [clock milliseconds]+100000] FIELDS 2 f1 f2 + assert_aof_content $aof { {select *} {hset h1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6} @@ -1146,6 +1677,14 @@ start_server {tags {"external:skip needs:debug"}} { {hpexpireat h2 * FIELDS 2 f1 f2} {hdel h2 *} {hdel h2 *} + {hsetex h3 PXAT * FIELDS 2 f1 v1 f2 v2} + {hsetex h3 PXAT * FIELDS 2 f1 v1 f2 v2} + {hsetex h3 PXAT * FIELDS 2 f1 v1 f2 v2} + {hsetex h3 PXAT * FIELDS 2 f1 v1 f2 v2} + {hpexpireat h3 * FIELDS 2 f1 f2} + {hpexpireat h3 * FIELDS 2 f1 f2} + {hpexpireat h3 * FIELDS 2 f1 f2} + {hpexpireat h3 * FIELDS 2 f1 f2} } array set keyAndFields1 [dumpAllHashes r] @@ -1265,6 +1804,23 @@ start_server {tags {"external:skip needs:debug"}} { $primary hpexpireat h5 [expr [clock milliseconds]-100000] FIELDS 1 f $primary hset h9 f v + $primary hsetex h10 EX 100000 FIELDS 1 f v + $primary hsetex h11 EXAT [expr [clock seconds] + 1000] FIELDS 1 f v + $primary hsetex h12 PX 100000 FIELDS 1 f v + $primary hsetex h13 PXAT [expr [clock milliseconds]+100000] FIELDS 1 f v + $primary hsetex h14 PXAT 1 FIELDS 1 f v + + $primary hsetex h15 FIELDS 1 f v + $primary hgetex h15 EX 100000 FIELDS 1 f + $primary hsetex h16 FIELDS 1 f v + $primary hgetex h16 EXAT [expr [clock seconds] + 1000] FIELDS 1 f + $primary hsetex h17 FIELDS 1 f v + $primary hgetex h17 PX 100000 FIELDS 1 f + $primary hsetex h18 FIELDS 1 f v + $primary hgetex h18 PXAT [expr [clock milliseconds]+100000] FIELDS 1 f + $primary hsetex h19 FIELDS 1 f v + $primary hgetex h19 PXAT 1 FIELDS 1 f + # Wait for replica to get the keys and TTLs assert {[$primary wait 1 0] == 1} @@ -1273,5 +1829,102 @@ start_server {tags {"external:skip needs:debug"}} { assert_equal [dumpAllHashes $primary] [dumpAllHashes $replica] } } + + test "Test HSETEX command replication" { + r flushall + set repl [attach_to_replication_stream] + + # Create a field and delete it in a single command due to timestamp + # being in the past. It will be propagated as HDEL. + r hsetex h1 PXAT 1 FIELDS 1 f1 v1 + + # Following ones will be propagated with PXAT arg + r hsetex h1 EX 100000 FIELDS 1 f1 v1 + r hsetex h1 EXAT [expr [clock seconds] + 1000] FIELDS 1 f1 v1 + r hsetex h1 PX 100000 FIELDS 1 f1 v1 + r hsetex h1 PXAT [expr [clock milliseconds]+100000] FIELDS 1 f1 v1 + + # Propagate with KEEPTTL flag + r hsetex h1 KEEPTTL FIELDS 1 f1 v1 + + # Following commands will fail and won't be propagated + r hsetex h1 FNX FIELDS 1 f1 v11 + r hsetex h1 FXX FIELDS 1 f2 v2 + + # Propagate with FNX and FXX flags + r hsetex h1 FNX FIELDS 1 f2 v2 + r hsetex h1 FXX FIELDS 1 f2 v22 + + assert_replication_stream $repl { + {select *} + {hdel h1 f1} + {hsetex h1 PXAT * FIELDS 1 f1 v1} + {hsetex h1 PXAT * FIELDS 1 f1 v1} + {hsetex h1 PXAT * FIELDS 1 f1 v1} + {hsetex h1 PXAT * FIELDS 1 f1 v1} + {hsetex h1 KEEPTTL FIELDS 1 f1 v1} + {hsetex h1 FNX FIELDS 1 f2 v2} + {hsetex h1 FXX FIELDS 1 f2 v22} + } + close_replication_stream $repl + } {} {needs:repl} + + test "Test HGETEX command replication" { + r flushall + r debug set-active-expire 0 + set repl [attach_to_replication_stream] + + # If no fields are found, command won't be replicated + r hgetex h1 EX 10000 FIELDS 1 f0 + r hgetex h1 PERSIST FIELDS 1 f0 + + # Get without setting expiry will not be replicated + r hsetex h1 FIELDS 1 f0 v0 + r hgetex h1 FIELDS 1 f0 + + # Lazy expired field will be replicated as HDEL + r hsetex h1 PX 10 FIELDS 1 f1 v1 + after 15 + r hgetex h1 EX 1000 FIELDS 1 f1 + + # If new TTL is in the past, it will be replicated as HDEL + r hsetex h1 EX 10000 FIELDS 1 f2 v2 + r hgetex h1 EXAT 1 FIELDS 1 f2 + + # A field will expire lazily and other field will be deleted due to + # TTL is being in the past. It'll be propagated as two HDEL's. + r hsetex h1 PX 10 FIELDS 1 f3 v3 + after 15 + r hsetex h1 FIELDS 1 f4 v4 + r hgetex h1 EXAT 1 FIELDS 2 f3 f4 + + # TTL update, it will be replicated as HPEXPIREAT + r hsetex h1 FIELDS 1 f5 v5 + r hgetex h1 EX 10000 FIELDS 1 f5 + + # If PERSIST flag is used, it will be replicated as HPERSIST + r hsetex h1 EX 10000 FIELDS 1 f6 v6 + r hgetex h1 PERSIST FIELDS 1 f6 + + assert_replication_stream $repl { + {select *} + {hsetex h1 FIELDS 1 f0 v0} + {hsetex h1 PXAT * FIELDS 1 f1 v1} + {hdel h1 f1} + {hsetex h1 PXAT * FIELDS 1 f2 v2} + {hdel h1 f2} + {hsetex h1 PXAT * FIELDS 1 f3 v3} + {hsetex h1 FIELDS 1 f4 v4} + {multi} + {hdel h1 f3} + {hdel h1 f4} + {exec} + {hsetex h1 FIELDS 1 f5 v5} + {hpexpireat h1 * FIELDS 1 f5} + {hsetex h1 PXAT * FIELDS 1 f6 v6} + {hpersist h1 FIELDS 1 f6} + } + close_replication_stream $repl + } {} {needs:repl} } } diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index 1cb42245515..a3d6867f865 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -371,6 +371,7 @@ start_server {tags {"hash"}} { assert_error "WRONGTYPE Operation against a key*" {r hsetnx wrongtype field1 val1} assert_error "WRONGTYPE Operation against a key*" {r hlen wrongtype} assert_error "WRONGTYPE Operation against a key*" {r hscan wrongtype 0} + assert_error "WRONGTYPE Operation against a key*" {r hgetdel wrongtype fields 1 a} } test {HMGET - small hash} { @@ -710,6 +711,89 @@ start_server {tags {"hash"}} { r config set hash-max-listpack-value $original_max_value } + test {HGETDEL input validation} { + r del key1 + assert_error "*wrong number of arguments*" {r hgetdel} + assert_error "*wrong number of arguments*" {r hgetdel key1} + assert_error "*wrong number of arguments*" {r hgetdel key1 FIELDS} + assert_error "*wrong number of arguments*" {r hgetdel key1 FIELDS 0} + assert_error "*wrong number of arguments*" {r hgetdel key1 FIELDX} + assert_error "*argument FIELDS is missing*" {r hgetdel key1 XFIELDX 1 a} + assert_error "*numfields*parameter*must match*number of arguments*" {r hgetdel key1 FIELDS 2 a} + assert_error "*numfields*parameter*must match*number of arguments*" {r hgetdel key1 FIELDS 2 a b c} + assert_error "*Number of fields must be a positive integer*" {r hgetdel key1 FIELDS 0 a} + assert_error "*Number of fields must be a positive integer*" {r hgetdel key1 FIELDS -1 a} + assert_error "*Number of fields must be a positive integer*" {r hgetdel key1 FIELDS b a} + assert_error "*Number of fields must be a positive integer*" {r hgetdel key1 FIELDS 9223372036854775808 a} + } + + foreach type {listpack ht} { + set orig_config [lindex [r config get hash-max-listpack-entries] 1] + r del key1 + + if {$type == "listpack"} { + r config set hash-max-listpack-entries $orig_config + r hset key1 f1 1 f2 2 f3 3 strfield strval + assert_encoding listpack key1 + } else { + r config set hash-max-listpack-entries 0 + r hset key1 f1 1 f2 2 f3 3 strfield strval + assert_encoding hashtable key1 + } + + test {HGETDEL basic test} { + r del key1 + r hset key1 f1 1 f2 2 f3 3 strfield strval + assert_equal [r hgetdel key1 fields 1 f2] 2 + assert_equal [r hlen key1] 3 + assert_equal [r hget key1 f1] 1 + assert_equal [r hget key1 f2] "" + assert_equal [r hget key1 f3] 3 + assert_equal [r hget key1 strfield] strval + + assert_equal [r hgetdel key1 fields 1 f1] 1 + assert_equal [lsort [r hgetall key1]] [lsort "f3 3 strfield strval"] + assert_equal [r hgetdel key1 fields 1 f3] 3 + assert_equal [r hgetdel key1 fields 1 strfield] strval + assert_equal [r hgetall key1] "" + assert_equal [r exists key1] 0 + } + + test {HGETDEL test with non existing fields} { + r del key1 + r hset key1 f1 1 f2 2 f3 3 + assert_equal [r hgetdel key1 fields 4 x1 x2 x3 x4] "{} {} {} {}" + assert_equal [r hgetdel key1 fields 4 x1 x2 f3 x4] "{} {} 3 {}" + assert_equal [lsort [r hgetall key1]] [lsort "f1 1 f2 2"] + assert_equal [r hgetdel key1 fields 3 f1 f2 f3] "1 2 {}" + assert_equal [r hgetdel key1 fields 3 f1 f2 f3] "{} {} {}" + } + + r config set hash-max-listpack-entries $orig_config + } + + test {HGETDEL propagated as HDEL command to replica} { + set repl [attach_to_replication_stream] + r hset key1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r hgetdel key1 fields 1 f1 + r hgetdel key1 fields 2 f2 f3 + + # make sure non-existing fields are not replicated + r hgetdel key1 fields 2 f7 f8 + + # delete more + r hgetdel key1 fields 3 f4 f5 f6 + + assert_replication_stream $repl { + {select *} + {hset key1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5} + {hdel key1 f1} + {hdel key1 f2 f3} + {hdel key1 f4 f5 f6} + } + close_replication_stream $repl + } {} {needs:repl} + test {Hash ziplist regression test for large keys} { r hset hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk a r hset hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk b