Configurable DB hash seed for SCAN family commands consistency (valkey-io#2608)

Introduce a new config `hash-seed` which can be set only at startup and
controls the hash seed for the server. This includes all hash tables.
This change makes it so that both primaries and replicas will return the
same results for SCAN/HSCAN/ZSCAN/SSCAN cursors. This is useful in order
to make sure SCAN behaves correctly after a failover.

Resolves valkey-io#4

---------

Signed-off-by: Sarthak Aggarwal <[email protected]>
Signed-off-by: Sarthak Aggarwal <[email protected]>
Co-authored-by: Viktor Söderqvist <[email protected]>

Author: 2 people

src/config.c

@@ -3187,6 +3187,15 @@ static int applyClientMaxMemoryUsage(const char **err) {
     return 1;
 }
 
+#define HASH_SEED_MAX_LEN 256
+static int isValidDbHashSeed(sds val, const char **err) {
+    if (sdslen(val) > HASH_SEED_MAX_LEN) {
+        *err = "hash-seed must be less than or equal to " STRINGIFY(HASH_SEED_MAX_LEN) " characters";
+        return 0;
+    }
+    return 1;
+}
+
 standardConfig static_configs[] = {
     /* Bool configs */
     createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, 1, NULL, NULL),
@@ -3277,6 +3286,7 @@ standardConfig static_configs[] = {
     createSDSConfig("primaryauth", "masterauth", MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.primary_auth, NULL, NULL, NULL),
     createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.requirepass, NULL, NULL, updateRequirePass),
     createSDSConfig("availability-zone", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.availability_zone, "", NULL, NULL),
+    createSDSConfig("hash-seed", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.hash_seed, NULL, isValidDbHashSeed, NULL),
 
     /* Enum Configs */
     createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL),

src/server.c

@@ -7373,8 +7373,13 @@ __attribute__((weak)) int main(int argc, char **argv) {
         sdsfree(options);
     }
     if (server.sentinel_mode) sentinelCheckConfigFile();
+    if (server.hash_seed != NULL) {
+        memset(hashseed, 0, sizeof(hashseed));
+        getHashSeedFromString(hashseed, sizeof(hashseed), server.hash_seed);
+        hashtableSetHashFunctionSeed(hashseed);
+    }
 
-        /* Do system checks */
+    /* Do system checks */
 #ifdef __linux__
     linuxMemoryWarnings();
     sds err_msg = NULL;

src/server.h

@@ -2214,6 +2214,7 @@ struct valkeyServer {
                                                             * dropping packets of a specific type */
     unsigned long cluster_blacklist_ttl;                   /* Duration in seconds that a node is denied re-entry into
                                                             * the cluster after it is forgotten with CLUSTER FORGET. */
+    sds hash_seed;                                         /* Configurable DB hash seed */
     int cluster_slot_stats_enabled;                        /* Cluster slot usage statistics tracking enabled. */
     mstime_t cluster_mf_timeout;                           /* Milliseconds to do a manual failover. */
     unsigned long cluster_slot_migration_log_max_len;      /* Maximum count of migrations to display in the

src/util.c

@@ -1037,6 +1037,21 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) {
     return 0;
 }
 
+/* Populate the provided seed array by hashing the provided string with SHA256
+ * and copying the first outlen bytes of the digest into the seed buffer. */
+void getHashSeedFromString(unsigned char *seed_array, size_t outlen, const char *value) {
+    SHA256_CTX ctx;
+    unsigned char digest[SHA256_BLOCK_SIZE];
+
+    sha256_init(&ctx);
+    sha256_update(&ctx, (const BYTE *)value, strlen(value));
+    sha256_final(&ctx, digest);
+
+    if (outlen > SHA256_BLOCK_SIZE) outlen = SHA256_BLOCK_SIZE;
+    memcpy(seed_array, digest, outlen);
+}
+
+
 /* Parses a version string on the form "major.minor.patch" and returns an
  * integer on the form 0xMMmmpp. Returns -1 on parse error. */
 int version2num(const char *version) {

src/util.h

@@ -91,6 +91,7 @@ int trimDoubleString(char *buf, size_t len);
 int d2string(char *buf, size_t len, double value);
 int fixedpoint_d2string(char *dst, size_t dstlen, double dvalue, int fractional_digits);
 int ld2string(char *buf, size_t len, long double value, ld2string_mode mode);
+void getHashSeedFromString(unsigned char *seed_array, size_t len, const char *value);
 int double2ll(double d, long long *out);
 int version2num(const char *version);
 int yesnotoi(char *s);

tests/integration/scan-family-consistency.tcl

@@ -0,0 +1,60 @@
+test {scan family consistency with configured hash seed} {
+    start_server {tags {"external:skip"}} {
+
+        set fixed_seed "aabbccddeeffgghh"
+        set shared_overrides [list appendonly no save "" hash-seed $fixed_seed activedefrag no hz 1]
+
+        start_server [list overrides $shared_overrides] {
+            set primary_host [srv 0 host]
+            set primary_port [srv 0 port]
+
+            start_server [list overrides $shared_overrides] {
+                set primary [srv -1 client]
+                set replica [srv 0 client]
+
+                $primary flushall
+                $replica replicaof $primary_host $primary_port
+                wait_for_sync $replica
+
+                set n 50
+                for {set i 0} {$i < $n} {incr i} {
+                    $primary set "k:$i" x
+                    $primary hset h "f:$i" $i
+                    $primary sadd s "m:$i"
+                    $primary zadd z $i "m:$i"
+                }
+
+                wait_for_condition 200 50 {
+                    [$replica dbsize] == [$primary dbsize]
+                } else {
+                    fail "replica did not catch up dbsize (primary=[$primary dbsize], replica=[$replica dbsize])"
+                }
+                set cursor {{0} {}}
+                while {1} {
+                    set primary_cursor_next [$primary scan [lindex $cursor 0]]
+                    set replica_cursor_next [$replica scan [lindex $cursor 0]]
+                    assert_equal $primary_cursor_next $replica_cursor_next
+                    if {[lindex $primary_cursor_next 0] eq "0"} {
+                        assert_equal "0" [lindex $replica_cursor_next 0]
+                        break
+                    }
+                    set cursor $primary_cursor_next
+                }
+
+                foreach {cmd key} {hscan h sscan s zscan z} {
+                    set cursor {{0} {}}
+                    while {1} {
+                        set primary_cursor_next [$primary $cmd $key [lindex $cursor 0]]
+                        set replica_cursor_next [$replica $cmd $key [lindex $cursor 0]]
+                        assert_equal $primary_cursor_next $replica_cursor_next
+                        if {[lindex $primary_cursor_next 0] eq "0"} {
+                            assert_equal "0" [lindex $replica_cursor_next 0]
+                            break
+                        }
+                        set cursor $primary_cursor_next
+                    }
+                }
+            }
+        }
+    }
+}

tests/unit/introspection.tcl

@@ -1225,6 +1225,7 @@ start_server {tags {"introspection"}} {
             disable-thp
             aclfile
             unixsocket
+            hash-seed
             pidfile
             syslog-ident
             appendfilename
@@ -1956,3 +1957,11 @@ test {CONFIG REWRITE handles alias config properly} {
         assert_equal [r config get hash-max-listpack-entries] {hash-max-listpack-entries 100}
     }
 } {} {external:skip}
+
+test {CONFIG hash-seed is immutable and settable at startup} {
+    start_server {tags {"introspection"} overrides {hash-seed aabbccddeeffgghh}} {
+        assert_error "ERR CONFIG SET failed (possibly related to argument 'hash-seed') - can't set immutable config*" {
+            r config set hash-seed newseed
+        }
+    }
+} {} {external:skip}

valkey.conf

@@ -527,6 +527,13 @@ locale-collate ""
 #
 # availability-zone "zone-name"
 
+# Use a fixed hash seed for hashtable instead of a random one.
+# Setting this option makes commands like SCAN return keys in a consistent
+# order across restarts and failovers. The seed can be any string up to 256 characters.
+# The value is immutable and must be provided only at server startup.
+#
+# hash-seed example-seed-val
+
 ################################ SNAPSHOTTING  ################################
 
 # Save the DB to disk.