Fail to save the cluster config file will not exit the process

src/cluster_legacy.c

@@ -1049,13 +1049,22 @@ int clusterSaveConfig(int do_fsync) {
     return retval;
 }
 
+/* Save the cluster configuration file. If the save fails, exit the process. */
 void clusterSaveConfigOrDie(int do_fsync) {
     if (clusterSaveConfig(do_fsync) == C_ERR) {
         serverLog(LL_WARNING, "Fatal: can't update cluster config file.");
         exit(1);
     }
 }
 
+/* Save the cluster configuration file. If the save fails, print the log. */
+void clusterSaveConfigOrLog(int do_fsync) {
+    if (clusterSaveConfig(do_fsync) == C_ERR) {
+        serverLog(LL_WARNING, "Cluster config file is applying a change even though "
+                              "it is unable to write to disk.");
+    }
+}
+
 /* Lock the cluster config using flock(), and retain the file descriptor used to
  * acquire the lock so that the file will be locked as long as the process is up.
  *
@@ -1321,6 +1330,7 @@ void clusterInit(void) {
     server.cluster->currentEpoch = 0;
     server.cluster->state = CLUSTER_FAIL;
     server.cluster->fail_reason = CLUSTER_FAIL_NONE;
+    server.cluster->safe_to_join = 0;
     server.cluster->size = 0;
     server.cluster->todo_before_sleep = 0;
     server.cluster->nodes = dictCreate(&clusterNodesDictType);
@@ -5013,6 +5023,12 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) {
      * size + 1 */
     if (!clusterNodeIsVotingPrimary(myself)) return;
 
+    if (!server.cluster->safe_to_join) {
+        serverLog(LL_WARNING, "Failover auth denied to %.40s (%s): it is not safe to vote in this moment)",
+                  node->name, node->human_nodename);
+        return;
+    }
+
     /* Request epoch must be >= our currentEpoch.
      * Note that it is impossible for it to actually be greater since
      * our currentEpoch was updated as a side effect of receiving this
@@ -6023,7 +6039,7 @@ void clusterBeforeSleep(void) {
     /* Save the config, possibly using fsync. */
     if (flags & CLUSTER_TODO_SAVE_CONFIG) {
         int fsync = flags & CLUSTER_TODO_FSYNC_CONFIG;
-        clusterSaveConfigOrDie(fsync);
+        clusterSaveConfigOrLog(fsync);
     }
 
     if (flags & CLUSTER_TODO_BROADCAST_ALL) {
@@ -6267,8 +6283,12 @@ void clusterUpdateState(void) {
      * to not count the DB loading time. */
     if (first_call_time == 0) first_call_time = mstime();
     if (clusterNodeIsPrimary(myself) && server.cluster->state == CLUSTER_FAIL &&
-        mstime() - first_call_time < CLUSTER_WRITABLE_DELAY)
+        mstime() - first_call_time < CLUSTER_WRITABLE_DELAY) {
+        server.cluster->safe_to_join = 0;
         return;
+    } else {
+        server.cluster->safe_to_join = 1;
+    }
 
     /* Start assuming the state is OK. We'll turn it into FAIL if there
      * are the right conditions. */

src/cluster_legacy.h

@@ -403,6 +403,7 @@ struct clusterState {
     uint64_t currentEpoch;
     int state;              /* CLUSTER_OK, CLUSTER_FAIL, ... */
     int fail_reason;        /* Why the cluster state changes to fail. */
+    int safe_to_join;       /* Can the restarted node safely join the cluster? */
     int size;               /* Num of primary nodes with at least one slot */
     dict *nodes;            /* Hash table of name -> clusterNode structures */
     dict *shards;           /* Hash table of shard_id -> list (of nodes) structures */

tests/unit/cluster/misc.tcl

@@ -33,3 +33,34 @@ start_cluster 1 1 {tags {external:skip cluster}} {
     }
 }
 
+# Create a folder called "nodes.conf" to trigger temp nodes.conf rename
+# failure and it will cause cluster config file save to fail at the rename.
+proc create_nodes_conf_folder {srv_idx} {
+    set dir [lindex [R $srv_idx config get dir] 1]
+    set cluster_conf [lindex [R $srv_idx config get cluster-config-file] 1]
+    set cluster_conf_path [file join $dir $cluster_conf]
+    if {[file exists $cluster_conf_path]} { exec rm -f $cluster_conf_path }
+    exec mkdir -p $cluster_conf_path
+}
+
+start_cluster 1 1 {tags {external:skip cluster}} {
+    test {Fail to save the cluster configuration file will not exit the process} {
+        # Create folder that can cause the rename fail.
+        create_nodes_conf_folder 0
+        create_nodes_conf_folder 1
+
+        # Trigger a takeover so that cluster will need to update the config file.
+        R 1 cluster failover takeover
+
+        assert_equal {PONG} [R 0 ping]
+        assert_equal {PONG} [R 1 ping]
+        assert_equal 1 [process_is_alive [srv 0 pid]]
+        assert_equal 1 [process_is_alive [srv -1 pid]]
+
+        # Make sure relevant logs are printed.
+        verify_log_message 0 "*Could not rename tmp cluster config file*" 0
+        verify_log_message -1 "*Could not rename tmp cluster config file*" 0
+        verify_log_message 0 "*Cluster config file is applying a change even though it is unable to write to disk*" 0
+        verify_log_message -1 "*Cluster config file is applying a change even though it is unable to write to disk*" 0
+    }
+}

tests/unit/latency-monitor.tcl

@@ -201,7 +201,7 @@ start_cluster 1 1 {tags {"latency-monitor cluster external:skip needs:latency"}
         # We don't assert anything since we can't be sure whether it will be counted.
         R 0 cluster saveconfig
         R 1 cluster saveconfig
-        R 1 cluster failover force
+        R 1 cluster failover takeover
         R 0 latency latest
         R 1 latency latest
     }