Add maxmemory-reserved parameter for evicting key earlier to avoid OOM

src/config.c

@@ -2496,18 +2496,33 @@ static int updateReplBacklogSize(const char **err) {
     return 1;
 }
 
+static int updateMaxmemoryReserved(const char **err) {
+    if (server.maxmemory) {
+        if (isMaxmemoryReservedLessThanMaxmemory(err) == C_ERR) {
+            return 0;
+        }
+        calculateKeyEvictionMemory();
+        startEvictionTimeProc();
+    } else {
+        if (server.maxmemory_reserved) {
+            *err = "Current maxmemory value is 0, the new reserved memory is invalid";
+            return 0;
+        }
+        server.key_eviction_memory = 0;
+    }
+    return 1;
+}
+
 static int updateMaxmemory(const char **err) {
-    UNUSED(err);
     if (server.maxmemory) {
-        size_t used = zmalloc_used_memory() - freeMemoryGetNotCountedMemory();
-        if (server.maxmemory < used) {
-            serverLog(LL_WARNING,
-                      "WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory "
-                      "usage (%zu). This will result in key eviction and/or the inability to accept new write commands "
-                      "depending on the maxmemory-policy.",
-                      server.maxmemory, used);
+        if (isMaxmemoryReservedLessThanMaxmemory(err) == C_ERR) {
+            return 0;
         }
+        calculateKeyEvictionMemory();
         startEvictionTimeProc();
+    } else {
+        server.maxmemory_reserved = 0;
+        server.key_eviction_memory = 0;
     }
     return 1;
 }
@@ -3329,6 +3344,7 @@ standardConfig static_configs[] = {
 
     /* Unsigned Long Long configs */
     createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory),
+    createULongLongConfig("maxmemory-reserved", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory_reserved, 0, MEMORY_CONFIG, NULL, updateMaxmemoryReserved),
     createULongLongConfig("cluster-link-sendbuf-limit", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.cluster_link_msg_queue_limit_bytes, 0, MEMORY_CONFIG, NULL, NULL),
 
     /* Size_t configs */

src/evict.c

@@ -377,7 +377,7 @@ size_t freeMemoryGetNotCountedMemory(void) {
  *              limit.
  *              (Populated both for C_ERR and C_OK)
  */
-int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level) {
+int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level, unsigned long long maxmemory) {
     size_t mem_reported, mem_used, mem_tofree;
 
     /* Check if we are over the memory usage limit. If we are not, no need
@@ -386,11 +386,12 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
     if (total) *total = mem_reported;
 
     /* We may return ASAP if there is no need to compute the level. */
-    if (!server.maxmemory) {
+    if (!maxmemory) {
         if (level) *level = 0;
         return C_OK;
     }
-    if (mem_reported <= server.maxmemory && !level) return C_OK;
+
+    if (mem_reported <= maxmemory && !level) return C_OK;
 
     /* Remove the size of replicas output buffers and AOF buffer from the
      * count of used memory. */
@@ -399,15 +400,19 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
     mem_used = (mem_used > overhead) ? mem_used - overhead : 0;
 
     /* Compute the ratio of memory usage. */
-    if (level) *level = (float)mem_used / (float)server.maxmemory;
+    if (level) {
+        *level = (float)mem_used / (float)server.maxmemory;
+    }
 
-    if (mem_reported <= server.maxmemory) return C_OK;
+    if (mem_reported <= maxmemory) return C_OK;
 
     /* Check if we are still over the memory limit. */
-    if (mem_used <= server.maxmemory) return C_OK;
+    /* if function parameter 'maxmemory' is equal to maxmemory and mem_used > maxmemory then OOM /
+    / if function parameter 'maxmemory' is equal to maxmemory_soft and mem_used > function parameter 'maxmemory' then there is no OOM but eviction happens */
+    if (mem_used <= maxmemory) return C_OK;
 
     /* Compute how much memory we need to free. */
-    mem_tofree = mem_used - server.maxmemory;
+    mem_tofree = mem_used - server.key_eviction_memory;
 
     if (logical) *logical = mem_used;
     if (tofree) *tofree = mem_tofree;
@@ -522,20 +527,24 @@ int performEvictions(void) {
     if (!isSafeToPerformEvictions()) return EVICT_OK;
 
     int keys_freed = 0;
-    size_t mem_reported, mem_tofree;
+    size_t mem_reported, mem_tofree, mem_used;
     long long mem_freed = 0; /* Maybe become negative */
     mstime_t latency, eviction_latency;
     long long delta;
     int replicas = listLength(server.replicas);
     int result = EVICT_FAIL;
 
-    if (getMaxmemoryState(&mem_reported, NULL, &mem_tofree, NULL) == C_OK) {
+    if (getMaxmemoryState(&mem_reported, &mem_used, &mem_tofree, NULL, server.key_eviction_memory) == C_OK) {
         result = EVICT_OK;
         goto update_metrics;
     }
 
     if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION || (iAmPrimary() && server.import_mode)) {
-        result = EVICT_FAIL; /* We need to free memory, but policy forbids or we are in import mode. */
+        if (mem_used >= server.maxmemory) {
+            result = EVICT_FAIL; /* We need to free memory, but policy forbids or we are in import mode. */
+        } else {
+            result = EVICT_OK; /* used_memory greater than key_eviction_memory, but not reach OOM */
+        }
         goto update_metrics;
     }
 
@@ -697,7 +706,7 @@ int performEvictions(void) {
                  * across the dbAsyncDelete() call, while the thread can
                  * release the memory all the time. */
                 if (server.lazyfree_lazy_eviction) {
-                    if (getMaxmemoryState(NULL, NULL, NULL, NULL) == C_OK) {
+                    if (getMaxmemoryState(NULL, NULL, NULL, NULL, server.key_eviction_memory) == C_OK) {
                         break;
                     }
                 }
@@ -712,21 +721,23 @@ int performEvictions(void) {
                 }
             }
         } else {
-            goto cant_free; /* nothing to free... */
+            break;
         }
     }
-    /* at this point, the memory is OK, or we have reached the time limit */
-    result = (isEvictionProcRunning) ? EVICT_RUNNING : EVICT_OK;
 
-cant_free:
+    if (mem_freed >= (long long)(mem_used - server.key_eviction_memory)) {
+        /* at this point, the memory is OK, or we have reached the time limit */
+        result = (isEvictionProcRunning) ? EVICT_RUNNING : EVICT_OK;
+    }
+
     if (result == EVICT_FAIL) {
         /* At this point, we have run out of evictable items.  It's possible
          * that some items are being freed in the lazyfree thread.  Perform a
          * short wait here if such jobs exist, but don't wait long.  */
         mstime_t lazyfree_latency;
         latencyStartMonitor(lazyfree_latency);
         while (bioPendingJobsOfType(BIO_LAZY_FREE) && elapsedUs(evictionTimer) < eviction_time_limit_us) {
-            if (getMaxmemoryState(NULL, NULL, NULL, NULL) == C_OK) {
+            if (getMaxmemoryState(NULL, NULL, NULL, NULL, server.key_eviction_memory) == C_OK) {
                 result = EVICT_OK;
                 break;
             }

src/module.c

@@ -4007,7 +4007,7 @@ int VM_GetContextFlags(ValkeyModuleCtx *ctx) {
 
     /* OOM flag. */
     float level;
-    int retval = getMaxmemoryState(NULL, NULL, NULL, &level);
+    int retval = getMaxmemoryState(NULL, NULL, NULL, &level, server.maxmemory);
     if (retval == C_ERR) flags |= VALKEYMODULE_CTX_FLAGS_OOM;
     if (level > 0.75) flags |= VALKEYMODULE_CTX_FLAGS_OOM_WARNING;
 
@@ -6405,7 +6405,7 @@ ValkeyModuleCallReply *VM_Call(ValkeyModuleCtx *ctx, const char *cmdname, const
                 /* On background thread we can not count on server.pre_command_oom_state.
                  * Because it is only set on the main thread, in such case we will check
                  * the actual memory usage. */
-                oom_state = (getMaxmemoryState(NULL, NULL, NULL, NULL) == C_ERR);
+                oom_state = (getMaxmemoryState(NULL, NULL, NULL, NULL, server.maxmemory) == C_ERR);
             } else {
                 oom_state = server.pre_command_oom_state;
             }
@@ -10956,7 +10956,7 @@ size_t VM_MallocSizeDict(ValkeyModuleDict *dict) {
  */
 float VM_GetUsedMemoryRatio(void) {
     float level;
-    getMaxmemoryState(NULL, NULL, NULL, &level);
+    getMaxmemoryState(NULL, NULL, NULL, &level, server.maxmemory);
     return level;
 }
 

src/server.c

@@ -2785,6 +2785,17 @@ void initServer(void) {
     server.client_mem_usage_buckets = NULL;
     resetReplicationBuffer();
 
+    if (server.maxmemory) {
+        const char *err = NULL;
+        if (isMaxmemoryReservedLessThanMaxmemory(&err) == C_ERR) {
+            serverLog(LL_WARNING, "%s", err);
+            exit(1);
+        }
+        server.key_eviction_memory = server.maxmemory - server.maxmemory_reserved;
+    } else {
+        server.key_eviction_memory = 0;
+    }
+
     /* Make sure the locale is set on startup based on the config file. */
     if (setlocale(LC_COLLATE, server.locale_collate) == NULL) {
         serverLog(LL_WARNING, "Failed to configure LOCALE for invalid locale name.");
@@ -5717,6 +5728,7 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) {
         char used_memory_scripts_hmem[64];
         char used_memory_rss_hmem[64];
         char maxmemory_hmem[64];
+        char maxmemory_reserved_hmem[64];
         size_t zmalloc_used = zmalloc_used_memory();
         size_t total_system_mem = server.system_memory_size;
         const char *evict_policy = evictPolicyToString();
@@ -5738,6 +5750,7 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) {
         bytesToHuman(used_memory_scripts_hmem, sizeof(used_memory_scripts_hmem), mh->lua_caches + mh->functions_caches);
         bytesToHuman(used_memory_rss_hmem, sizeof(used_memory_rss_hmem), server.cron_malloc_stats.process_rss);
         bytesToHuman(maxmemory_hmem, sizeof(maxmemory_hmem), server.maxmemory);
+        bytesToHuman(maxmemory_reserved_hmem, sizeof(maxmemory_reserved_hmem), server.maxmemory_reserved);
 
         if (sections++) info = sdscat(info, "\r\n");
         info = sdscatprintf(
@@ -5776,6 +5789,8 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) {
                 "maxmemory:%lld\r\n", server.maxmemory,
                 "maxmemory_human:%s\r\n", maxmemory_hmem,
                 "maxmemory_policy:%s\r\n", evict_policy,
+                "maxmemory_reserved:%lld\r\n", server.maxmemory_reserved,
+                "maxmemory_reserved_human:%s\r\n", maxmemory_reserved_hmem,
                 "allocator_frag_ratio:%.2f\r\n", mh->allocator_frag,
                 "allocator_frag_bytes:%zu\r\n", mh->allocator_frag_bytes,
                 "allocator_rss_ratio:%.2f\r\n", mh->allocator_rss,
@@ -6761,6 +6776,23 @@ int validateProcTitleTemplate(const char *template) {
     return ok;
 }
 
+int isMaxmemoryReservedLessThanMaxmemory(const char **err) {
+    if (server.maxmemory <= server.maxmemory_reserved) {
+        *err = "The maxmemory reserved value should be smaller than maxmemory.";
+        return C_ERR;
+    }
+    return C_OK;
+}
+
+void calculateKeyEvictionMemory(void) {
+    server.key_eviction_memory = server.maxmemory - server.maxmemory_reserved;
+    size_t used = zmalloc_used_memory() - freeMemoryGetNotCountedMemory();
+    if (server.key_eviction_memory < used) {
+        serverLog(LL_WARNING, "WARNING: the difference between memory usage and maxmemory is less than reserved memory. "
+                              "This will result in key eviction depending on the maxmemory-policy. But server can still accept new write commands.");
+    }
+}
+
 int serverSetProcTitle(char *title) {
 #ifdef USE_SETPROCTITLE
     if (!title) title = server.exec_argv[0];

src/server.h

@@ -1991,6 +1991,8 @@ struct valkeyServer {
     /* Limits */
     unsigned int maxclients;                    /* Max number of simultaneous clients */
     unsigned long long maxmemory;               /* Max number of memory bytes to use */
+    unsigned long long maxmemory_reserved;      /* Memory reserved below `maxmemory` (in bytes) before key eviction is triggered */
+    unsigned long long key_eviction_memory;     /* Available memory threshold (in bytes) that initiates key eviction */
     ssize_t maxmemory_clients;                  /* Memory limit for total client buffers */
     int maxmemory_policy;                       /* Policy for key eviction */
     int maxmemory_samples;                      /* Precision of random sampling */
@@ -2594,6 +2596,8 @@ int validateProcTitleTemplate(const char *template);
 int serverCommunicateSystemd(const char *sd_notify_msg);
 void serverSetCpuAffinity(const char *cpulist);
 void dictVanillaFree(void *val);
+int isMaxmemoryReservedLessThanMaxmemory(const char **err);
+void calculateKeyEvictionMemory(void);
 
 /* ERROR STATS constants */
 
@@ -3137,7 +3141,7 @@ int zslLexValueGteMin(sds value, zlexrangespec *spec);
 int zslLexValueLteMax(sds value, zlexrangespec *spec);
 
 /* Core functions */
-int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level);
+int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level, unsigned long long maxmemory);
 size_t freeMemoryGetNotCountedMemory(void);
 int overMaxmemoryAfterAlloc(size_t moremem);
 uint64_t getCommandFlags(client *c);

tests/unit/maxmemory.tcl

@@ -263,8 +263,70 @@ start_server {tags {"maxmemory external:skip"}} {
             }
         }
     }
-}
 
+    test "enable maxmemory-reserved, test maxmemory-reserved with maxmemory update" {
+         # make sure to start with a blank instance
+         r flushall
+         # we set maxmemory as 0, and we expect maxmemory-reserved as 0 too.
+         r config set maxmemory 0
+         assert_equal 0 [lindex [r config get maxmemory] 1]
+         assert_equal 0 [lindex [r config get maxmemory-reserved] 1]
+         # we increase maxmemory and maxmemory-reserved both
+         r config set maxmemory 10000000
+         r config set maxmemory-reserved 4000000
+         assert_equal 10000000 [lindex [r config get maxmemory] 1]
+         assert_equal 4000000 [lindex [r config get maxmemory-reserved] 1]
+         # we decrease maxmemory and maxmemory-reserved no change
+         r config set maxmemory 6000000
+         r config set maxmemory-reserved 4000000
+         assert_equal 6000000 [lindex [r config get maxmemory] 1]
+         assert_equal 4000000 [lindex [r config get maxmemory-reserved] 1]
+	 catch {r config set maxmemory 3000000} err
+	 assert_match "*maxmemory reserved value should be smaller than maxmemory*" $err
+    }
+
+    foreach policy {
+        allkeys-random allkeys-lru allkeys-lfu volatile-lru volatile-lfu volatile-random volatile-ttl
+    } {
+        test "enable maxmemory-reserved, test eviction key number with policy ($policy) and different maxmemory value" {
+            r flushall
+            r config set maxmemory 0
+            # make sure to start with a blank instance
+            set num_eviction_key_init [s evicted_keys]
+            set used 1165448
+            set limit_maxmemory_value1 [expr {$used+40*1024}]
+            set limit_maxmemory_value2 [expr {$used+70*1024}]
+            set limit_maxmemory_value3 [expr {$used+100*1024}]
+            r config set maxmemory $limit_maxmemory_value1
+            r config set maxmemory-policy $policy
+            r config set maxmemory-reserved 30720
+            set numkeys 5000
+            for {set j 0} {$j < $numkeys} {incr j} {
+                catch {r set $j $j EX 10000}
+            }
+            set num_eviction_key_maxmemory_1 [s evicted_keys]
+            set diff_num_key_eviction_one [expr {$num_eviction_key_maxmemory_1 - $num_eviction_key_init}]    
+            r flushall
+            r config set maxmemory $limit_maxmemory_value2
+            for {set j 0} {$j < $numkeys} {incr j} {
+                catch {r set $j $j EX 10000}
+            }
+            set num_eviction_key_maxmemory_2 [s evicted_keys]
+            set diff_num_key_eviction_two [expr {$num_eviction_key_maxmemory_2 - $num_eviction_key_maxmemory_1}]
+            r flushall
+            r config set maxmemory $limit_maxmemory_value3
+            for {set j 0} {$j < $numkeys} {incr j} {
+                catch {r set $j $j EX 10000}
+            }
+            set num_eviction_key_maxmemory_3 [s evicted_keys]
+            set diff_num_key_eviction_three [expr {$num_eviction_key_maxmemory_3 - $num_eviction_key_maxmemory_2}]
+            assert_morethan $diff_num_key_eviction_two $diff_num_key_eviction_three
+            assert_morethan $diff_num_key_eviction_one $diff_num_key_eviction_two
+            r flushall
+            r config set maxmemory 0
+        }
+    }
+}
 # Calculate query buffer memory of slave
 proc slave_query_buffer {srv} {
     set clients [split [$srv client list] "\r\n"]

valkey.conf

@@ -1279,6 +1279,13 @@ acllog-max-len 128
 #
 # maxmemory-policy noeviction
 
+# `maxmemory-reserved` specifies a fixed amount of memory set aside from `maxmemory`.
+# When the available memory, the difference between memory usage and `maxmemory`, falls 
+# below this threshold, proactive key eviction is triggered. However, this does not immediately 
+# result in write commands being rejected; only reaching the `maxmemory` limit will do so.
+#
+# maxmemory-reserved <bytes>
+
 # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated
 # algorithms (in order to save memory), so you can tune it for speed or
 # accuracy. By default the server will check five keys and pick the one that was