WIP: Build without Lua scripting via USE_LUA (#1204)

.github/workflows/ci.yml

@@ -68,6 +68,24 @@ jobs:
       - name: show-kernel-log
         run: sudo dmesg -c
 
+  test-without-lua:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: make
+        # Fail build if there are warnings
+        # build with TLS just for compilation coverage
+        run: make -j4 all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes USE_LUA=no
+      - name: test
+        run: |
+          sudo apt-get install tcl8.6 tclx
+          ./runtest --verbose --tags -slow --tags -scripting --dump-logs
+      - name: module api test
+        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs --tags -scripting
+      - name: unit tests
+        run: |
+          ./src/valkey-unit-tests
+
   build-debian-old:
     runs-on: ubuntu-latest
     container: debian:buster

README.md

@@ -48,6 +48,10 @@ as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run:
 
     % make USE_SYSTEMD=yes
 
+To build without Lua scripting (it is enabled by default), run:
+
+    % make USE_LUA=no
+
 To append a suffix to Valkey program names, use:
 
     % make PROG_SUFFIX="-alt"

src/Makefile

@@ -31,7 +31,20 @@ endif
 ifneq ($(OPTIMIZATION),-O0)
 	OPTIMIZATION+=-fno-omit-frame-pointer
 endif
-DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv
+
+# Lua enablement flags.  Lua is enabled by default,
+# but can be disabled with USE_LUA=no
+ifneq ($(USE_LUA),no)
+	LUA_LIBS=../deps/lua/src/liblua.a
+	LUA_CFLAGS=-I../deps/lua/src -DUSE_LUA
+	LUA_TARGET=lua
+	LUA_TEST_FLAGS=
+else
+	# Do not run tests that require Lua scripting
+	LUA_TEST_FLAGS=--tags -scripting
+endif
+
+DEPENDENCY_TARGETS=hiredis linenoise ${LUA_TARGET} hdr_histogram fpconv
 NODEPS:=clean distclean
 
 # Default settings
@@ -248,7 +261,7 @@ ifdef OPENSSL_PREFIX
 endif
 
 # Include paths to dependencies
-FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv
+FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise ${LUA_CFLAGS} -I../deps/hdr_histogram -I../deps/fpconv
 
 # Determine systemd support and/or build preference (defaulting to auto-detection)
 BUILD_WITH_SYSTEMD=no
@@ -477,15 +490,15 @@ endif
 
 # valkey-server
 $(SERVER_NAME): $(ENGINE_SERVER_OBJ)
-	$(SERVER_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
+	$(SERVER_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ${LUA_LIBS} ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
 
 # Valkey static library, used to compile against for unit testing
 $(ENGINE_LIB_NAME): $(ENGINE_SERVER_OBJ)
 	$(SERVER_AR) rcs $@ $^
 
 # valkey-unit-tests
 $(ENGINE_UNIT_TESTS): $(ENGINE_TEST_OBJ) $(ENGINE_LIB_NAME)
-	$(SERVER_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
+	$(SERVER_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ${LUA_LIBS} ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
 
 # valkey-sentinel
 $(ENGINE_SENTINEL_NAME): $(SERVER_NAME)
@@ -561,13 +574,13 @@ distclean: clean
 .PHONY: distclean
 
 test: $(SERVER_NAME) $(ENGINE_CHECK_AOF_NAME) $(ENGINE_CLI_NAME) $(ENGINE_BENCHMARK_NAME)
-	@(cd ..; ./runtest)
+	@(cd ..; ./runtest ${LUA_TEST_FLAGS})
 
 test-unit: $(ENGINE_UNIT_TESTS)
 	./$(ENGINE_UNIT_TESTS)
 
 test-modules: $(SERVER_NAME)
-	@(cd ..; ./runtest-moduleapi)
+	@(cd ..; ./runtest-moduleapi ${LUA_TEST_FLAGS})
 
 test-sentinel: $(ENGINE_SENTINEL_NAME) $(ENGINE_CLI_NAME)
 	@(cd ..; ./runtest-sentinel)

src/defrag.c

@@ -170,6 +170,7 @@ robj *activeDefragStringOb(robj *ob) {
     return activeDefragStringObEx(ob, 1);
 }
 
+#ifdef USE_LUA
 /* Defrag helper for lua scripts
  *
  * returns NULL in case the allocation wasn't moved.
@@ -189,6 +190,7 @@ luaScript *activeDefragLuaScript(luaScript *script) {
 
     return ret;
 }
+#endif
 
 /* Defrag helper for dict main allocations (dict struct, and hash tables).
  * Receives a pointer to the dict* and return a new dict* when the dict
@@ -299,10 +301,12 @@ void activeDefragSdsDict(dict *d, int val_type) {
     dictDefragFunctions defragfns = {
         .defragAlloc = activeDefragAlloc,
         .defragKey = (dictDefragAllocFunction *)activeDefragSds,
-        .defragVal = (val_type == DEFRAG_SDS_DICT_VAL_IS_SDS       ? (dictDefragAllocFunction *)activeDefragSds
-                      : val_type == DEFRAG_SDS_DICT_VAL_IS_STROB   ? (dictDefragAllocFunction *)activeDefragStringOb
-                      : val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR   ? (dictDefragAllocFunction *)activeDefragAlloc
+        .defragVal = (val_type == DEFRAG_SDS_DICT_VAL_IS_SDS     ? (dictDefragAllocFunction *)activeDefragSds
+                      : val_type == DEFRAG_SDS_DICT_VAL_IS_STROB ? (dictDefragAllocFunction *)activeDefragStringOb
+                      : val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR ? (dictDefragAllocFunction *)activeDefragAlloc
+#ifdef USE_LUA
                       : val_type == DEFRAG_SDS_DICT_VAL_LUA_SCRIPT ? (dictDefragAllocFunction *)activeDefragLuaScript
+#endif
                                                                    : NULL)};
     do {
         cursor = dictScanDefrag(d, cursor, activeDefragSdsDictCallback, &defragfns, NULL);
@@ -812,7 +816,9 @@ void defragOtherGlobals(void) {
     /* there are many more pointers to defrag (e.g. client argv, output / aof buffers, etc.
      * but we assume most of these are short lived, we only need to defrag allocations
      * that remain static for a long time */
+#ifdef USE_LUA
     activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_LUA_SCRIPT);
+#endif
     moduleDefragGlobals();
     kvstoreDictLUTDefrag(server.pubsub_channels, dictDefragTables);
     kvstoreDictLUTDefrag(server.pubsubshard_channels, dictDefragTables);

src/eval.c

@@ -35,7 +35,11 @@
  * 2. scriptingInit() - initServer() function from server.c invokes this to initialize LUA at startup.
  *                      It is also invoked between 2 eval invocations to reset Lua.
  */
+
 #include "server.h"
+
+#ifdef USE_LUA
+
 #include "sha1.h"
 #include "rand.h"
 #include "cluster.h"
@@ -764,6 +768,10 @@ dict *evalScriptsDict(void) {
     return lctx.lua_scripts;
 }
 
+unsigned long evalScriptsDictSize(void) {
+    return dictSize(evalScriptsDict());
+}
+
 unsigned long evalScriptsMemory(void) {
     return lctx.lua_scripts_mem + dictMemUsage(lctx.lua_scripts) + dictSize(lctx.lua_scripts) * sizeof(luaScript) +
            listLength(lctx.lua_scripts_lru_list) * sizeof(listNode);
@@ -1756,3 +1764,64 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) {
         rctx->start_time = getMonotonicUs();
     }
 }
+
+#else /* USE_LUA is no */
+
+/* These stubs are used when Lua is disabled at compile time.
+ * They typically do nothing and report 0. */
+
+void scriptingInit(int setup) {
+    UNUSED(setup);
+}
+
+int ldbPendingChildren(void) {
+    return 0;
+}
+
+int ldbRemoveChild(pid_t pid) {
+    UNUSED(pid);
+    return 0;
+}
+
+unsigned long evalMemory(void) {
+    return 0;
+}
+
+unsigned long evalScriptsMemory(void) {
+    return 0;
+}
+
+uint64_t evalGetCommandFlags(client *c, uint64_t cmd_flags) {
+    UNUSED(c);
+    /* Pass through the cmd_flags */
+    return cmd_flags;
+}
+
+unsigned long evalScriptsDictSize(void) {
+    return 0;
+}
+
+void ldbKillForkedSessions(void) {
+}
+
+void evalCommand(client *c) {
+    addReplyError(c, "No default engine loaded");
+}
+
+void evalRoCommand(client *c) {
+    addReplyError(c, "No default engine loaded");
+}
+
+void evalShaCommand(client *c) {
+    addReplyError(c, "No default engine loaded");
+}
+
+void evalShaRoCommand(client *c) {
+    addReplyError(c, "No default engine loaded");
+}
+
+void scriptCommand(client *c) {
+    addReplyError(c, "No default engine loaded");
+}
+
+#endif

src/function_lua.c

@@ -40,6 +40,9 @@
  */
 
 #include "functions.h"
+
+#ifdef USE_LUA
+
 #include "script_lua.h"
 #include <lua.h>
 #include <lauxlib.h>
@@ -507,3 +510,5 @@ int luaEngineInitEngine(void) {
     };
     return functionsRegisterEngine(LUA_ENGINE_NAME, lua_engine);
 }
+
+#endif

src/functions.c

@@ -1114,9 +1114,11 @@ size_t functionsLibCtxFunctionsLen(functionsLibCtx *functions_ctx) {
 int functionsInit(void) {
     engines = dictCreate(&engineDictType);
 
+#ifdef USE_LUA
     if (luaEngineInitEngine() != C_OK) {
         return C_ERR;
     }
+#endif
 
     /* Must be initialized after engines initialization */
     curr_functions_lib_ctx = functionsLibCtxCreate();

src/lazyfree.c

@@ -51,13 +51,17 @@ void lazyFreeErrors(void *args[]) {
 
 /* Release the lua_scripts dict. */
 void lazyFreeLuaScripts(void *args[]) {
+#ifndef USE_LUA
+    UNUSED(args);
+#else
     dict *lua_scripts = args[0];
     list *lua_scripts_lru_list = args[1];
     lua_State *lua = args[2];
     long long len = dictSize(lua_scripts);
     freeLuaScriptsSync(lua_scripts, lua_scripts_lru_list, lua);
     atomic_fetch_sub_explicit(&lazyfree_objects, len, memory_order_relaxed);
     atomic_fetch_add_explicit(&lazyfreed_objects, len, memory_order_relaxed);
+#endif
 }
 
 /* Release the functions ctx. */
@@ -222,6 +226,7 @@ void freeErrorsRadixTreeAsync(rax *errors) {
     }
 }
 
+#ifdef USE_LUA
 /* Free lua_scripts dict and lru list, if the dict is huge enough, free them in async way.
  * Close lua interpreter, if there are a lot of lua scripts, close it in async way. */
 void freeLuaScriptsAsync(dict *lua_scripts, list *lua_scripts_lru_list, lua_State *lua) {
@@ -232,6 +237,7 @@ void freeLuaScriptsAsync(dict *lua_scripts, list *lua_scripts_lru_list, lua_Stat
         freeLuaScriptsSync(lua_scripts, lua_scripts_lru_list, lua);
     }
 }
+#endif
 
 /* Free functions ctx, if the functions ctx contains enough functions, free it in async way. */
 void freeFunctionsAsync(functionsLibCtx *functions_lib_ctx) {

src/object.c

@@ -1306,7 +1306,7 @@ sds getMemoryDoctorReport(void) {
         }
 
         /* Too many scripts are cached? */
-        if (dictSize(evalScriptsDict()) > 1000) {
+        if (evalScriptsDictSize() > 1000) {
             many_scripts = 1;
             num_reports++;
         }

src/script_lua.c

@@ -27,6 +27,8 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#if USE_LUA
+
 #include "script_lua.h"
 #include "fpconv_dtoa.h"
 
@@ -1780,3 +1782,10 @@ void luaCallFunction(scriptRunCtx *run_ctx,
 unsigned long luaMemory(lua_State *lua) {
     return lua_gc(lua, LUA_GCCOUNT, 0) * 1024LL;
 }
+
+#else
+
+/* We need some declaration to prevent compiler warnings */
+#include <math.h>
+
+#endif

src/server.c

@@ -5669,7 +5669,7 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) {
                 "used_memory_vm_eval:%lld\r\n", memory_lua,
                 "used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */
                 "used_memory_scripts_eval:%lld\r\n", (long long)mh->lua_caches,
-                "number_of_cached_scripts:%lu\r\n", dictSize(evalScriptsDict()),
+                "number_of_cached_scripts:%lu\r\n", evalScriptsDictSize(),
                 "number_of_functions:%lu\r\n", functionsNum(),
                 "number_of_libraries:%lu\r\n", functionsLibNum(),
                 "used_memory_vm_functions:%lld\r\n", memory_functions,

src/server.h

@@ -50,7 +50,11 @@
 #include <syslog.h>
 #include <netinet/in.h>
 #include <sys/socket.h>
+
+#ifdef USE_LUA
 #include <lua.h>
+#endif
+
 #include <signal.h>
 
 #ifdef HAVE_LIBSYSTEMD
@@ -3635,30 +3639,42 @@ int redis_check_rdb_main(int argc, char **argv, FILE *fp);
 int redis_check_aof_main(int argc, char **argv);
 
 /* Scripting */
+
 void scriptingInit(int setup);
 int ldbRemoveChild(pid_t pid);
 void ldbKillForkedSessions(void);
 int ldbPendingChildren(void);
+
+unsigned long evalMemory(void);
+unsigned long evalScriptsDictSize(void);
+unsigned long evalScriptsMemory(void);
+uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags);
+
+void freeFunctionsAsync(functionsLibCtx *lib_ctx);
+uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags);
+int isInsideYieldingLongCommand(void);
+
+#ifdef USE_LUA
+/* The functions above are cross-cutting are stubbed when !USE_LUA.
+ * The declarations below are only included when Lua scripting is enabled. */
 void luaLdbLineHook(lua_State *lua, lua_Debug *ar);
 void freeLuaScriptsSync(dict *lua_scripts, list *lua_scripts_lru_list, lua_State *lua);
 void freeLuaScriptsAsync(dict *lua_scripts, list *lua_scripts_lru_list, lua_State *lua);
-void freeFunctionsAsync(functionsLibCtx *lib_ctx);
 int ldbIsEnabled(void);
 void ldbLog(sds entry);
 void ldbLogRespReply(char *reply);
 void sha1hex(char *digest, char *script, size_t len);
 unsigned long evalMemory(void);
 dict *evalScriptsDict(void);
-unsigned long evalScriptsMemory(void);
-uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags);
-uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags);
-int isInsideYieldingLongCommand(void);
 
 typedef struct luaScript {
     uint64_t flags;
     robj *body;
     listNode *node; /* list node in lua_scripts_lru_list list. */
 } luaScript;
+
+#endif
+
 /* Cache of recently used small arguments to avoid malloc calls. */
 #define LUA_CMD_OBJCACHE_SIZE 32
 #define LUA_CMD_OBJCACHE_MAX_LEN 64

tests/README.md

@@ -90,6 +90,7 @@ The following compatibility and capability tags are currently used:
 | `external:skip`           | Not compatible with external servers. |
 | `cluster:skip`            | Not compatible with `--cluster-mode`. |
 | `large-memory`            | Test that requires more than 100MB |
+| `scripting`               | Tests that require a Lua engine. |
 | `tls:skip`                | Not compatible with `--tls`. |
 | `needs:repl`              | Uses replication and needs to be able to `SYNC` from server. |
 | `needs:debug`             | Uses the `DEBUG` command or other debugging focused commands (like `OBJECT REFCOUNT`). |