Fix string passing at interface C++ to JS

lib/CMakeLists.txt

@@ -324,6 +324,7 @@ if(EMSCRIPTEN)
           _duckdb_web_pending_query_cancel, \
           _duckdb_web_pending_query_poll, \
           _duckdb_web_pending_query_start, \
+          _duckdb_web_pending_query_start_buffer, \
           _duckdb_web_prepared_close, \
           _duckdb_web_prepared_create, \
           _duckdb_web_prepared_run, \
@@ -333,6 +334,7 @@ if(EMSCRIPTEN)
           _duckdb_web_query_run_buffer, \
           _duckdb_web_reset, \
           _duckdb_web_tokenize, \
+          _duckdb_web_tokenize_buffer, \
           _duckdb_web_udf_scalar_create \
       ]' \
       -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \

lib/src/webdb_api.cc

@@ -151,6 +151,13 @@ void duckdb_web_tokenize(WASMResponse* packed, const char* query) {
     auto tokens = webdb.Tokenize(query);
     WASMResponseBuffer::Get().Store(*packed, arrow::Result(std::move(tokens)));
 }
+/// Tokenize a query
+void duckdb_web_tokenize_buffer(WASMResponse* packed, const uint8_t* buffer, size_t buffer_length) {
+    GET_WEBDB(*packed);
+    std::string_view query(reinterpret_cast<const char*>(buffer), buffer_length);
+    auto tokens = webdb.Tokenize(query);
+    WASMResponseBuffer::Get().Store(*packed, arrow::Result(std::move(tokens)));
+}
 /// Create scalar UDF queries
 void duckdb_web_udf_scalar_create(WASMResponse* packed, ConnectionHdl connHdl, const char* args) {
     auto c = reinterpret_cast<WebDB::Connection*>(connHdl);
@@ -203,6 +210,14 @@ void duckdb_web_pending_query_start(WASMResponse* packed, ConnectionHdl connHdl,
     auto r = c->PendingQuery(script, allow_stream_result);
     WASMResponseBuffer::Get().Store(*packed, std::move(r));
 }
+/// Start a pending query
+void duckdb_web_pending_query_start_buffer(WASMResponse* packed, ConnectionHdl connHdl, const uint8_t* buffer,
+                                           size_t buffer_length, bool allow_stream_result) {
+    auto c = reinterpret_cast<WebDB::Connection*>(connHdl);
+    std::string_view S(reinterpret_cast<const char*>(buffer), buffer_length);
+    auto r = c->PendingQuery(S, allow_stream_result);
+    WASMResponseBuffer::Get().Store(*packed, std::move(r));
+}
 /// Poll a pending query
 void duckdb_web_pending_query_poll(WASMResponse* packed, ConnectionHdl connHdl, const char* script) {
     auto c = reinterpret_cast<WebDB::Connection*>(connHdl);

packages/duckdb-wasm/src/bindings/bindings_base.ts

@@ -134,11 +134,17 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
 
     /** Tokenize a script */
     public tokenize(text: string): ScriptTokens {
-        const [s, d, n] = callSRet(this.mod, 'duckdb_web_tokenize', ['string'], [text]);
+        const BUF = TEXT_ENCODER.encode(text);
+        const bufferPtr = this.mod._malloc(BUF.length );
+        const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length );
+        bufferOfs.set(BUF);
+        const [s, d, n] = callSRet(this.mod, 'duckdb_web_tokenize_buffer', ['number', 'number'], [bufferPtr, BUF.length]);
         if (s !== StatusCode.SUCCESS) {
+            this.mod._free(bufferPtr);
             throw new Error(readString(this.mod, d, n));
         }
         const res = readString(this.mod, d, n);
+        this.mod._free(bufferPtr);
         dropResponseBuffers(this.mod);
         return JSON.parse(res) as ScriptTokens;
     }
@@ -169,6 +175,7 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
         bufferOfs.set(BUF);
         const [s, d, n] = callSRet(this.mod, 'duckdb_web_query_run_buffer', ['number', 'number', 'number'], [conn, bufferPtr, BUF.length]);
         if (s !== StatusCode.SUCCESS) {
+            this.mod._free(bufferPtr);
             throw new Error(readString(this.mod, d, n));
         }
         const res = copyBuffer(this.mod, d, n);
@@ -183,19 +190,21 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
      *  Results can then be fetched using `fetchQueryResults`
      */
     public startPendingQuery(conn: number, text: string, allowStreamResult: boolean = false): Uint8Array | null {
-        const [s, d, n] = callSRet(
-            this.mod,
-            'duckdb_web_pending_query_start',
-            ['number', 'string', 'boolean'],
-            [conn, text, allowStreamResult],
-        );
+        const BUF = TEXT_ENCODER.encode(text);
+        const bufferPtr = this.mod._malloc(BUF.length );
+        const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length );
+        bufferOfs.set(BUF);
+        const [s, d, n] = callSRet(this.mod, 'duckdb_web_pending_query_start_buffer', ['number', 'number', 'number', 'boolean'], [conn, bufferPtr, BUF.length, allowStreamResult]);
         if (s !== StatusCode.SUCCESS) {
+            this.mod._free(bufferPtr);
             throw new Error(readString(this.mod, d, n));
         }
         if (d == 0) {
+            this.mod._free(bufferPtr);
             return null;
         }
         const res = copyBuffer(this.mod, d, n);
+        this.mod._free(bufferPtr);
         dropResponseBuffers(this.mod);
         return res;
     }