Skip to content

Commit 901eba4

Browse files
henrypark133claude
andauthored
fix(ci): WASM WIT compat sqlite3 duplicate symbol conflict (nearai#953)
* fix(ci): use explicit features in WASM WIT compat test to avoid sqlite3 symbol conflicts The `import` feature (added in nearai#903) brings in `rusqlite[bundled]` which conflicts with `libsql-ffi` — both bundle SQLite C code, causing duplicate symbol linker errors. Use explicit features matching the test matrix instead of `--all-features`. Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: replace rusqlite with libsql in import module to fix sqlite3 symbol conflict The `import` feature used `rusqlite[bundled]` which bundled its own SQLite C code, conflicting with `libsql-ffi` (also bundles SQLite). This caused duplicate `sqlite3_*` symbol linker errors when both features were enabled via `--all-features`. Replace `rusqlite` with `libsql` (already a dependency) in the import reader. The `import` feature now implies `libsql`. This eliminates the duplicate symbol conflict and allows `--all-features` to compile cleanly. Also restores `--all-features` in the WASM WIT compat CI test (now safe) and converts all import test helpers from rusqlite to libsql. Co-Authored-By: Claude Opus 4.6 <[email protected]> * style: apply cargo fmt formatting fixes Co-Authored-By: Claude Opus 4.6 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]>
1 parent dff9a38 commit 901eba4

9 files changed

Lines changed: 434 additions & 365 deletions

Cargo.lock

Lines changed: 1 addition & 36 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ ed25519-dalek = { version = "2.2.0", features = ["std"] }
176176
hex = "0.4.3"
177177

178178
# OpenClaw import (feature gated)
179-
rusqlite = { version = "0.32", optional = true, features = ["bundled"] }
180179
json5 = { version = "0.4", optional = true }
181180

182181
# macOS keychain
@@ -214,7 +213,7 @@ libsql = ["dep:libsql"]
214213
integration = []
215214
html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"]
216215
bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"]
217-
import = ["dep:rusqlite", "dep:json5"]
216+
import = ["dep:json5", "libsql"]
218217

219218
[[test]]
220219
name = "html_to_markdown"

src/import/openclaw/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ impl OpenClawImporter {
7777
// Pre-read all conversation data to validate before writing
7878
let mut all_conversations = Vec::new();
7979
for (_agent_name, db_path) in &agent_dbs {
80-
match reader.read_conversations(db_path) {
80+
match reader.read_conversations(db_path).await {
8181
Ok(convs) => all_conversations.extend(convs),
8282
Err(e) => {
8383
tracing::warn!("Failed to read conversations: {}", e);
@@ -88,7 +88,7 @@ impl OpenClawImporter {
8888
// Pre-read all memory chunks
8989
let mut all_chunks = Vec::new();
9090
for (_agent_name, db_path) in &agent_dbs {
91-
match reader.read_memory_chunks(db_path) {
91+
match reader.read_memory_chunks(db_path).await {
9292
Ok(chunks) => all_chunks.extend(chunks),
9393
Err(e) => {
9494
tracing::warn!("Failed to read memory chunks: {}", e);

src/import/openclaw/reader.rs

Lines changed: 101 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,16 @@ pub struct OpenClawMessage {
8080
pub created_at: Option<chrono::DateTime<chrono::Utc>>,
8181
}
8282

83+
/// Open an OpenClaw SQLite database file via libsql for read-only access.
84+
#[cfg(feature = "import")]
85+
async fn open_sqlite(db_path: &Path) -> Result<libsql::Connection, ImportError> {
86+
let db = libsql::Builder::new_local(db_path)
87+
.build()
88+
.await
89+
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
90+
db.connect().map_err(|e| ImportError::Sqlite(e.to_string()))
91+
}
92+
8393
/// Reader for OpenClaw data files and databases.
8494
pub struct OpenClawReader {
8595
openclaw_dir: PathBuf,
@@ -226,115 +236,123 @@ impl OpenClawReader {
226236

227237
/// Read all memory chunks from an OpenClaw SQLite database.
228238
#[cfg(feature = "import")]
229-
pub fn read_memory_chunks(
239+
pub async fn read_memory_chunks(
230240
&self,
231241
db_path: &Path,
232242
) -> Result<Vec<OpenClawMemoryChunk>, ImportError> {
233-
use rusqlite::Connection;
234-
235-
let conn = Connection::open(db_path).map_err(|e| ImportError::Sqlite(e.to_string()))?;
236-
237-
let mut stmt = conn
238-
.prepare("SELECT path, content, embedding, chunk_index FROM chunks")
239-
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
243+
let conn = open_sqlite(db_path).await?;
240244

241-
let chunks = stmt
242-
.query_map([], |row| {
243-
let path: String = row.get(0)?;
244-
let content: String = row.get(1)?;
245-
let embedding_bytes: Option<Vec<u8>> = row.get(2)?;
246-
let chunk_index: i32 = row.get(3)?;
247-
248-
// Convert binary embedding blob to Vec<f32> if present
249-
let embedding = embedding_bytes.map(|bytes| {
250-
bytes
251-
.chunks(4)
252-
.map(|chunk| {
253-
if chunk.len() == 4 {
254-
f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])
255-
} else {
256-
0.0
257-
}
258-
})
259-
.collect()
260-
});
261-
262-
Ok(OpenClawMemoryChunk {
263-
path,
264-
content,
265-
embedding,
266-
chunk_index,
267-
})
268-
})
245+
let mut rows = conn
246+
.query(
247+
"SELECT path, content, embedding, chunk_index FROM chunks",
248+
(),
249+
)
250+
.await
269251
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
270252

271253
let mut result = Vec::new();
272-
for chunk_result in chunks {
273-
result.push(chunk_result.map_err(|e| ImportError::Sqlite(e.to_string()))?);
254+
while let Some(row) = rows
255+
.next()
256+
.await
257+
.map_err(|e| ImportError::Sqlite(e.to_string()))?
258+
{
259+
let path: String = row.get(0).map_err(|e| ImportError::Sqlite(e.to_string()))?;
260+
let content: String = row.get(1).map_err(|e| ImportError::Sqlite(e.to_string()))?;
261+
let embedding_blob: Option<Vec<u8>> =
262+
row.get(2).map_err(|e| ImportError::Sqlite(e.to_string()))?;
263+
let chunk_index: i32 = row.get(3).map_err(|e| ImportError::Sqlite(e.to_string()))?;
264+
265+
// Convert binary embedding blob to Vec<f32> if present
266+
let embedding = embedding_blob.map(|bytes| {
267+
bytes
268+
.chunks(4)
269+
.map(|chunk| {
270+
if chunk.len() == 4 {
271+
f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])
272+
} else {
273+
0.0
274+
}
275+
})
276+
.collect()
277+
});
278+
279+
result.push(OpenClawMemoryChunk {
280+
path,
281+
content,
282+
embedding,
283+
chunk_index,
284+
});
274285
}
275286

276287
Ok(result)
277288
}
278289

279290
/// Read all conversations from an OpenClaw SQLite database.
280291
#[cfg(feature = "import")]
281-
pub fn read_conversations(
292+
pub async fn read_conversations(
282293
&self,
283294
db_path: &Path,
284295
) -> Result<Vec<OpenClawConversation>, ImportError> {
285-
use rusqlite::Connection;
296+
let conn = open_sqlite(db_path).await?;
286297

287-
let conn = Connection::open(db_path).map_err(|e| ImportError::Sqlite(e.to_string()))?;
288-
289-
// First, read all conversations
290-
let mut conv_stmt = conn
291-
.prepare("SELECT id, channel, created_at FROM conversations ORDER BY created_at DESC")
298+
let mut conv_rows = conn
299+
.query(
300+
"SELECT id, channel, created_at FROM conversations ORDER BY created_at DESC",
301+
(),
302+
)
303+
.await
292304
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
293305

294306
let mut conversations = Vec::new();
295-
let conv_rows = conv_stmt
296-
.query_map([], |row| {
297-
let id: String = row.get(0)?;
298-
let channel: String = row.get(1)?;
299-
let created_at: Option<String> = row.get(2)?;
300-
301-
let created_at = created_at
302-
.and_then(|s| chrono::DateTime::parse_from_rfc3339(&s).ok())
303-
.map(|dt| dt.with_timezone(&chrono::Utc));
304-
305-
Ok((id, channel, created_at))
306-
})
307-
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
307+
while let Some(row) = conv_rows
308+
.next()
309+
.await
310+
.map_err(|e| ImportError::Sqlite(e.to_string()))?
311+
{
312+
let id: String = row.get(0).map_err(|e| ImportError::Sqlite(e.to_string()))?;
313+
let channel: String = row.get(1).map_err(|e| ImportError::Sqlite(e.to_string()))?;
314+
let created_at: Option<String> =
315+
row.get(2).map_err(|e| ImportError::Sqlite(e.to_string()))?;
308316

309-
for row_result in conv_rows {
310-
let (id, channel, created_at) =
311-
row_result.map_err(|e| ImportError::Sqlite(e.to_string()))?;
317+
let created_at = created_at
318+
.and_then(|s| chrono::DateTime::parse_from_rfc3339(&s).ok())
319+
.map(|dt| dt.with_timezone(&chrono::Utc));
312320

313321
// Read messages for this conversation
314-
let mut msg_stmt = conn.prepare(
315-
"SELECT role, content, created_at FROM messages WHERE conversation_id = ? ORDER BY created_at"
316-
)
317-
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
318-
319-
let messages = msg_stmt
320-
.query_map([&id], |row| {
321-
let role: String = row.get(0)?;
322-
let content: String = row.get(1)?;
323-
let created_at: Option<String> = row.get(2)?;
324-
325-
let created_at = created_at
326-
.and_then(|s| chrono::DateTime::parse_from_rfc3339(&s).ok())
327-
.map(|dt| dt.with_timezone(&chrono::Utc));
322+
let mut msg_rows = conn
323+
.query(
324+
"SELECT role, content, created_at FROM messages WHERE conversation_id = ?1 ORDER BY created_at",
325+
libsql::params![id.as_str()],
326+
)
327+
.await
328+
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
328329

329-
Ok(OpenClawMessage {
330-
role,
331-
content,
332-
created_at,
333-
})
334-
})
330+
let mut messages = Vec::new();
331+
while let Some(msg_row) = msg_rows
332+
.next()
333+
.await
335334
.map_err(|e| ImportError::Sqlite(e.to_string()))?
336-
.collect::<Result<Vec<_>, _>>()
337-
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
335+
{
336+
let role: String = msg_row
337+
.get(0)
338+
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
339+
let content: String = msg_row
340+
.get(1)
341+
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
342+
let msg_created_at: Option<String> = msg_row
343+
.get(2)
344+
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
345+
346+
let msg_created_at = msg_created_at
347+
.and_then(|s| chrono::DateTime::parse_from_rfc3339(&s).ok())
348+
.map(|dt| dt.with_timezone(&chrono::Utc));
349+
350+
messages.push(OpenClawMessage {
351+
role,
352+
content,
353+
created_at: msg_created_at,
354+
});
355+
}
338356

339357
conversations.push(OpenClawConversation {
340358
id,

0 commit comments

Comments
 (0)