From 6650a95cf35545ce1427837ba7f0efa147a38f63 Mon Sep 17 00:00:00 2001 From: Pontus Abrahamsson Date: Tue, 10 Mar 2026 10:09:05 +0100 Subject: [PATCH] wip --- apps/api/src/trpc/routers/search.ts | 107 +++++---- .../src/components/select-attachment.tsx | 15 +- packages/db/src/queries/inbox.ts | 209 +++++------------- .../db/src/queries/transaction-matching.ts | 90 +++++--- packages/db/src/queries/transactions.ts | 178 ++------------- 5 files changed, 191 insertions(+), 408 deletions(-) diff --git a/apps/api/src/trpc/routers/search.ts b/apps/api/src/trpc/routers/search.ts index 30695b9bff..5820f03d7b 100644 --- a/apps/api/src/trpc/routers/search.ts +++ b/apps/api/src/trpc/routers/search.ts @@ -62,63 +62,62 @@ export const searchRouter = createTRPCRouter({ .query(async ({ input, ctx: { db, teamId } }) => { const { q, transactionId, limit = 30 } = input; - const [inboxResults, invoiceResults] = await Promise.all([ - getInboxSearch(db, { - teamId: teamId!, - q: q ?? undefined, - transactionId: transactionId ?? undefined, - limit: limit, - }), - getInvoices(db, { - teamId: teamId!, - q: q ?? undefined, - statuses: ["unpaid", "overdue", "paid"], - pageSize: limit, - sort: null, - }), - ]); + const inboxResults = await getInboxSearch(db, { + teamId: teamId!, + q: q ?? undefined, + transactionId: transactionId ?? undefined, + limit, + minConfidence: !q && transactionId ? 0.9 : undefined, + }); + + const inboxItems = inboxResults.map((item) => ({ + type: "inbox" as const, + id: item!.id, + fileName: item!.fileName ?? null, + filePath: item!.filePath ?? [], + displayName: item!.displayName ?? null, + amount: item!.amount ?? null, + currency: item!.currency ?? null, + contentType: item!.contentType ?? null, + date: item!.date ?? null, + size: item!.size ?? null, + description: item!.description ?? null, + status: item!.status ?? null, + website: item!.website ?? null, + baseAmount: item!.baseAmount ?? null, + baseCurrency: item!.baseCurrency ?? null, + taxAmount: item!.taxAmount ?? null, + taxRate: item!.taxRate ?? null, + taxType: item!.taxType ?? null, + createdAt: item!.createdAt, + })); - // Transform inbox results - const inboxItems = - inboxResults.map((item) => ({ - type: "inbox" as const, - id: item.id, - fileName: item.fileName ?? null, - filePath: item.filePath ?? [], - displayName: item.displayName ?? null, - amount: item.amount ?? null, - currency: item.currency ?? null, - contentType: item.contentType ?? null, - date: item.date ?? null, - size: item.size ?? null, - description: item.description ?? null, - status: item.status ?? null, - website: item.website ?? null, - baseAmount: item.baseAmount ?? null, - baseCurrency: item.baseCurrency ?? null, - taxAmount: item.taxAmount ?? null, - taxRate: item.taxRate ?? null, - taxType: item.taxType ?? null, - createdAt: item.createdAt, - })) ?? []; + if (!q && transactionId) { + return inboxItems; + } + + const invoiceResults = await getInvoices(db, { + teamId: teamId!, + q: q ?? undefined, + statuses: ["unpaid", "overdue", "paid"], + pageSize: limit, + sort: null, + }); - // Transform invoice results - const invoices = - invoiceResults.data.map((invoice) => ({ - type: "invoice" as const, - id: invoice.id, - invoiceNumber: invoice.invoiceNumber ?? null, - customerName: invoice.customerName ?? null, - amount: invoice.amount ?? null, - currency: invoice.currency ?? null, - filePath: invoice.filePath ?? [], - dueDate: invoice.dueDate ?? null, - status: invoice.status, - size: invoice.fileSize ?? null, - createdAt: invoice.createdAt, - })) ?? []; + const invoices = invoiceResults.data.map((invoice) => ({ + type: "invoice" as const, + id: invoice.id, + invoiceNumber: invoice.invoiceNumber ?? null, + customerName: invoice.customerName ?? null, + amount: invoice.amount ?? null, + currency: invoice.currency ?? null, + filePath: invoice.filePath ?? [], + dueDate: invoice.dueDate ?? null, + status: invoice.status, + size: invoice.fileSize ?? null, + createdAt: invoice.createdAt, + })); - // Combine and return results return [...inboxItems, ...invoices]; }), }); diff --git a/apps/dashboard/src/components/select-attachment.tsx b/apps/dashboard/src/components/select-attachment.tsx index d643a38f80..c6e7530e6c 100644 --- a/apps/dashboard/src/components/select-attachment.tsx +++ b/apps/dashboard/src/components/select-attachment.tsx @@ -33,19 +33,21 @@ export function SelectAttachment({ }: Props) { const [debouncedValue, setDebouncedValue] = useDebounceValue("", 200); const [isOpen, setIsOpen] = useState(false); + const [hasFocused, setHasFocused] = useState(false); const { data: user } = useUserQuery(); const { setParams } = useDocumentParams(); const trpc = useTRPC(); - // Only fetch suggestions when user is actively searching (not just on focus) + const isSearching = debouncedValue.length > 0; + const { data: items, isLoading } = useQuery({ ...trpc.search.attachments.queryOptions({ - q: debouncedValue.length > 0 ? debouncedValue : undefined, + q: isSearching ? debouncedValue : undefined, transactionId, - limit: debouncedValue.length > 0 ? 30 : 3, + limit: isSearching ? 30 : 1, }), - enabled: Boolean(debouncedValue.length > 0 || transactionId), // Enable for search OR suggestions + enabled: Boolean(isSearching || (transactionId && hasFocused)), }); const handleOnSelect = (item: Attachment) => { @@ -104,7 +106,7 @@ export function SelectAttachment({ } const showBestMatch = - !!transactionId && index === 0 && items?.length > 1; + !!transactionId && index === 0 && items.length > 1; return { id: item.id, @@ -166,7 +168,8 @@ export function SelectAttachment({ : []; const handleFocus = () => { - if (!isOpen && !debouncedValue) { + setHasFocused(true); + if (!isOpen) { setIsOpen(true); } }; diff --git a/packages/db/src/queries/inbox.ts b/packages/db/src/queries/inbox.ts index 10ebbbfb18..c811a2531a 100644 --- a/packages/db/src/queries/inbox.ts +++ b/packages/db/src/queries/inbox.ts @@ -22,6 +22,7 @@ import { calculateNameScore as calculateUnifiedNameScore, scoreMatch, } from "../utils/transaction-matching"; +import { findInboxTopMatches } from "./transaction-matching"; export type GetInboxParams = { teamId: string; @@ -690,8 +691,9 @@ export async function deleteInboxMany( export type GetInboxSearchParams = { teamId: string; limit?: number; - q?: string; // Search query (text or amount) - transactionId?: string; // For AI suggestions + q?: string; + transactionId?: string; + minConfidence?: number; }; export async function getInboxSearch( @@ -699,7 +701,7 @@ export async function getInboxSearch( params: GetInboxSearchParams, ) { try { - const { teamId, q, transactionId, limit = 10 } = params; + const { teamId, q, transactionId, limit = 10, minConfidence } = params; const whereConditions: SQL[] = [ eq(inbox.teamId, teamId), @@ -867,163 +869,62 @@ export async function getInboxSearch( // PRIORITY 2: AI suggestions for transaction if (transactionId) { - // Get transaction details for context-aware matching - const transactionData = await db - .select({ - id: transactions.id, - name: transactions.name, - amount: transactions.amount, - currency: transactions.currency, - baseAmount: transactions.baseAmount, - baseCurrency: transactions.baseCurrency, - date: transactions.date, - merchantName: transactions.merchantName, - counterpartyName: transactions.counterpartyName, - description: transactions.description, - }) - .from(transactions) - .where( - and( - eq(transactions.id, transactionId), - eq(transactions.teamId, teamId), - ), - ) - .limit(1); - - if (transactionData.length > 0) { - const transaction = transactionData[0]!; + const matches = await findInboxTopMatches(db, { + teamId, + transactionId, + limit, + }); - // Check if transaction already has attachments - if so, don't show suggestions - const [hasAttachments] = await db - .select({ count: sql`count(*)` }) - .from(transactionAttachments) - .where( - and( - eq(transactionAttachments.transactionId, transactionId), - eq(transactionAttachments.teamId, teamId), - ), - ); + if (matches.length === 0) return []; - const attachmentCount = hasAttachments?.count - ? Number(hasAttachments.count) - : 0; + const filtered = minConfidence + ? matches.filter((m) => m.confidenceScore >= minConfidence) + : matches; - if (attachmentCount > 0) { - return []; - } + if (filtered.length === 0) return []; - const unifiedTransactionAmount = Math.abs(transaction.amount || 0); - const unifiedTransactionBaseAmount = Math.abs( - transaction.baseAmount || 0, - ); - const unifiedCandidates = await db.transaction(async (tx) => { - await tx.execute( - sql`SET LOCAL pg_trgm.word_similarity_threshold = 0.3`, - ); - return tx - .select({ - id: inbox.id, - createdAt: inbox.createdAt, - fileName: inbox.fileName, - amount: inbox.amount, - currency: inbox.currency, - filePath: inbox.filePath, - contentType: inbox.contentType, - date: inbox.date, - displayName: inbox.displayName, - size: inbox.size, - description: inbox.description, - baseAmount: inbox.baseAmount, - baseCurrency: inbox.baseCurrency, - status: inbox.status, - type: inbox.type, - website: inbox.website, - taxAmount: inbox.taxAmount, - taxRate: inbox.taxRate, - taxType: inbox.taxType, - }) - .from(inbox) - .where( - and( - ...whereConditions, - sql`${inbox.date} IS NOT NULL`, - sql`${inbox.date} BETWEEN (${sql.param(transaction.date)}::date - INTERVAL '123 days') - AND (${sql.param(transaction.date)}::date + INTERVAL '30 days')`, - or( - and( - eq(inbox.currency, transaction.currency || ""), - sql`ABS(ABS(COALESCE(${inbox.amount}, 0)) - ${unifiedTransactionAmount}) < GREATEST(1, ${unifiedTransactionAmount} * 0.25)`, - ), - sql`(${transaction.merchantName || transaction.name} %> ${inbox.displayName})`, - and( - eq(inbox.baseCurrency, transaction.baseCurrency || ""), - sql`${inbox.baseCurrency} IS NOT NULL`, - sql`ABS(ABS(COALESCE(${inbox.baseAmount}, 0)) - ${unifiedTransactionBaseAmount}) < GREATEST(50, ${unifiedTransactionBaseAmount} * 0.15)`, - ), - ), - ), - ) - .orderBy( - sql`word_similarity(${transaction.merchantName || transaction.name}, COALESCE(${inbox.displayName}, '')) DESC`, - sql`ABS(ABS(COALESCE(${inbox.amount}, 0)) - ${unifiedTransactionAmount}) / GREATEST(1.0, ${unifiedTransactionAmount})`, - sql`ABS(${inbox.date} - ${sql.param(transaction.date)}::date)`, - ) - .limit(Math.max(limit * 3, 30)); - }); + const matchedIds = filtered.map((m) => m.inboxId); + const fullItems = await db + .select({ + id: inbox.id, + createdAt: inbox.createdAt, + fileName: inbox.fileName, + amount: inbox.amount, + currency: inbox.currency, + filePath: inbox.filePath, + contentType: inbox.contentType, + date: inbox.date, + displayName: inbox.displayName, + size: inbox.size, + description: inbox.description, + status: inbox.status, + website: inbox.website, + baseAmount: inbox.baseAmount, + baseCurrency: inbox.baseCurrency, + taxAmount: inbox.taxAmount, + taxRate: inbox.taxRate, + taxType: inbox.taxType, + type: inbox.type, + }) + .from(inbox) + .where(inArray(inbox.id, matchedIds)); - const unifiedScored = unifiedCandidates - .map((candidate) => { - const nameScore = calculateUnifiedNameScore( - candidate.displayName, - transaction.name, - transaction.merchantName || transaction.counterpartyName, - ); - const amountScore = calculateUnifiedAmountScore( - candidate, - transaction, - ); - const currencyScore = calculateUnifiedCurrencyScore( - candidate.currency || undefined, - transaction.currency || undefined, - candidate.baseCurrency || undefined, - transaction.baseCurrency || undefined, - ); - const dateScore = calculateUnifiedDateScore( - candidate.date!, - transaction.date, - candidate.type, - ); - const isExactAmount = - candidate.amount !== null && - Math.abs( - Math.abs(candidate.amount || 0) - - Math.abs(transaction.amount || 0), - ) < 0.01; - const isSameCurrency = candidate.currency === transaction.currency; - const confidence = scoreMatch({ - nameScore, - amountScore, - dateScore, - currencyScore, - isSameCurrency, - isExactAmount, - }); - - return { - ...candidate, - nameScore, - amountScore, - currencyScore, - dateScore, - confidenceScore: confidence, - }; - }) - .filter((candidate) => candidate.confidenceScore >= 0.6) - .sort((a, b) => b.confidenceScore - a.confidenceScore) - .slice(0, limit); + const itemMap = new Map(fullItems.map((item) => [item.id, item])); - return unifiedScored; - } + return filtered + .map((m) => { + const item = itemMap.get(m.inboxId); + if (!item) return null; + return { + ...item, + nameScore: m.nameScore ?? 0, + amountScore: m.amountScore, + currencyScore: m.currencyScore, + dateScore: m.dateScore, + confidenceScore: m.confidenceScore, + }; + }) + .filter(Boolean); } // PRIORITY 3: Recent unmatched items diff --git a/packages/db/src/queries/transaction-matching.ts b/packages/db/src/queries/transaction-matching.ts index 8f08de66d9..39a456d01e 100644 --- a/packages/db/src/queries/transaction-matching.ts +++ b/packages/db/src/queries/transaction-matching.ts @@ -607,11 +607,14 @@ function resolveMatchType( return "suggested"; } -export async function findMatches( +export async function findTopMatches( db: Database, - params: FindMatchesParams & { excludeTransactionIds?: Set }, -): Promise { - const { teamId, inboxId, excludeTransactionIds } = params; + params: FindMatchesParams & { + excludeTransactionIds?: Set; + limit?: number; + }, +): Promise { + const { teamId, inboxId, excludeTransactionIds, limit = 1 } = params; const [calibration, [inboxItem]] = await Promise.all([ getTeamCalibration(db, teamId), @@ -639,7 +642,7 @@ export async function findMatches( ); const autoThreshold = calibration.calibratedAutoThreshold; - if (!inboxItem?.date) return null; + if (!inboxItem?.date) return []; const normalizedInboxName = normalizeNameForLearning(inboxItem.displayName); const inboxAmount = Math.abs(inboxItem.amount || 0); @@ -830,26 +833,37 @@ export async function findMatches( scoredCandidates.map((c) => c.transactionId), ); - for (const candidate of scoredCandidates) { - if (dismissedTxIds.has(candidate.transactionId)) { - logger.info("Skipping dismissed match candidate, trying next", { - teamId, - inboxId, - transactionId: candidate.transactionId, - }); - continue; - } - return candidate; - } + return scoredCandidates + .filter((c) => { + if (dismissedTxIds.has(c.transactionId)) { + logger.info("Skipping dismissed match candidate, trying next", { + teamId, + inboxId, + transactionId: c.transactionId, + }); + return false; + } + return true; + }) + .slice(0, limit); +} - return null; +export async function findMatches( + db: Database, + params: FindMatchesParams & { excludeTransactionIds?: Set }, +): Promise { + const results = await findTopMatches(db, { ...params, limit: 1 }); + return results[0] ?? null; } -export async function findInboxMatches( +export async function findInboxTopMatches( db: Database, - params: FindInboxMatchesParams & { excludeInboxIds?: Set }, -): Promise { - const { teamId, transactionId, excludeInboxIds } = params; + params: FindInboxMatchesParams & { + excludeInboxIds?: Set; + limit?: number; + }, +): Promise { + const { teamId, transactionId, excludeInboxIds, limit = 1 } = params; const [calibration, [transactionItem]] = await Promise.all([ getTeamCalibration(db, teamId), @@ -882,7 +896,7 @@ export async function findInboxMatches( ); const autoThreshold = calibration.calibratedAutoThreshold; - if (!transactionItem?.date) return null; + if (!transactionItem?.date) return []; const normalizedTransactionName = normalizeNameForLearning( transactionItem.merchantName || transactionItem.name, @@ -1046,19 +1060,27 @@ export async function findInboxMatches( scoredCandidates.map((c) => c.inboxId), ); - for (const candidate of scoredCandidates) { - if (dismissedInboxIds.has(candidate.inboxId)) { - logger.info("Skipping dismissed reverse match candidate, trying next", { - teamId, - transactionId, - inboxId: candidate.inboxId, - }); - continue; - } - return candidate; - } + return scoredCandidates + .filter((c) => { + if (dismissedInboxIds.has(c.inboxId)) { + logger.info("Skipping dismissed reverse match candidate, trying next", { + teamId, + transactionId, + inboxId: c.inboxId, + }); + return false; + } + return true; + }) + .slice(0, limit); +} - return null; +export async function findInboxMatches( + db: Database, + params: FindInboxMatchesParams & { excludeInboxIds?: Set }, +): Promise { + const results = await findInboxTopMatches(db, { ...params, limit: 1 }); + return results[0] ?? null; } export async function createMatchSuggestion( diff --git a/packages/db/src/queries/transactions.ts b/packages/db/src/queries/transactions.ts index 7b94f56475..076d3cdec6 100644 --- a/packages/db/src/queries/transactions.ts +++ b/packages/db/src/queries/transactions.ts @@ -46,6 +46,7 @@ import { } from "../utils/transaction-matching"; import { createActivity } from "./activities"; import { type Attachment, createAttachments } from "./transaction-attachments"; +import { findTopMatches } from "./transaction-matching"; const logger = createLoggerWithContext("transactions"); @@ -1146,7 +1147,6 @@ export async function searchTransactionMatch( query, inboxId, maxResults = 5, - minConfidenceScore = 0.5, includeAlreadyMatched = false, } = params; @@ -1336,167 +1336,25 @@ export async function searchTransactionMatch( if (inboxId) { try { - const inboxItem = await db - .select({ - id: inbox.id, - displayName: inbox.displayName, - amount: inbox.amount, - currency: inbox.currency, - date: inbox.date, - baseAmount: inbox.baseAmount, - baseCurrency: inbox.baseCurrency, - }) - .from(inbox) - .where(and(eq(inbox.id, inboxId), eq(inbox.teamId, teamId))) - .limit(1); - - if (!inboxItem.length) { - return []; - } - - const item = inboxItem[0]!; - const inboxAmount = Math.abs(item.amount || 0); - const inboxBaseAmount = Math.abs(item.baseAmount || 0); - - const candidateTransactions = await db.transaction(async (tx) => { - await tx.execute( - sql`SET LOCAL pg_trgm.word_similarity_threshold = 0.3`, - ); - return tx - .select({ - transactionId: transactions.id, - name: transactions.name, - transactionAmount: transactions.amount, - transactionCurrency: transactions.currency, - transactionDate: transactions.date, - baseAmount: transactions.baseAmount, - baseCurrency: transactions.baseCurrency, - isAlreadyMatched: sql` - (EXISTS (SELECT 1 FROM ${transactionAttachments} WHERE ${eq(transactionAttachments.transactionId, transactions.id)} AND ${eq(transactionAttachments.teamId, teamId)}) OR ${transactions.status} = 'completed') - `.as("is_already_matched"), - attachmentFilename: sql` - (SELECT ${transactionAttachments.name} FROM ${transactionAttachments} - WHERE ${eq(transactionAttachments.transactionId, transactions.id)} AND ${eq(transactionAttachments.teamId, teamId)} - LIMIT 1) - `.as("attachment_filename"), - merchantName: transactions.merchantName, - }) - .from(transactions) - .where( - and( - eq(transactions.teamId, teamId), - eq(transactions.status, "posted"), - sql`${transactions.date} IS NOT NULL`, - sql`${transactions.date} BETWEEN ${item.date}::date - INTERVAL '90 days' AND ${item.date}::date + INTERVAL '30 days'`, - ...(includeAlreadyMatched - ? [] - : [ - sql`NOT (EXISTS (SELECT 1 FROM ${transactionAttachments} WHERE ${eq(transactionAttachments.transactionId, transactions.id)} AND ${eq(transactionAttachments.teamId, teamId)}) OR ${transactions.status} = 'completed')`, - ]), - or( - and( - eq(transactions.currency, item.currency ?? ""), - sql`ABS(ABS(${transactions.amount}) - ${inboxAmount}) < GREATEST(1, ${inboxAmount} * 0.25)`, - ), - sql`(${item.displayName ?? ""} %> ${transactions.name} OR ${item.displayName ?? ""} %> ${transactions.merchantName})`, - and( - sql`${transactions.baseCurrency} IS NOT NULL`, - sql`${item.baseCurrency ?? ""} != ''`, - eq(transactions.baseCurrency, item.baseCurrency ?? ""), - sql`ABS(ABS(COALESCE(${transactions.baseAmount}, 0)) - ${inboxBaseAmount}) < GREATEST(50, ${inboxBaseAmount} * 0.15)`, - ), - ), - ), - ) - .orderBy( - sql`GREATEST(word_similarity(${item.displayName ?? ""}, ${transactions.name}), word_similarity(${item.displayName ?? ""}, ${transactions.merchantName})) DESC`, - sql`ABS(ABS(${transactions.amount}) - ${inboxAmount}) / GREATEST(1.0, ${inboxAmount})`, - sql`ABS(${transactions.date} - ${item.date}::date)`, - ) - .limit(Math.max(maxResults * 3, 30)); + const matches = await findTopMatches(db, { + teamId, + inboxId, + limit: maxResults, }); - const scoredResults = candidateTransactions - .map((transaction) => { - const nameScore = calculateNameScore( - item.displayName, - transaction.name, - transaction.merchantName, - ); - const amountScore = calculateAmountScore( - { - amount: item.amount, - currency: item.currency, - baseAmount: item.baseAmount, - baseCurrency: item.baseCurrency, - }, - { - amount: transaction.transactionAmount, - currency: transaction.transactionCurrency, - baseAmount: transaction.baseAmount, - baseCurrency: transaction.baseCurrency, - }, - ); - const currencyScore = calculateCurrencyScore( - item.currency || undefined, - transaction.transactionCurrency || undefined, - item.baseCurrency || undefined, - transaction.baseCurrency || undefined, - ); - const dateScore = calculateDateScore( - item.date!, - transaction.transactionDate, - ); - const isExactAmount = - item.amount !== null && - Math.abs( - Math.abs(item.amount || 0) - - Math.abs(transaction.transactionAmount || 0), - ) < 0.01; - const isSameCurrency = - item.currency === transaction.transactionCurrency; - const confidence = scoreMatch({ - nameScore, - amountScore, - dateScore, - currencyScore, - isSameCurrency, - isExactAmount, - }); - - const result = { - transaction_id: transaction.transactionId, - name: transaction.name, - transaction_amount: transaction.transactionAmount, - transaction_currency: transaction.transactionCurrency, - transaction_date: transaction.transactionDate, - name_score: Math.round(nameScore * 1000) / 1000, - amount_score: Math.round(amountScore * 1000) / 1000, - currency_score: Math.round(currencyScore * 1000) / 1000, - date_score: Math.round(dateScore * 1000) / 1000, - confidence_score: Math.round(confidence * 1000) / 1000, - is_already_matched: transaction.isAlreadyMatched, - matched_attachment_filename: - transaction.attachmentFilename ?? undefined, - }; - - return result; - }) - .filter((result) => result.confidence_score >= minConfidenceScore) - .sort((a, b) => { - if (a.confidence_score !== b.confidence_score) { - return b.confidence_score - a.confidence_score; - } - - if (a.is_already_matched !== b.is_already_matched) { - return a.is_already_matched ? 1 : -1; - } - - return 0; - }) - .slice(0, maxResults); - - return scoredResults; + return matches.map((m) => ({ + transaction_id: m.transactionId, + name: m.name, + transaction_amount: m.amount, + transaction_currency: m.currency, + transaction_date: m.date, + name_score: m.nameScore ?? 0, + amount_score: m.amountScore, + currency_score: m.currencyScore, + date_score: m.dateScore, + confidence_score: m.confidenceScore, + is_already_matched: m.isAlreadyMatched, + })); } catch { return []; }