From e8996e91b78949b2fae5404256c94bd343d6d44d Mon Sep 17 00:00:00 2001 From: Kiran K Date: Wed, 16 Jul 2025 20:30:31 +0530 Subject: [PATCH 01/17] Add metadata filtering to analytics & events wip --- apps/web/app/(ee)/api/events/route.ts | 1 + apps/web/lib/zod/schemas/analytics.ts | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/apps/web/app/(ee)/api/events/route.ts b/apps/web/app/(ee)/api/events/route.ts index efcb8bfa09..04b0b7487d 100644 --- a/apps/web/app/(ee)/api/events/route.ts +++ b/apps/web/app/(ee)/api/events/route.ts @@ -10,6 +10,7 @@ import { eventsQuerySchema } from "@/lib/zod/schemas/analytics"; import { Folder, Link } from "@dub/prisma/client"; import { NextResponse } from "next/server"; +// GET /api/events export const GET = withWorkspace( async ({ searchParams, workspace, session }) => { throwIfClicksUsageExceeded(workspace); diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index 2098ad20d7..430c079b3c 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -226,6 +226,13 @@ export const analyticsQuerySchema = z .describe( "Filter for root domains. If true, filter for domains only. If false, filter for links only. If undefined, return both.", ), + metadata: z + .record(z.string(), z.string()) + .optional() + .describe( + "Filter by event metadata key-value pairs. Format: key=value. Only available for lead and sale events.", + ) + .openapi({ example: { product: "premium" } }), }) .merge(utmTagsSchema); From fea11d2b45488f1b9ec826e3fc843e74f611b621 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Wed, 16 Jul 2025 22:03:57 +0530 Subject: [PATCH 02/17] Update analytics.ts --- apps/web/lib/zod/schemas/analytics.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index 430c079b3c..4ec793fb6a 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -287,6 +287,7 @@ export const analyticsFilterTB = z tenantId: true, folderId: true, sortBy: true, + metadata: true, }), ); From 2d5934a0e599eaa6798d69c61355e7e5f6ec2c51 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Thu, 17 Jul 2025 17:36:12 +0530 Subject: [PATCH 03/17] Add filter extraction and processing for event metadata in analytics --- apps/web/lib/analytics/get-events.ts | 41 ++++++++++++++++++++++++++ apps/web/lib/zod/schemas/analytics.ts | 20 +++++++++++-- packages/tinybird/pipes/v2_events.pipe | 18 +++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/apps/web/lib/analytics/get-events.ts b/apps/web/lib/analytics/get-events.ts index bca1c30aac..79803cfa49 100644 --- a/apps/web/lib/analytics/get-events.ts +++ b/apps/web/lib/analytics/get-events.ts @@ -24,6 +24,43 @@ import { import { EventsFilters } from "./types"; import { getStartEndDates } from "./utils/get-start-end-dates"; +interface Filter { + operand: string; + operator: "equals"; // can be expanded later + value: string; +} + +const extractFiltersFromQuery = (query: EventsFilters["query"]) => { + if (!query) { + return undefined; + } + + const result = z + .object({ + metadata: z.record(z.string(), z.string()), + }) + .safeParse(query); + + if (!result.success) { + console.log(`Ignoring the invalid query ${JSON.stringify(query)}`); + return undefined; + } + + const filters: Filter[] = Object.entries(result.data.metadata) + .slice(0, 1) + .map(([key, value]) => { + return { + operand: key, + operator: "equals", + value, + }; + }); + + console.log("filters", filters); + + return filters; +}; + // Fetch data for /api/events export const getEvents = async (params: EventsFilters) => { let { @@ -39,6 +76,7 @@ export const getEvents = async (params: EventsFilters) => { order, sortOrder, dataAvailableFrom, + query, } = params; const { startDate, endDate } = getStartEndDates({ @@ -78,6 +116,8 @@ export const getEvents = async (params: EventsFilters) => { }[eventType] ?? clickEventSchemaTBEndpoint, }); + const filters = extractFiltersFromQuery(query); + const response = await pipe({ ...params, eventType, @@ -89,6 +129,7 @@ export const getEvents = async (params: EventsFilters) => { offset: (params.page - 1) * params.limit, start: startDate.toISOString().replace("T", " ").replace("Z", ""), end: endDate.toISOString().replace("T", " ").replace("Z", ""), + filters: filters ? JSON.stringify(filters) : undefined, }); const [linksMap, customersMap] = await Promise.all([ diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index 4ec793fb6a..57bb1065be 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -226,13 +226,26 @@ export const analyticsQuerySchema = z .describe( "Filter for root domains. If true, filter for domains only. If false, filter for links only. If undefined, return both.", ), - metadata: z - .record(z.string(), z.string()) + query: z + .string() + .transform((v) => { + try { + return JSON.parse(v); + } catch (e) { + return undefined; + } + }) .optional() .describe( "Filter by event metadata key-value pairs. Format: key=value. Only available for lead and sale events.", ) - .openapi({ example: { product: "premium" } }), + .openapi({ + example: { + metadata: { + eventType: "project_transfer", + }, + }, + }), }) .merge(utmTagsSchema); @@ -299,6 +312,7 @@ export const eventsFilterTB = analyticsFilterTB limit: z.coerce.number().default(PAGINATION_LIMIT), order: z.enum(["asc", "desc"]).default("desc"), sortBy: z.enum(["timestamp"]).default("timestamp"), + filters: z.any() }), ); diff --git a/packages/tinybird/pipes/v2_events.pipe b/packages/tinybird/pipes/v2_events.pipe index 6f7eebf092..9d5ea0d03e 100644 --- a/packages/tinybird/pipes/v2_events.pipe +++ b/packages/tinybird/pipes/v2_events.pipe @@ -133,6 +133,15 @@ SQL > AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND url = {{ url }} {% end %} + + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('operator', '') == 'equals' %} + AND JSONExtractString(metadata, {{ item.get('operand', '') }}) = {{ item.get('value', '') }} + {% end %} + {% end %} + {% end %} + ORDER BY timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %} LIMIT {{ Int32(limit, 100) }} {% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %} @@ -174,6 +183,15 @@ SQL > {% if defined(utm_term) %} AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND url = {{ url }} {% end %} + + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('operator', '') == 'equals' %} + AND JSONExtractString(metadata, {{ item.get('operand', '') }}) = {{ item.get('value', '') }} + {% end %} + {% end %} + {% end %} + ORDER BY timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %} LIMIT {{ Int32(limit, 100) }} {% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %} From 9aefa0367cdd79d46d8588e74f6ae4778bd48d84 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Thu, 17 Jul 2025 17:41:24 +0530 Subject: [PATCH 04/17] Implement query filter parsing for analytics and events; refactor filter extraction logic --- apps/web/lib/analytics/get-analytics.ts | 5 +++ apps/web/lib/analytics/get-events.ts | 40 +------------------ .../analytics/utils/analytics-query-parser.ts | 39 ++++++++++++++++++ apps/web/lib/zod/schemas/analytics.ts | 3 +- 4 files changed, 47 insertions(+), 40 deletions(-) create mode 100644 apps/web/lib/analytics/utils/analytics-query-parser.ts diff --git a/apps/web/lib/analytics/get-analytics.ts b/apps/web/lib/analytics/get-analytics.ts index c3ca94f3d6..c24dc6a155 100644 --- a/apps/web/lib/analytics/get-analytics.ts +++ b/apps/web/lib/analytics/get-analytics.ts @@ -13,6 +13,7 @@ import { SINGULAR_ANALYTICS_ENDPOINTS, } from "./constants"; import { AnalyticsFilters } from "./types"; +import { parseFiltersFromQuery } from "./utils/analytics-query-parser"; import { getStartEndDates } from "./utils/get-start-end-dates"; // Fetch data for /api/analytics @@ -32,6 +33,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => { timezone = "UTC", isDeprecatedClicksEndpoint = false, dataAvailableFrom, + query, } = params; const tagIds = combineTagIds(params); @@ -104,6 +106,8 @@ export const getAnalytics = async (params: AnalyticsFilters) => { : analyticsResponse[groupBy], }); + const filters = parseFiltersFromQuery(query); + const response = await pipe({ ...params, ...(UTM_TAGS_PLURAL_LIST.includes(groupBy) @@ -119,6 +123,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => { timezone, country, region, + filters: filters ? JSON.stringify(filters) : undefined, }); if (groupBy === "count") { diff --git a/apps/web/lib/analytics/get-events.ts b/apps/web/lib/analytics/get-events.ts index 79803cfa49..a73d71ce8a 100644 --- a/apps/web/lib/analytics/get-events.ts +++ b/apps/web/lib/analytics/get-events.ts @@ -22,45 +22,9 @@ import { saleEventSchemaTBEndpoint, } from "../zod/schemas/sales"; import { EventsFilters } from "./types"; +import { parseFiltersFromQuery } from "./utils/analytics-query-parser"; import { getStartEndDates } from "./utils/get-start-end-dates"; -interface Filter { - operand: string; - operator: "equals"; // can be expanded later - value: string; -} - -const extractFiltersFromQuery = (query: EventsFilters["query"]) => { - if (!query) { - return undefined; - } - - const result = z - .object({ - metadata: z.record(z.string(), z.string()), - }) - .safeParse(query); - - if (!result.success) { - console.log(`Ignoring the invalid query ${JSON.stringify(query)}`); - return undefined; - } - - const filters: Filter[] = Object.entries(result.data.metadata) - .slice(0, 1) - .map(([key, value]) => { - return { - operand: key, - operator: "equals", - value, - }; - }); - - console.log("filters", filters); - - return filters; -}; - // Fetch data for /api/events export const getEvents = async (params: EventsFilters) => { let { @@ -116,7 +80,7 @@ export const getEvents = async (params: EventsFilters) => { }[eventType] ?? clickEventSchemaTBEndpoint, }); - const filters = extractFiltersFromQuery(query); + const filters = parseFiltersFromQuery(query); const response = await pipe({ ...params, diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/utils/analytics-query-parser.ts new file mode 100644 index 0000000000..969f388207 --- /dev/null +++ b/apps/web/lib/analytics/utils/analytics-query-parser.ts @@ -0,0 +1,39 @@ +import { z } from "zod"; +import { EventsFilters } from "../types"; + +interface Filter { + operand: string; + operator: "equals"; // can be expanded later + value: string; +} + +export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { + if (!query) { + return undefined; + } + + const result = z + .object({ + metadata: z.record(z.string(), z.string()), + }) + .safeParse(query); + + if (!result.success) { + console.log(`Ignoring the invalid query ${JSON.stringify(query)}`); + return undefined; + } + + const filters: Filter[] = Object.entries(result.data.metadata) + .slice(0, 1) + .map(([key, value]) => { + return { + operand: key, + operator: "equals", + value, + }; + }); + + console.log("filters", filters); + + return filters; +}; diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index 57bb1065be..225b29e26f 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -274,6 +274,7 @@ export const analyticsFilterTB = z .optional() .describe("The folder IDs to retrieve analytics for."), isMegaFolder: z.boolean().optional(), + filters: z.any(), }) .merge( analyticsQuerySchema.pick({ @@ -300,7 +301,6 @@ export const analyticsFilterTB = z tenantId: true, folderId: true, sortBy: true, - metadata: true, }), ); @@ -312,7 +312,6 @@ export const eventsFilterTB = analyticsFilterTB limit: z.coerce.number().default(PAGINATION_LIMIT), order: z.enum(["asc", "desc"]).default("desc"), sortBy: z.enum(["timestamp"]).default("timestamp"), - filters: z.any() }), ); From 5a367672eb64799ac776cab3324dc1e73913d560 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Thu, 17 Jul 2025 18:30:55 +0530 Subject: [PATCH 05/17] Update analytics-query-parser.ts --- apps/web/lib/analytics/utils/analytics-query-parser.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/utils/analytics-query-parser.ts index 969f388207..ed55b0f28d 100644 --- a/apps/web/lib/analytics/utils/analytics-query-parser.ts +++ b/apps/web/lib/analytics/utils/analytics-query-parser.ts @@ -7,16 +7,16 @@ interface Filter { value: string; } +const querySchema = z.object({ + metadata: z.record(z.string(), z.string()), +}); + export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { if (!query) { return undefined; } - const result = z - .object({ - metadata: z.record(z.string(), z.string()), - }) - .safeParse(query); + const result = querySchema.safeParse(query); if (!result.success) { console.log(`Ignoring the invalid query ${JSON.stringify(query)}`); From 127ca9c8399e2cd0ebe87e840552c180470a0d1f Mon Sep 17 00:00:00 2001 From: Kiran K Date: Thu, 17 Jul 2025 18:49:57 +0530 Subject: [PATCH 06/17] Update analytics.ts --- apps/web/lib/zod/schemas/analytics.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index 225b29e26f..23aec48dbb 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -237,9 +237,10 @@ export const analyticsQuerySchema = z }) .optional() .describe( - "Filter by event metadata key-value pairs. Format: key=value. Only available for lead and sale events.", + "Filter by event metadata key-value pairs as a JSON string. Only available for lead and sale events.", ) .openapi({ + type: "object", example: { metadata: { eventType: "project_transfer", @@ -274,7 +275,10 @@ export const analyticsFilterTB = z .optional() .describe("The folder IDs to retrieve analytics for."), isMegaFolder: z.boolean().optional(), - filters: z.any(), + filters: z + .string() + .optional() + .describe("The filters to apply to the analytics."), }) .merge( analyticsQuerySchema.pick({ From 8ce567b7fd5236dd578fec3d5f1d0387cd10e670 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Fri, 18 Jul 2025 11:19:43 +0530 Subject: [PATCH 07/17] support Stripe-like query filter --- .../analytics/utils/analytics-query-parser.ts | 107 +++++-- apps/web/lib/zod/schemas/analytics.ts | 16 +- .../tests/misc/analytics-query-parser.test.ts | 294 ++++++++++++++++++ 3 files changed, 382 insertions(+), 35 deletions(-) create mode 100644 apps/web/tests/misc/analytics-query-parser.test.ts diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/utils/analytics-query-parser.ts index ed55b0f28d..bc817bfff9 100644 --- a/apps/web/lib/analytics/utils/analytics-query-parser.ts +++ b/apps/web/lib/analytics/utils/analytics-query-parser.ts @@ -1,39 +1,104 @@ -import { z } from "zod"; import { EventsFilters } from "../types"; interface Filter { operand: string; - operator: "equals"; // can be expanded later + operator: "=" | ">" | "<" | ">=" | "<=" | "!="; value: string; } -const querySchema = z.object({ - metadata: z.record(z.string(), z.string()), -}); - +// Query parser that can parse the query string into a list of filters export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { if (!query) { return undefined; } - const result = querySchema.safeParse(query); + const filters: Filter[] = []; - if (!result.success) { - console.log(`Ignoring the invalid query ${JSON.stringify(query)}`); - return undefined; - } + // Split the query by logical operators (AND/OR) to handle multiple conditions + // For now, we'll focus on single conditions, but this structure allows for future expansion + const conditions = query.split(/\s+(?:AND|and|OR|or)\s+/); + + for (const condition of conditions) { + const trimmedCondition = condition.trim(); + + if (!trimmedCondition) { + continue; + } + + const filter = parseCondition(trimmedCondition); - const filters: Filter[] = Object.entries(result.data.metadata) - .slice(0, 1) - .map(([key, value]) => { - return { - operand: key, - operator: "equals", - value, - }; - }); + if (filter) { + filters.push(filter); + } + } console.log("filters", filters); - return filters; + return filters.length > 0 ? filters : undefined; }; + +// Parses a single condition in the format: field:value, field>value, or metadata['key']:value +function parseCondition(condition: string): Filter | null { + // Unified regex pattern that handles both regular fields and metadata fields + // This regex captures: + // 1. field - either a regular field name OR metadata with bracket notation + // 2. operator - :, >, <, >=, <=, != + // 3. value - the value after the operator (supports quoted and unquoted values) + const unifiedPattern = + /^([a-zA-Z_][a-zA-Z0-9_]*|metadata\['[^']*'\](?:\['[^']*'\])*)\s*([:><=!]+)\s*(.+)$/; + + const match = condition.match(unifiedPattern); + + if (!match) { + return null; + } + + // Extract the matched groups + const [, fieldOrMetadata, operator, value] = match; + + // Clean up the value by removing surrounding quotes if present + const cleanValue = value.trim().replace(/^['"`]|['"`]$/g, ""); + + // Determine the operand based on whether it's metadata or a regular field + let operand: string; + + if (fieldOrMetadata.startsWith("metadata")) { + const keyPath = fieldOrMetadata.replace(/^metadata/, ""); + + operand = keyPath + .replace(/^\['|'\]$/g, "") // Remove leading [' and trailing '] + .replace(/\['/g, ".") // Replace [' with . + .replace(/'\]/g, ""); // Remove trailing '] + } else { + operand = fieldOrMetadata; + } + + const filterOperator = mapOperator(operator); + + return { + operand, + operator: filterOperator, + value: cleanValue, + }; +} + +// Maps operator strings to our internal operator types +function mapOperator(operator: string): Filter["operator"] { + switch (operator) { + case ":": + return "="; + case ">": + return ">"; + case "<": + return "<"; + case ">=": + return ">="; + case "<=": + return "<="; + case "!=": + return "!="; + default: + // For unsupported operators, default to equals + return "="; + } +} diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index 23aec48dbb..e9cce39716 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -228,24 +228,12 @@ export const analyticsQuerySchema = z ), query: z .string() - .transform((v) => { - try { - return JSON.parse(v); - } catch (e) { - return undefined; - } - }) .optional() .describe( - "Filter by event metadata key-value pairs as a JSON string. Only available for lead and sale events.", + "Search the events by a custom metadata value. Only available for lead and sale events.", ) .openapi({ - type: "object", - example: { - metadata: { - eventType: "project_transfer", - }, - }, + example: "metadata['key']:'value'", }), }) .merge(utmTagsSchema); diff --git a/apps/web/tests/misc/analytics-query-parser.test.ts b/apps/web/tests/misc/analytics-query-parser.test.ts new file mode 100644 index 0000000000..f3eed2a523 --- /dev/null +++ b/apps/web/tests/misc/analytics-query-parser.test.ts @@ -0,0 +1,294 @@ +import { parseFiltersFromQuery } from "@/lib/analytics/utils/analytics-query-parser"; +import { describe, expect, it } from "vitest"; + +describe("Analytics Query Parser", () => { + describe("parseFiltersFromQuery", () => { + describe("basic fields", () => { + it("should parse simple field", () => { + const result = parseFiltersFromQuery("amount:100"); + expect(result).toEqual([ + { operand: "amount", operator: "=", value: "100" }, + ]); + }); + + it("should parse field with underscore", () => { + const result = parseFiltersFromQuery("user_id:123"); + expect(result).toEqual([ + { operand: "user_id", operator: "=", value: "123" }, + ]); + }); + + it("should parse field with numbers", () => { + const result = parseFiltersFromQuery("field123:value"); + expect(result).toEqual([ + { operand: "field123", operator: "=", value: "value" }, + ]); + }); + + it("should parse field with quoted value", () => { + const result = parseFiltersFromQuery("email:'john@example.com'"); + expect(result).toEqual([ + { operand: "email", operator: "=", value: "john@example.com" }, + ]); + }); + + it("should parse field with double quoted value", () => { + const result = parseFiltersFromQuery('status:"active"'); + expect(result).toEqual([ + { operand: "status", operator: "=", value: "active" }, + ]); + }); + + it("should parse field with backtick quoted value", () => { + const result = parseFiltersFromQuery("description:`quoted value`"); + expect(result).toEqual([ + { operand: "description", operator: "=", value: "quoted value" }, + ]); + }); + }); + + describe("nested properties", () => { + it("should parse simple nested property", () => { + const result = parseFiltersFromQuery("metadata['key']:value"); + expect(result).toEqual([ + { operand: "key", operator: "=", value: "value" }, + ]); + }); + + it.skip("should parse nested property with double quotes", () => { + const result = parseFiltersFromQuery('metadata["key"]:"quoted value"'); + expect(result).toEqual([ + { operand: "key", operator: "=", value: "quoted value" }, + ]); + }); + + it("should parse deeply nested property", () => { + const result = parseFiltersFromQuery( + "metadata['level1']['level2']['level3']:value", + ); + expect(result).toEqual([ + { operand: "level1.level2.level3", operator: "=", value: "value" }, + ]); + }); + + it("should parse nested property with complex path", () => { + const result = parseFiltersFromQuery( + "metadata['user']['preferences']['theme']:dark", + ); + expect(result).toEqual([ + { operand: "user.preferences.theme", operator: "=", value: "dark" }, + ]); + }); + }); + + describe("operators", () => { + it("should parse equals operator (:) for regular field", () => { + const result = parseFiltersFromQuery("amount:100"); + expect(result).toEqual([ + { operand: "amount", operator: "=", value: "100" }, + ]); + }); + + it("should parse equals operator (:) for nested property", () => { + const result = parseFiltersFromQuery("metadata['key']:value"); + expect(result).toEqual([ + { operand: "key", operator: "=", value: "value" }, + ]); + }); + + it("should parse greater than operator", () => { + const result = parseFiltersFromQuery("amount>50"); + expect(result).toEqual([ + { operand: "amount", operator: ">", value: "50" }, + ]); + }); + + it("should parse less than operator", () => { + const result = parseFiltersFromQuery("amount<100"); + expect(result).toEqual([ + { operand: "amount", operator: "<", value: "100" }, + ]); + }); + + it("should parse greater than or equal operator", () => { + const result = parseFiltersFromQuery("amount>=100"); + expect(result).toEqual([ + { operand: "amount", operator: ">=", value: "100" }, + ]); + }); + + it("should parse less than or equal operator", () => { + const result = parseFiltersFromQuery("amount<=50"); + expect(result).toEqual([ + { operand: "amount", operator: "<=", value: "50" }, + ]); + }); + + it("should parse not equals operator", () => { + const result = parseFiltersFromQuery("status!=completed"); + expect(result).toEqual([ + { operand: "status", operator: "!=", value: "completed" }, + ]); + }); + + it("should parse not equals operator for nested property", () => { + const result = parseFiltersFromQuery("metadata['status']!=completed"); + expect(result).toEqual([ + { operand: "status", operator: "!=", value: "completed" }, + ]); + }); + }); + + describe("multiple conditions", () => { + it("should parse multiple conditions with AND", () => { + const result = parseFiltersFromQuery("amount>=100 AND status:pending"); + expect(result).toEqual([ + { operand: "amount", operator: ">=", value: "100" }, + { operand: "status", operator: "=", value: "pending" }, + ]); + }); + + it("should parse multiple conditions with OR", () => { + const result = parseFiltersFromQuery("status:active OR status:pending"); + expect(result).toEqual([ + { operand: "status", operator: "=", value: "active" }, + { operand: "status", operator: "=", value: "pending" }, + ]); + }); + + it("should parse multiple conditions with mixed AND/OR", () => { + const result = parseFiltersFromQuery( + "amount>100 AND status:active OR email:test@example.com", + ); + expect(result).toEqual([ + { operand: "amount", operator: ">", value: "100" }, + { operand: "status", operator: "=", value: "active" }, + { operand: "email", operator: "=", value: "test@example.com" }, + ]); + }); + + it("should parse multiple nested property conditions", () => { + const result = parseFiltersFromQuery( + "metadata['product_id']:123 AND metadata['category']:electronics", + ); + expect(result).toEqual([ + { operand: "product_id", operator: "=", value: "123" }, + { operand: "category", operator: "=", value: "electronics" }, + ]); + }); + + it("should parse mixed nested and regular field conditions", () => { + const result = parseFiltersFromQuery( + "amount>100 AND metadata['user_type']:premium", + ); + expect(result).toEqual([ + { operand: "amount", operator: ">", value: "100" }, + { operand: "user_type", operator: "=", value: "premium" }, + ]); + }); + }); + + describe("edge cases", () => { + it("should handle empty query", () => { + const result = parseFiltersFromQuery(""); + expect(result).toBeUndefined(); + }); + + it("should handle null query", () => { + const result = parseFiltersFromQuery(null as any); + expect(result).toBeUndefined(); + }); + + it("should handle undefined query", () => { + const result = parseFiltersFromQuery(undefined as any); + expect(result).toBeUndefined(); + }); + + it("should handle whitespace-only query", () => { + const result = parseFiltersFromQuery(" "); + expect(result).toBeUndefined(); + }); + + it.skip("should handle invalid field names", () => { + const result = parseFiltersFromQuery("123field:value"); + expect(result).toBeNull(); + }); + + it("should handle invalid operators", () => { + const result = parseFiltersFromQuery("field==value"); + expect(result).toEqual([ + { operand: "field", operator: "=", value: "value" }, + ]); + }); + + it("should handle extra whitespace", () => { + const result = parseFiltersFromQuery(" amount : 100 "); + expect(result).toEqual([ + { operand: "amount", operator: "=", value: "100" }, + ]); + }); + + it("should handle numeric values", () => { + const result = parseFiltersFromQuery("amount:123.45"); + expect(result).toEqual([ + { operand: "amount", operator: "=", value: "123.45" }, + ]); + }); + + it("should handle boolean values", () => { + const result = parseFiltersFromQuery("active:true"); + expect(result).toEqual([ + { operand: "active", operator: "=", value: "true" }, + ]); + }); + + it("should handle special characters in values", () => { + const result = parseFiltersFromQuery( + "description:'Hello, World! @#$%'", + ); + expect(result).toEqual([ + { + operand: "description", + operator: "=", + value: "Hello, World! @#$%", + }, + ]); + }); + }); + + describe("real-world examples", () => { + it("should parse e-commerce filter", () => { + const result = parseFiltersFromQuery( + "amount>=100 AND metadata['category']:electronics AND status:completed", + ); + expect(result).toEqual([ + { operand: "amount", operator: ">=", value: "100" }, + { operand: "category", operator: "=", value: "electronics" }, + { operand: "status", operator: "=", value: "completed" }, + ]); + }); + + it("should parse user analytics filter", () => { + const result = parseFiltersFromQuery( + "metadata['user_type']:premium AND metadata['region']:us AND amount>50", + ); + expect(result).toEqual([ + { operand: "user_type", operator: "=", value: "premium" }, + { operand: "region", operator: "=", value: "us" }, + { operand: "amount", operator: ">", value: "50" }, + ]); + }); + + it("should parse lead generation filter", () => { + const result = parseFiltersFromQuery( + "metadata['source']:website AND metadata['campaign']:summer2024 AND status:qualified", + ); + expect(result).toEqual([ + { operand: "source", operator: "=", value: "website" }, + { operand: "campaign", operator: "=", value: "summer2024" }, + { operand: "status", operator: "=", value: "qualified" }, + ]); + }); + }); + }); +}); From 8c69b5775bad0916224d1e66895b3012b7e72a5b Mon Sep 17 00:00:00 2001 From: Kiran K Date: Fri, 18 Jul 2025 11:26:31 +0530 Subject: [PATCH 08/17] Update analytics-query-parser.ts --- .../analytics/utils/analytics-query-parser.ts | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/utils/analytics-query-parser.ts index bc817bfff9..0593c46bc9 100644 --- a/apps/web/lib/analytics/utils/analytics-query-parser.ts +++ b/apps/web/lib/analytics/utils/analytics-query-parser.ts @@ -1,6 +1,6 @@ import { EventsFilters } from "../types"; -interface Filter { +interface InternalFilter { operand: string; operator: "=" | ">" | "<" | ">=" | "<=" | "!="; value: string; @@ -12,7 +12,7 @@ export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { return undefined; } - const filters: Filter[] = []; + const filters: InternalFilter[] = []; // Split the query by logical operators (AND/OR) to handle multiple conditions // For now, we'll focus on single conditions, but this structure allows for future expansion @@ -32,20 +32,17 @@ export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { } } - console.log("filters", filters); - return filters.length > 0 ? filters : undefined; }; // Parses a single condition in the format: field:value, field>value, or metadata['key']:value -function parseCondition(condition: string): Filter | null { - // Unified regex pattern that handles both regular fields and metadata fields +function parseCondition(condition: string): InternalFilter | null { // This regex captures: - // 1. field - either a regular field name OR metadata with bracket notation + // 1. field - either a regular field name OR metadata with bracket notation (supports both single and double quotes) // 2. operator - :, >, <, >=, <=, != // 3. value - the value after the operator (supports quoted and unquoted values) const unifiedPattern = - /^([a-zA-Z_][a-zA-Z0-9_]*|metadata\['[^']*'\](?:\['[^']*'\])*)\s*([:><=!]+)\s*(.+)$/; + /^([a-zA-Z_][a-zA-Z0-9_]*|metadata\[['"][^'"]*['"]\](?:\[['"][^'"]*['"]\])*)\s*([:><=!]+)\s*(.+)$/; const match = condition.match(unifiedPattern); @@ -56,34 +53,29 @@ function parseCondition(condition: string): Filter | null { // Extract the matched groups const [, fieldOrMetadata, operator, value] = match; - // Clean up the value by removing surrounding quotes if present - const cleanValue = value.trim().replace(/^['"`]|['"`]$/g, ""); - - // Determine the operand based on whether it's metadata or a regular field let operand: string; + // Determine the operand based on whether it's metadata or a regular field if (fieldOrMetadata.startsWith("metadata")) { const keyPath = fieldOrMetadata.replace(/^metadata/, ""); operand = keyPath - .replace(/^\['|'\]$/g, "") // Remove leading [' and trailing '] - .replace(/\['/g, ".") // Replace [' with . - .replace(/'\]/g, ""); // Remove trailing '] + .replace(/^\[['"]|['"]\]$/g, "") // Remove leading [' or [" and trailing '] or "] + .replace(/\[['"]/g, ".") // Replace [' or [" with . + .replace(/['"]\]/g, ""); // Remove trailing '] or "] } else { operand = fieldOrMetadata; } - const filterOperator = mapOperator(operator); - return { operand, - operator: filterOperator, - value: cleanValue, + operator: mapOperator(operator), + value: value.trim().replace(/^['"`]|['"`]$/g, ""), }; } // Maps operator strings to our internal operator types -function mapOperator(operator: string): Filter["operator"] { +function mapOperator(operator: string): InternalFilter["operator"] { switch (operator) { case ":": return "="; From d6c229b239423263869c58c66bb533decc9a7455 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Fri, 18 Jul 2025 11:26:36 +0530 Subject: [PATCH 09/17] Update analytics-query-parser.test.ts --- apps/web/tests/misc/analytics-query-parser.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/web/tests/misc/analytics-query-parser.test.ts b/apps/web/tests/misc/analytics-query-parser.test.ts index f3eed2a523..3f6c43283c 100644 --- a/apps/web/tests/misc/analytics-query-parser.test.ts +++ b/apps/web/tests/misc/analytics-query-parser.test.ts @@ -55,7 +55,7 @@ describe("Analytics Query Parser", () => { ]); }); - it.skip("should parse nested property with double quotes", () => { + it("should parse nested property with double quotes", () => { const result = parseFiltersFromQuery('metadata["key"]:"quoted value"'); expect(result).toEqual([ { operand: "key", operator: "=", value: "quoted value" }, @@ -209,9 +209,9 @@ describe("Analytics Query Parser", () => { expect(result).toBeUndefined(); }); - it.skip("should handle invalid field names", () => { + it("should handle invalid field names", () => { const result = parseFiltersFromQuery("123field:value"); - expect(result).toBeNull(); + expect(result).toBeUndefined(); }); it("should handle invalid operators", () => { From 9e7bd009f1c8c8a5c95002a5225d8c00185089d7 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Fri, 18 Jul 2025 13:26:13 +0530 Subject: [PATCH 10/17] Refactor analytics query parser to use descriptive operator names and update tests accordingly --- .../analytics/utils/analytics-query-parser.ts | 26 +++++++----- .../tests/misc/analytics-query-parser.test.ts | 28 ++++++------- packages/tinybird/pipes/v2_events.pipe | 40 +++++++++++++++++-- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/utils/analytics-query-parser.ts index 0593c46bc9..a3109a5412 100644 --- a/apps/web/lib/analytics/utils/analytics-query-parser.ts +++ b/apps/web/lib/analytics/utils/analytics-query-parser.ts @@ -2,7 +2,13 @@ import { EventsFilters } from "../types"; interface InternalFilter { operand: string; - operator: "=" | ">" | "<" | ">=" | "<=" | "!="; + operator: + | "equals" + | "notEquals" + | "greaterThan" + | "lessThan" + | "greaterThanOrEqual" + | "lessThanOrEqual"; value: string; } @@ -59,10 +65,12 @@ function parseCondition(condition: string): InternalFilter | null { if (fieldOrMetadata.startsWith("metadata")) { const keyPath = fieldOrMetadata.replace(/^metadata/, ""); - operand = keyPath + const extractedKey = keyPath .replace(/^\[['"]|['"]\]$/g, "") // Remove leading [' or [" and trailing '] or "] .replace(/\[['"]/g, ".") // Replace [' or [" with . .replace(/['"]\]/g, ""); // Remove trailing '] or "] + + operand = `metadata.${extractedKey}`; } else { operand = fieldOrMetadata; } @@ -78,19 +86,19 @@ function parseCondition(condition: string): InternalFilter | null { function mapOperator(operator: string): InternalFilter["operator"] { switch (operator) { case ":": - return "="; + return "equals"; case ">": - return ">"; + return "greaterThan"; case "<": - return "<"; + return "lessThan"; case ">=": - return ">="; + return "greaterThanOrEqual"; case "<=": - return "<="; + return "lessThanOrEqual"; case "!=": - return "!="; + return "notEquals"; default: // For unsupported operators, default to equals - return "="; + return "equals"; } } diff --git a/apps/web/tests/misc/analytics-query-parser.test.ts b/apps/web/tests/misc/analytics-query-parser.test.ts index 3f6c43283c..58aa830f65 100644 --- a/apps/web/tests/misc/analytics-query-parser.test.ts +++ b/apps/web/tests/misc/analytics-query-parser.test.ts @@ -51,14 +51,14 @@ describe("Analytics Query Parser", () => { it("should parse simple nested property", () => { const result = parseFiltersFromQuery("metadata['key']:value"); expect(result).toEqual([ - { operand: "key", operator: "=", value: "value" }, + { operand: "metadata.key", operator: "=", value: "value" }, ]); }); it("should parse nested property with double quotes", () => { const result = parseFiltersFromQuery('metadata["key"]:"quoted value"'); expect(result).toEqual([ - { operand: "key", operator: "=", value: "quoted value" }, + { operand: "metadata.key", operator: "=", value: "quoted value" }, ]); }); @@ -67,7 +67,7 @@ describe("Analytics Query Parser", () => { "metadata['level1']['level2']['level3']:value", ); expect(result).toEqual([ - { operand: "level1.level2.level3", operator: "=", value: "value" }, + { operand: "metadata.level1.level2.level3", operator: "=", value: "value" }, ]); }); @@ -76,7 +76,7 @@ describe("Analytics Query Parser", () => { "metadata['user']['preferences']['theme']:dark", ); expect(result).toEqual([ - { operand: "user.preferences.theme", operator: "=", value: "dark" }, + { operand: "metadata.user.preferences.theme", operator: "=", value: "dark" }, ]); }); }); @@ -92,7 +92,7 @@ describe("Analytics Query Parser", () => { it("should parse equals operator (:) for nested property", () => { const result = parseFiltersFromQuery("metadata['key']:value"); expect(result).toEqual([ - { operand: "key", operator: "=", value: "value" }, + { operand: "metadata.key", operator: "=", value: "value" }, ]); }); @@ -134,7 +134,7 @@ describe("Analytics Query Parser", () => { it("should parse not equals operator for nested property", () => { const result = parseFiltersFromQuery("metadata['status']!=completed"); expect(result).toEqual([ - { operand: "status", operator: "!=", value: "completed" }, + { operand: "metadata.status", operator: "!=", value: "completed" }, ]); }); }); @@ -172,8 +172,8 @@ describe("Analytics Query Parser", () => { "metadata['product_id']:123 AND metadata['category']:electronics", ); expect(result).toEqual([ - { operand: "product_id", operator: "=", value: "123" }, - { operand: "category", operator: "=", value: "electronics" }, + { operand: "metadata.product_id", operator: "=", value: "123" }, + { operand: "metadata.category", operator: "=", value: "electronics" }, ]); }); @@ -183,7 +183,7 @@ describe("Analytics Query Parser", () => { ); expect(result).toEqual([ { operand: "amount", operator: ">", value: "100" }, - { operand: "user_type", operator: "=", value: "premium" }, + { operand: "metadata.user_type", operator: "=", value: "premium" }, ]); }); }); @@ -263,7 +263,7 @@ describe("Analytics Query Parser", () => { ); expect(result).toEqual([ { operand: "amount", operator: ">=", value: "100" }, - { operand: "category", operator: "=", value: "electronics" }, + { operand: "metadata.category", operator: "=", value: "electronics" }, { operand: "status", operator: "=", value: "completed" }, ]); }); @@ -273,8 +273,8 @@ describe("Analytics Query Parser", () => { "metadata['user_type']:premium AND metadata['region']:us AND amount>50", ); expect(result).toEqual([ - { operand: "user_type", operator: "=", value: "premium" }, - { operand: "region", operator: "=", value: "us" }, + { operand: "metadata.user_type", operator: "=", value: "premium" }, + { operand: "metadata.region", operator: "=", value: "us" }, { operand: "amount", operator: ">", value: "50" }, ]); }); @@ -284,8 +284,8 @@ describe("Analytics Query Parser", () => { "metadata['source']:website AND metadata['campaign']:summer2024 AND status:qualified", ); expect(result).toEqual([ - { operand: "source", operator: "=", value: "website" }, - { operand: "campaign", operator: "=", value: "summer2024" }, + { operand: "metadata.source", operator: "=", value: "website" }, + { operand: "metadata.campaign", operator: "=", value: "summer2024" }, { operand: "status", operator: "=", value: "qualified" }, ]); }); diff --git a/packages/tinybird/pipes/v2_events.pipe b/packages/tinybird/pipes/v2_events.pipe index 9d5ea0d03e..b7027b7942 100644 --- a/packages/tinybird/pipes/v2_events.pipe +++ b/packages/tinybird/pipes/v2_events.pipe @@ -136,8 +136,24 @@ SQL > {% if defined(filters) %} {% for item in JSON(filters, '[]') %} - {% if item.get('operator', '') == 'equals' %} - AND JSONExtractString(metadata, {{ item.get('operand', '') }}) = {{ item.get('value', '') }} + {% if item.get('operand', '').startswith('metadata.') %} + {% set metadataKey = item.get('operand', '').split('.')[1] %} + {% set operator = item.get('operator', 'equals') %} + {% set value = item.get('value', '') %} + + {% if operator == 'equals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }} + {% elif operator == 'notEquals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} + {% elif operator == 'greaterThan' %} + AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }} + {% elif operator == 'lessThan' %} + AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }} + {% elif operator == 'greaterThanOrEqual' %} + AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }} + {% elif operator == 'lessThanOrEqual' %} + AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }} + {% end %} {% end %} {% end %} {% end %} @@ -186,8 +202,24 @@ SQL > {% if defined(filters) %} {% for item in JSON(filters, '[]') %} - {% if item.get('operator', '') == 'equals' %} - AND JSONExtractString(metadata, {{ item.get('operand', '') }}) = {{ item.get('value', '') }} + {% if item.get('operand', '').startswith('metadata.') %} + {% set metadataKey = item.get('operand', '').split('.')[1] %} + {% set operator = item.get('operator', 'equals') %} + {% set value = item.get('value', '') %} + + {% if operator == 'equals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }} + {% elif operator == 'notEquals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} + {% elif operator == 'greaterThan' %} + AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }} + {% elif operator == 'lessThan' %} + AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }} + {% elif operator == 'greaterThanOrEqual' %} + AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }} + {% elif operator == 'lessThanOrEqual' %} + AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }} + {% end %} {% end %} {% end %} {% end %} From 76d567036725439a66fe8ae515d8296459f6384d Mon Sep 17 00:00:00 2001 From: Kiran K Date: Fri, 18 Jul 2025 13:51:24 +0530 Subject: [PATCH 11/17] Update analytics-query-parser.ts --- .../web/lib/analytics/utils/analytics-query-parser.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/utils/analytics-query-parser.ts index a3109a5412..759862c9c9 100644 --- a/apps/web/lib/analytics/utils/analytics-query-parser.ts +++ b/apps/web/lib/analytics/utils/analytics-query-parser.ts @@ -33,9 +33,16 @@ export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { const filter = parseCondition(trimmedCondition); - if (filter) { - filters.push(filter); + if (!filter) { + continue; } + + // We only allow metadata filters for now + if (!filter.operand.startsWith("metadata.")) { + continue; + } + + filters.push(filter); } return filters.length > 0 ? filters : undefined; From 01dab207794cfe295a766b078b1cbffb2200ac46 Mon Sep 17 00:00:00 2001 From: Kiran K Date: Fri, 18 Jul 2025 14:07:13 +0530 Subject: [PATCH 12/17] fix tests --- .../analytics/utils/analytics-query-parser.ts | 21 +- .../tests/misc/analytics-query-parser.test.ts | 342 ++++-------------- 2 files changed, 81 insertions(+), 282 deletions(-) diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/utils/analytics-query-parser.ts index 759862c9c9..59153398f5 100644 --- a/apps/web/lib/analytics/utils/analytics-query-parser.ts +++ b/apps/web/lib/analytics/utils/analytics-query-parser.ts @@ -13,7 +13,10 @@ interface InternalFilter { } // Query parser that can parse the query string into a list of filters -export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { +export const parseFiltersFromQuery = ( + query: EventsFilters["query"], + allowedOperands = ["metadata"], +) => { if (!query) { return undefined; } @@ -37,8 +40,19 @@ export const parseFiltersFromQuery = (query: EventsFilters["query"]) => { continue; } - // We only allow metadata filters for now - if (!filter.operand.startsWith("metadata.")) { + const isAllowed = allowedOperands.some((allowed) => { + if (filter.operand === allowed) { + return true; + } + + if (filter.operand.startsWith(`${allowed}.`)) { + return true; + } + + return false; + }); + + if (!isAllowed) { continue; } @@ -93,6 +107,7 @@ function parseCondition(condition: string): InternalFilter | null { function mapOperator(operator: string): InternalFilter["operator"] { switch (operator) { case ":": + case "=": return "equals"; case ">": return "greaterThan"; diff --git a/apps/web/tests/misc/analytics-query-parser.test.ts b/apps/web/tests/misc/analytics-query-parser.test.ts index 58aa830f65..d409450c59 100644 --- a/apps/web/tests/misc/analytics-query-parser.test.ts +++ b/apps/web/tests/misc/analytics-query-parser.test.ts @@ -2,293 +2,77 @@ import { parseFiltersFromQuery } from "@/lib/analytics/utils/analytics-query-par import { describe, expect, it } from "vitest"; describe("Analytics Query Parser", () => { - describe("parseFiltersFromQuery", () => { - describe("basic fields", () => { - it("should parse simple field", () => { - const result = parseFiltersFromQuery("amount:100"); - expect(result).toEqual([ - { operand: "amount", operator: "=", value: "100" }, - ]); - }); - - it("should parse field with underscore", () => { - const result = parseFiltersFromQuery("user_id:123"); - expect(result).toEqual([ - { operand: "user_id", operator: "=", value: "123" }, - ]); - }); - - it("should parse field with numbers", () => { - const result = parseFiltersFromQuery("field123:value"); - expect(result).toEqual([ - { operand: "field123", operator: "=", value: "value" }, - ]); - }); - - it("should parse field with quoted value", () => { - const result = parseFiltersFromQuery("email:'john@example.com'"); - expect(result).toEqual([ - { operand: "email", operator: "=", value: "john@example.com" }, - ]); - }); - - it("should parse field with double quoted value", () => { - const result = parseFiltersFromQuery('status:"active"'); - expect(result).toEqual([ - { operand: "status", operator: "=", value: "active" }, - ]); - }); - - it("should parse field with backtick quoted value", () => { - const result = parseFiltersFromQuery("description:`quoted value`"); - expect(result).toEqual([ - { operand: "description", operator: "=", value: "quoted value" }, - ]); - }); - }); - - describe("nested properties", () => { - it("should parse simple nested property", () => { - const result = parseFiltersFromQuery("metadata['key']:value"); - expect(result).toEqual([ - { operand: "metadata.key", operator: "=", value: "value" }, - ]); - }); - - it("should parse nested property with double quotes", () => { - const result = parseFiltersFromQuery('metadata["key"]:"quoted value"'); - expect(result).toEqual([ - { operand: "metadata.key", operator: "=", value: "quoted value" }, - ]); - }); - - it("should parse deeply nested property", () => { - const result = parseFiltersFromQuery( - "metadata['level1']['level2']['level3']:value", - ); - expect(result).toEqual([ - { operand: "metadata.level1.level2.level3", operator: "=", value: "value" }, - ]); - }); - - it("should parse nested property with complex path", () => { - const result = parseFiltersFromQuery( - "metadata['user']['preferences']['theme']:dark", - ); - expect(result).toEqual([ - { operand: "metadata.user.preferences.theme", operator: "=", value: "dark" }, - ]); - }); - }); - - describe("operators", () => { - it("should parse equals operator (:) for regular field", () => { - const result = parseFiltersFromQuery("amount:100"); - expect(result).toEqual([ - { operand: "amount", operator: "=", value: "100" }, - ]); - }); - - it("should parse equals operator (:) for nested property", () => { - const result = parseFiltersFromQuery("metadata['key']:value"); - expect(result).toEqual([ - { operand: "metadata.key", operator: "=", value: "value" }, - ]); - }); - - it("should parse greater than operator", () => { - const result = parseFiltersFromQuery("amount>50"); - expect(result).toEqual([ - { operand: "amount", operator: ">", value: "50" }, - ]); - }); - - it("should parse less than operator", () => { - const result = parseFiltersFromQuery("amount<100"); - expect(result).toEqual([ - { operand: "amount", operator: "<", value: "100" }, - ]); - }); - - it("should parse greater than or equal operator", () => { - const result = parseFiltersFromQuery("amount>=100"); - expect(result).toEqual([ - { operand: "amount", operator: ">=", value: "100" }, - ]); - }); - - it("should parse less than or equal operator", () => { - const result = parseFiltersFromQuery("amount<=50"); - expect(result).toEqual([ - { operand: "amount", operator: "<=", value: "50" }, - ]); - }); - - it("should parse not equals operator", () => { - const result = parseFiltersFromQuery("status!=completed"); - expect(result).toEqual([ - { operand: "status", operator: "!=", value: "completed" }, - ]); - }); - - it("should parse not equals operator for nested property", () => { - const result = parseFiltersFromQuery("metadata['status']!=completed"); - expect(result).toEqual([ - { operand: "metadata.status", operator: "!=", value: "completed" }, - ]); - }); - }); - - describe("multiple conditions", () => { - it("should parse multiple conditions with AND", () => { - const result = parseFiltersFromQuery("amount>=100 AND status:pending"); - expect(result).toEqual([ - { operand: "amount", operator: ">=", value: "100" }, - { operand: "status", operator: "=", value: "pending" }, - ]); - }); - - it("should parse multiple conditions with OR", () => { - const result = parseFiltersFromQuery("status:active OR status:pending"); - expect(result).toEqual([ - { operand: "status", operator: "=", value: "active" }, - { operand: "status", operator: "=", value: "pending" }, - ]); - }); - - it("should parse multiple conditions with mixed AND/OR", () => { - const result = parseFiltersFromQuery( - "amount>100 AND status:active OR email:test@example.com", - ); - expect(result).toEqual([ - { operand: "amount", operator: ">", value: "100" }, - { operand: "status", operator: "=", value: "active" }, - { operand: "email", operator: "=", value: "test@example.com" }, - ]); - }); - - it("should parse multiple nested property conditions", () => { - const result = parseFiltersFromQuery( - "metadata['product_id']:123 AND metadata['category']:electronics", - ); - expect(result).toEqual([ - { operand: "metadata.product_id", operator: "=", value: "123" }, - { operand: "metadata.category", operator: "=", value: "electronics" }, - ]); - }); - - it("should parse mixed nested and regular field conditions", () => { - const result = parseFiltersFromQuery( - "amount>100 AND metadata['user_type']:premium", - ); - expect(result).toEqual([ - { operand: "amount", operator: ">", value: "100" }, - { operand: "metadata.user_type", operator: "=", value: "premium" }, - ]); - }); - }); - - describe("edge cases", () => { - it("should handle empty query", () => { - const result = parseFiltersFromQuery(""); - expect(result).toBeUndefined(); - }); - - it("should handle null query", () => { - const result = parseFiltersFromQuery(null as any); - expect(result).toBeUndefined(); - }); - - it("should handle undefined query", () => { - const result = parseFiltersFromQuery(undefined as any); - expect(result).toBeUndefined(); - }); - - it("should handle whitespace-only query", () => { - const result = parseFiltersFromQuery(" "); - expect(result).toBeUndefined(); - }); + it("should parse simple nested property", () => { + const result = parseFiltersFromQuery("metadata['key']:value"); + expect(result).toEqual([ + { operand: "metadata.key", operator: "equals", value: "value" }, + ]); + }); - it("should handle invalid field names", () => { - const result = parseFiltersFromQuery("123field:value"); - expect(result).toBeUndefined(); - }); + it("should parse nested property with double quotes", () => { + const result = parseFiltersFromQuery('metadata["key"]:"quoted value"'); + expect(result).toEqual([ + { operand: "metadata.key", operator: "equals", value: "quoted value" }, + ]); + }); - it("should handle invalid operators", () => { - const result = parseFiltersFromQuery("field==value"); - expect(result).toEqual([ - { operand: "field", operator: "=", value: "value" }, - ]); - }); + it("should parse deeply nested property", () => { + const result = parseFiltersFromQuery( + "metadata['level1']['level2']['level3']:value", + ); + expect(result).toEqual([ + { + operand: "metadata.level1.level2.level3", + operator: "equals", + value: "value", + }, + ]); + }); - it("should handle extra whitespace", () => { - const result = parseFiltersFromQuery(" amount : 100 "); - expect(result).toEqual([ - { operand: "amount", operator: "=", value: "100" }, - ]); - }); + it("should parse nested property with complex path", () => { + const result = parseFiltersFromQuery( + "metadata['user']['preferences']['theme']:dark", + ); + expect(result).toEqual([ + { + operand: "metadata.user.preferences.theme", + operator: "equals", + value: "dark", + }, + ]); + }); - it("should handle numeric values", () => { - const result = parseFiltersFromQuery("amount:123.45"); - expect(result).toEqual([ - { operand: "amount", operator: "=", value: "123.45" }, - ]); - }); + it("should parse equals operator (:) for nested property", () => { + const result = parseFiltersFromQuery("metadata['key']:value"); + expect(result).toEqual([ + { operand: "metadata.key", operator: "equals", value: "value" }, + ]); + }); - it("should handle boolean values", () => { - const result = parseFiltersFromQuery("active:true"); - expect(result).toEqual([ - { operand: "active", operator: "=", value: "true" }, - ]); - }); + it("should parse not equals operator for nested property", () => { + const result = parseFiltersFromQuery("metadata['status']!=completed"); + expect(result).toEqual([ + { operand: "metadata.status", operator: "notEquals", value: "completed" }, + ]); + }); - it("should handle special characters in values", () => { - const result = parseFiltersFromQuery( - "description:'Hello, World! @#$%'", - ); - expect(result).toEqual([ - { - operand: "description", - operator: "=", - value: "Hello, World! @#$%", - }, - ]); - }); - }); + it("should handle empty query", () => { + const result = parseFiltersFromQuery(""); + expect(result).toBeUndefined(); + }); - describe("real-world examples", () => { - it("should parse e-commerce filter", () => { - const result = parseFiltersFromQuery( - "amount>=100 AND metadata['category']:electronics AND status:completed", - ); - expect(result).toEqual([ - { operand: "amount", operator: ">=", value: "100" }, - { operand: "metadata.category", operator: "=", value: "electronics" }, - { operand: "status", operator: "=", value: "completed" }, - ]); - }); + it("should handle null query", () => { + const result = parseFiltersFromQuery(null as any); + expect(result).toBeUndefined(); + }); - it("should parse user analytics filter", () => { - const result = parseFiltersFromQuery( - "metadata['user_type']:premium AND metadata['region']:us AND amount>50", - ); - expect(result).toEqual([ - { operand: "metadata.user_type", operator: "=", value: "premium" }, - { operand: "metadata.region", operator: "=", value: "us" }, - { operand: "amount", operator: ">", value: "50" }, - ]); - }); + it("should handle undefined query", () => { + const result = parseFiltersFromQuery(undefined as any); + expect(result).toBeUndefined(); + }); - it("should parse lead generation filter", () => { - const result = parseFiltersFromQuery( - "metadata['source']:website AND metadata['campaign']:summer2024 AND status:qualified", - ); - expect(result).toEqual([ - { operand: "metadata.source", operator: "=", value: "website" }, - { operand: "metadata.campaign", operator: "=", value: "summer2024" }, - { operand: "status", operator: "=", value: "qualified" }, - ]); - }); - }); + it("should handle whitespace-only query", () => { + const result = parseFiltersFromQuery(" "); + expect(result).toBeUndefined(); }); }); From 4dd148f447cb8924ad947b66480591f67138b88b Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Mon, 4 Aug 2025 13:06:43 -0700 Subject: [PATCH 13/17] rearrange parseFiltersFromQuery --- apps/web/lib/analytics/get-analytics.ts | 2 +- apps/web/lib/analytics/get-events.ts | 2 +- ...lytics-query-parser.ts => query-parser.ts} | 2 +- .../tests/misc/analytics-query-parser.test.ts | 2 +- packages/tinybird/pipes/v2_events.pipe | 29 +++++++++++++------ 5 files changed, 24 insertions(+), 13 deletions(-) rename apps/web/lib/analytics/{utils/analytics-query-parser.ts => query-parser.ts} (98%) diff --git a/apps/web/lib/analytics/get-analytics.ts b/apps/web/lib/analytics/get-analytics.ts index 6b67511b38..1d34e46cf1 100644 --- a/apps/web/lib/analytics/get-analytics.ts +++ b/apps/web/lib/analytics/get-analytics.ts @@ -12,8 +12,8 @@ import { DIMENSIONAL_ANALYTICS_FILTERS, SINGULAR_ANALYTICS_ENDPOINTS, } from "./constants"; +import { parseFiltersFromQuery } from "./query-parser"; import { AnalyticsFilters } from "./types"; -import { parseFiltersFromQuery } from "./utils/analytics-query-parser"; import { getStartEndDates } from "./utils/get-start-end-dates"; // Fetch data for /api/analytics diff --git a/apps/web/lib/analytics/get-events.ts b/apps/web/lib/analytics/get-events.ts index 1fb8628a5c..975ecd44a0 100644 --- a/apps/web/lib/analytics/get-events.ts +++ b/apps/web/lib/analytics/get-events.ts @@ -21,8 +21,8 @@ import { saleEventResponseSchema, saleEventSchemaTBEndpoint, } from "../zod/schemas/sales"; +import { parseFiltersFromQuery } from "./query-parser"; import { EventsFilters } from "./types"; -import { parseFiltersFromQuery } from "./utils/analytics-query-parser"; import { getStartEndDates } from "./utils/get-start-end-dates"; // Fetch data for /api/events diff --git a/apps/web/lib/analytics/utils/analytics-query-parser.ts b/apps/web/lib/analytics/query-parser.ts similarity index 98% rename from apps/web/lib/analytics/utils/analytics-query-parser.ts rename to apps/web/lib/analytics/query-parser.ts index 59153398f5..4a15728526 100644 --- a/apps/web/lib/analytics/utils/analytics-query-parser.ts +++ b/apps/web/lib/analytics/query-parser.ts @@ -1,4 +1,4 @@ -import { EventsFilters } from "../types"; +import { EventsFilters } from "./types"; interface InternalFilter { operand: string; diff --git a/apps/web/tests/misc/analytics-query-parser.test.ts b/apps/web/tests/misc/analytics-query-parser.test.ts index d409450c59..7ef0669041 100644 --- a/apps/web/tests/misc/analytics-query-parser.test.ts +++ b/apps/web/tests/misc/analytics-query-parser.test.ts @@ -1,4 +1,4 @@ -import { parseFiltersFromQuery } from "@/lib/analytics/utils/analytics-query-parser"; +import { parseFiltersFromQuery } from "@/lib/analytics/query-parser"; import { describe, expect, it } from "vitest"; describe("Analytics Query Parser", () => { diff --git a/packages/tinybird/pipes/v2_events.pipe b/packages/tinybird/pipes/v2_events.pipe index 31a78ff10e..0ff64bcb57 100644 --- a/packages/tinybird/pipes/v2_events.pipe +++ b/packages/tinybird/pipes/v2_events.pipe @@ -134,14 +134,14 @@ SQL > AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND url = {{ url }} {% end %} - + {% if defined(filters) %} {% for item in JSON(filters, '[]') %} {% if item.get('operand', '').startswith('metadata.') %} {% set metadataKey = item.get('operand', '').split('.')[1] %} {% set operator = item.get('operator', 'equals') %} {% set value = item.get('value', '') %} - + {% if operator == 'equals' %} AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }} {% elif operator == 'notEquals' %} @@ -158,7 +158,7 @@ SQL > {% end %} {% end %} {% end %} - + ORDER BY timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %} LIMIT {{ Int32(limit, 100) }} {% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %} @@ -194,11 +194,22 @@ SQL > {% if defined(os) %} AND os = {{ os }} {% end %} {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} - {% if defined(utm_source) %} AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} - {% if defined(utm_medium) %} AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} - {% if defined(utm_campaign) %} AND url LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} - {% if defined(utm_term) %} AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} - {% if defined(utm_content) %} AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} + {% if defined(utm_source) %} + AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + {% end %} + {% if defined(utm_medium) %} + AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + {% end %} + {% if defined(utm_campaign) %} + AND url + LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + {% end %} + {% if defined(utm_term) %} + AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + {% end %} + {% if defined(utm_content) %} + AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + {% end %} {% if defined(url) %} AND url = {{ url }} {% end %} {% if defined(filters) %} @@ -207,7 +218,7 @@ SQL > {% set metadataKey = item.get('operand', '').split('.')[1] %} {% set operator = item.get('operator', 'equals') %} {% set value = item.get('value', '') %} - + {% if operator == 'equals' %} AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }} {% elif operator == 'notEquals' %} From cfdc5cf713c3c28fc334cf43be651b8ea078a1df Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Mon, 4 Aug 2025 14:09:07 -0700 Subject: [PATCH 14/17] finalize schemas --- .../(ee)/api/customers/[id]/activity/route.ts | 1 - apps/web/lib/analytics/get-customer-events.ts | 23 ++++--------------- apps/web/lib/analytics/get-events.ts | 1 + apps/web/lib/zod/schemas/leads.ts | 14 +++-------- apps/web/lib/zod/schemas/sales.ts | 23 +++++++------------ .../ui/customers/customer-activity-list.tsx | 20 ++-------------- 6 files changed, 19 insertions(+), 63 deletions(-) diff --git a/apps/web/app/(ee)/api/customers/[id]/activity/route.ts b/apps/web/app/(ee)/api/customers/[id]/activity/route.ts index 576c819be4..3fc3b2c422 100644 --- a/apps/web/app/(ee)/api/customers/[id]/activity/route.ts +++ b/apps/web/app/(ee)/api/customers/[id]/activity/route.ts @@ -17,7 +17,6 @@ export const GET = withWorkspace(async ({ workspace, params }) => { const events = await getCustomerEvents({ customerId: customer.id, - includeMetadata: true, }); // get the first partner link that this customer interacted with diff --git a/apps/web/lib/analytics/get-customer-events.ts b/apps/web/lib/analytics/get-customer-events.ts index 80b942a261..8f1ad591a0 100644 --- a/apps/web/lib/analytics/get-customer-events.ts +++ b/apps/web/lib/analytics/get-customer-events.ts @@ -8,23 +8,15 @@ import { clickEventResponseSchema, clickEventSchema, } from "../zod/schemas/clicks"; -import { - leadEventResponseSchema, - leadEventResponseSchemaExtended, -} from "../zod/schemas/leads"; -import { - saleEventResponseSchema, - saleEventResponseSchemaExtended, -} from "../zod/schemas/sales"; +import { leadEventResponseSchema } from "../zod/schemas/leads"; +import { saleEventResponseSchema } from "../zod/schemas/sales"; export const getCustomerEvents = async ({ customerId, linkIds, - includeMetadata, }: { customerId: string; linkIds?: string[]; - includeMetadata?: boolean; }) => { const pipe = tb.buildPipe({ pipe: "v2_customer_events", @@ -68,6 +60,7 @@ export const getCustomerEvents = async ({ ? { eventId: evt.event_id, eventName: evt.event_name, + metadata: evt.metadata ? JSON.parse(evt.metadata) : undefined, ...(evt.event === "sale" ? { sale: { @@ -83,14 +76,8 @@ export const getCustomerEvents = async ({ return { click: clickEventResponseSchema, - lead: (includeMetadata - ? leadEventResponseSchemaExtended - : leadEventResponseSchema - ).omit({ customer: true }), - sale: (includeMetadata - ? saleEventResponseSchemaExtended - : saleEventResponseSchema - ).omit({ customer: true }), + lead: leadEventResponseSchema.omit({ customer: true }), + sale: saleEventResponseSchema.omit({ customer: true }), }[evt.event].parse(eventData); }) .filter((d) => d !== null); diff --git a/apps/web/lib/analytics/get-events.ts b/apps/web/lib/analytics/get-events.ts index 975ecd44a0..ddf7756d7c 100644 --- a/apps/web/lib/analytics/get-events.ts +++ b/apps/web/lib/analytics/get-events.ts @@ -135,6 +135,7 @@ export const getEvents = async (params: EventsFilters) => { ? { eventId: evt.event_id, eventName: evt.event_name, + metadata: evt.metadata ? JSON.parse(evt.metadata) : undefined, customer: customersMap[evt.customer_id] ?? { id: evt.customer_id, name: "Deleted Customer", diff --git a/apps/web/lib/zod/schemas/leads.ts b/apps/web/lib/zod/schemas/leads.ts index b206d13b09..ee89bc0e55 100644 --- a/apps/web/lib/zod/schemas/leads.ts +++ b/apps/web/lib/zod/schemas/leads.ts @@ -119,6 +119,7 @@ export const leadEventSchemaTBEndpoint = z.object({ referer_url_processed: z.string().nullable(), qr: z.number().nullable(), ip: z.string().nullable(), + metadata: z.string().nullish(), }); // response from dub api @@ -126,8 +127,10 @@ export const leadEventResponseSchema = z .object({ event: z.literal("lead"), timestamp: z.coerce.string(), + // core event fields eventId: z.string(), eventName: z.string(), + metadata: z.any().nullish(), // nested objects click: clickEventSchema, link: linkEventSchema, @@ -135,14 +138,3 @@ export const leadEventResponseSchema = z }) .merge(commonDeprecatedEventFields) .openapi({ ref: "LeadEvent", title: "LeadEvent" }); - -export const leadEventResponseSchemaExtended = leadEventResponseSchema.merge( - z.object({ - metadata: z - .string() - .nullish() - .transform((val) => (val === "" ? null : val)) - .default(null) - .openapi({ type: "string" }), - }), -); diff --git a/apps/web/lib/zod/schemas/sales.ts b/apps/web/lib/zod/schemas/sales.ts index 6c3eb58e1a..54b9c4a44e 100644 --- a/apps/web/lib/zod/schemas/sales.ts +++ b/apps/web/lib/zod/schemas/sales.ts @@ -133,6 +133,7 @@ export const saleEventSchemaTBEndpoint = z.object({ referer_url_processed: z.string().nullable(), qr: z.number().nullable(), ip: z.string().nullable(), + metadata: z.string().nullish(), }); // response from dub api @@ -140,17 +141,20 @@ export const saleEventResponseSchema = z .object({ event: z.literal("sale"), timestamp: z.coerce.string(), + // core event fields eventId: z.string(), eventName: z.string(), - // nested objects - link: linkEventSchema, - click: clickEventSchema, - customer: CustomerSchema, sale: trackSaleRequestSchema.pick({ amount: true, invoiceId: true, paymentProcessor: true, }), + metadata: z.any().nullish(), + // nested objects + link: linkEventSchema, + click: clickEventSchema, + customer: CustomerSchema, + // deprecated fields saleAmount: z .number() .describe("Deprecated. Use `sale.amount` instead.") @@ -165,14 +169,3 @@ export const saleEventResponseSchema = z }) .merge(commonDeprecatedEventFields) .openapi({ ref: "SaleEvent", title: "SaleEvent" }); - -export const saleEventResponseSchemaExtended = saleEventResponseSchema.merge( - z.object({ - metadata: z - .string() - .nullish() - .transform((val) => (val === "" ? null : val)) - .default(null) - .openapi({ type: "string" }), - }), -); diff --git a/apps/web/ui/customers/customer-activity-list.tsx b/apps/web/ui/customers/customer-activity-list.tsx index a6037f6dcd..de7d52aecc 100644 --- a/apps/web/ui/customers/customer-activity-list.tsx +++ b/apps/web/ui/customers/customer-activity-list.tsx @@ -85,18 +85,10 @@ const activityData = { lead: { icon: UserCheck, content: (event) => { - let metadata = null; - - try { - metadata = event.metadata ? JSON.parse(event.metadata) : null; - } catch (e) { - // - } - return (
{event.eventName || "New lead"} - {metadata && } + {event.metadata && }
); }, @@ -105,18 +97,10 @@ const activityData = { sale: { icon: MoneyBill2, content: (event) => { - let metadata = null; - - try { - metadata = event.metadata ? JSON.parse(event.metadata) : null; - } catch (e) { - // - } - return (
{event.eventName || "New sale"} - {metadata && } + {event.metadata && }
); }, From a240dc50cc6c4a7e47221f0392ba686311e0c3e4 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Mon, 4 Aug 2025 14:24:10 -0700 Subject: [PATCH 15/17] add metadata column --- apps/web/ui/analytics/events/events-table.tsx | 17 +++++++++++++++++ .../web/ui/analytics/events/metadata-viewer.tsx | 13 +++++++++---- .../analytics/events/use-column-visibility.ts | 10 ++++++---- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/apps/web/ui/analytics/events/events-table.tsx b/apps/web/ui/analytics/events/events-table.tsx index 10f326928e..b109440566 100644 --- a/apps/web/ui/analytics/events/events-table.tsx +++ b/apps/web/ui/analytics/events/events-table.tsx @@ -42,6 +42,7 @@ import DeviceIcon from "../device-icon"; import { TRIGGER_DISPLAY } from "../trigger-display"; import { EventsContext } from "./events-provider"; import { EXAMPLE_EVENTS_DATA } from "./example-data"; +import { MetadataViewer } from "./metadata-viewer"; import { RowMenuButton } from "./row-menu-button"; import { eventColumns, useColumnVisibility } from "./use-column-visibility"; @@ -473,6 +474,22 @@ export default function EventsTable({ ), }, ]), + // Metadata + { + id: "metadata", + header: "Metadata", + accessorKey: "metadata", + minSize: 120, + size: 120, + maxSize: 120, + cell: ({ getValue }) => { + const metadata = getValue(); + if (!metadata || Object.keys(metadata).length === 0) { + return -; + } + return ; + }, + }, // Menu { id: "menu", diff --git a/apps/web/ui/analytics/events/metadata-viewer.tsx b/apps/web/ui/analytics/events/metadata-viewer.tsx index 8474291379..9077061745 100644 --- a/apps/web/ui/analytics/events/metadata-viewer.tsx +++ b/apps/web/ui/analytics/events/metadata-viewer.tsx @@ -1,13 +1,15 @@ import { Button, Tooltip, useCopyToClipboard } from "@dub/ui"; -import { cn, truncate } from "@dub/utils"; +import { cn, pluralize, truncate } from "@dub/utils"; import { Check, Copy } from "lucide-react"; import { Fragment } from "react"; // Display the event metadata export function MetadataViewer({ metadata, + previewItems = 3, }: { metadata: Record; + previewItems?: number; }) { const [copied, copyToClipboard] = useCopyToClipboard(); @@ -34,9 +36,9 @@ export function MetadataViewer({ }) .flat(); - const hasMoreItems = displayEntries.length > 3; + const hasMoreItems = displayEntries.length > previewItems; const visibleEntries = hasMoreItems - ? displayEntries.slice(0, 3) + ? displayEntries.slice(0, previewItems) : displayEntries; return ( @@ -93,7 +95,10 @@ export function MetadataViewer({ className="rounded-md border border-neutral-200 bg-white px-1.5 py-0.5 hover:bg-neutral-50" > {hasMoreItems - ? `+${displayEntries.length - 3} more` + ? `+${displayEntries.length - previewItems} ${pluralize( + "item", + displayEntries.length - previewItems, + )}` : "View metadata"} diff --git a/apps/web/ui/analytics/events/use-column-visibility.ts b/apps/web/ui/analytics/events/use-column-visibility.ts index 79b8f86c1f..e5d7b82e49 100644 --- a/apps/web/ui/analytics/events/use-column-visibility.ts +++ b/apps/web/ui/analytics/events/use-column-visibility.ts @@ -5,6 +5,7 @@ import { VisibilityState } from "@tanstack/react-table"; export const eventColumns = { clicks: { all: [ + "timestamp", "trigger", "link", "country", @@ -17,12 +18,12 @@ export const eventColumns = { "referer", "refererUrl", "ip", - "timestamp", ], defaultVisible: ["timestamp", "link", "referer", "country", "device"], }, leads: { all: [ + "timestamp", "event", "link", "customer", @@ -36,12 +37,14 @@ export const eventColumns = { "referer", "refererUrl", "ip", - "timestamp", + "metadata", ], defaultVisible: ["timestamp", "event", "link", "customer", "referer"], }, sales: { all: [ + "timestamp", + "saleAmount", "event", "customer", "link", @@ -56,8 +59,7 @@ export const eventColumns = { "referer", "refererUrl", "ip", - "timestamp", - "saleAmount", + "metadata", ], defaultVisible: [ "timestamp", From c9ee2b9f0e251cc2989da3f88a03baae0b92bd4f Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Mon, 4 Aug 2025 15:17:00 -0700 Subject: [PATCH 16/17] add metadata.productId filter test --- apps/web/lib/analytics/get-analytics.ts | 17 ++++++++++------- apps/web/tests/analytics/index.test.ts | 19 +++++++++++++++++++ .../query-parser.test.ts} | 0 3 files changed, 29 insertions(+), 7 deletions(-) rename apps/web/tests/{misc/analytics-query-parser.test.ts => analytics/query-parser.test.ts} (100%) diff --git a/apps/web/lib/analytics/get-analytics.ts b/apps/web/lib/analytics/get-analytics.ts index 1d34e46cf1..ba41488cfe 100644 --- a/apps/web/lib/analytics/get-analytics.ts +++ b/apps/web/lib/analytics/get-analytics.ts @@ -212,13 +212,16 @@ export const getAnalytics = async (params: AnalyticsFilters) => { }, }); - return topPartnersData.map((item) => { - const partner = partners.find((p) => p.id === item.partnerId); - return { - ...item, - partner, - }; - }); + return topPartnersData + .map((item) => { + const partner = partners.find((p) => p.id === item.partnerId); + if (!partner) return null; + return { + ...item, + partner, + }; + }) + .filter((d) => d !== null); } // Return array for other endpoints diff --git a/apps/web/tests/analytics/index.test.ts b/apps/web/tests/analytics/index.test.ts index f32f0ca79d..4a2e3f01ae 100644 --- a/apps/web/tests/analytics/index.test.ts +++ b/apps/web/tests/analytics/index.test.ts @@ -39,4 +39,23 @@ describe.runIf(env.CI).sequential("GET /analytics", async () => { expect(parsed.success).toBeTruthy(); }); }); + + test("filter events by metadata.productId", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "sales", + workspaceId, + interval: "30d", + query: "metadata['productId']:premiumProductId", + }, + }); + + expect(status).toEqual(200); + + // check to make sure all events have metadata.productId equal to premiumProductId + expect( + data.every((event) => event.metadata?.productId === "premiumProductId"), + ).toBe(true); + }); }); diff --git a/apps/web/tests/misc/analytics-query-parser.test.ts b/apps/web/tests/analytics/query-parser.test.ts similarity index 100% rename from apps/web/tests/misc/analytics-query-parser.test.ts rename to apps/web/tests/analytics/query-parser.test.ts From d6ece15a87a3e1bfa42fb1351131f61a48715019 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Mon, 4 Aug 2025 15:40:08 -0700 Subject: [PATCH 17/17] =?UTF-8?q?parseFiltersFromQuery=20=E2=86=92=20query?= =?UTF-8?q?Parser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/web/lib/analytics/get-analytics.ts | 4 +-- apps/web/lib/analytics/get-events.ts | 4 +-- apps/web/lib/analytics/query-parser.ts | 2 +- apps/web/lib/zod/schemas/analytics.ts | 1 + apps/web/tests/analytics/query-parser.test.ts | 26 ++++++++----------- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/apps/web/lib/analytics/get-analytics.ts b/apps/web/lib/analytics/get-analytics.ts index ba41488cfe..15be5ab386 100644 --- a/apps/web/lib/analytics/get-analytics.ts +++ b/apps/web/lib/analytics/get-analytics.ts @@ -12,7 +12,7 @@ import { DIMENSIONAL_ANALYTICS_FILTERS, SINGULAR_ANALYTICS_ENDPOINTS, } from "./constants"; -import { parseFiltersFromQuery } from "./query-parser"; +import { queryParser } from "./query-parser"; import { AnalyticsFilters } from "./types"; import { getStartEndDates } from "./utils/get-start-end-dates"; @@ -102,7 +102,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => { : analyticsResponse[groupBy], }); - const filters = parseFiltersFromQuery(query); + const filters = queryParser(query); const response = await pipe({ ...params, diff --git a/apps/web/lib/analytics/get-events.ts b/apps/web/lib/analytics/get-events.ts index ddf7756d7c..bf8c75773f 100644 --- a/apps/web/lib/analytics/get-events.ts +++ b/apps/web/lib/analytics/get-events.ts @@ -21,7 +21,7 @@ import { saleEventResponseSchema, saleEventSchemaTBEndpoint, } from "../zod/schemas/sales"; -import { parseFiltersFromQuery } from "./query-parser"; +import { queryParser } from "./query-parser"; import { EventsFilters } from "./types"; import { getStartEndDates } from "./utils/get-start-end-dates"; @@ -76,7 +76,7 @@ export const getEvents = async (params: EventsFilters) => { }[eventType] ?? clickEventSchemaTBEndpoint, }); - const filters = parseFiltersFromQuery(query); + const filters = queryParser(query); const response = await pipe({ ...params, diff --git a/apps/web/lib/analytics/query-parser.ts b/apps/web/lib/analytics/query-parser.ts index 4a15728526..8518732a31 100644 --- a/apps/web/lib/analytics/query-parser.ts +++ b/apps/web/lib/analytics/query-parser.ts @@ -13,7 +13,7 @@ interface InternalFilter { } // Query parser that can parse the query string into a list of filters -export const parseFiltersFromQuery = ( +export const queryParser = ( query: EventsFilters["query"], allowedOperands = ["metadata"], ) => { diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index 7d83e67847..1c9e2dcabb 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -234,6 +234,7 @@ export const analyticsQuerySchema = z ), query: z .string() + .max(10000) .optional() .describe( "Search the events by a custom metadata value. Only available for lead and sale events.", diff --git a/apps/web/tests/analytics/query-parser.test.ts b/apps/web/tests/analytics/query-parser.test.ts index 7ef0669041..156af8a8e2 100644 --- a/apps/web/tests/analytics/query-parser.test.ts +++ b/apps/web/tests/analytics/query-parser.test.ts @@ -1,25 +1,23 @@ -import { parseFiltersFromQuery } from "@/lib/analytics/query-parser"; +import { queryParser } from "@/lib/analytics/query-parser"; import { describe, expect, it } from "vitest"; describe("Analytics Query Parser", () => { it("should parse simple nested property", () => { - const result = parseFiltersFromQuery("metadata['key']:value"); + const result = queryParser("metadata['key']:value"); expect(result).toEqual([ { operand: "metadata.key", operator: "equals", value: "value" }, ]); }); it("should parse nested property with double quotes", () => { - const result = parseFiltersFromQuery('metadata["key"]:"quoted value"'); + const result = queryParser('metadata["key"]:"quoted value"'); expect(result).toEqual([ { operand: "metadata.key", operator: "equals", value: "quoted value" }, ]); }); it("should parse deeply nested property", () => { - const result = parseFiltersFromQuery( - "metadata['level1']['level2']['level3']:value", - ); + const result = queryParser("metadata['level1']['level2']['level3']:value"); expect(result).toEqual([ { operand: "metadata.level1.level2.level3", @@ -30,9 +28,7 @@ describe("Analytics Query Parser", () => { }); it("should parse nested property with complex path", () => { - const result = parseFiltersFromQuery( - "metadata['user']['preferences']['theme']:dark", - ); + const result = queryParser("metadata['user']['preferences']['theme']:dark"); expect(result).toEqual([ { operand: "metadata.user.preferences.theme", @@ -43,36 +39,36 @@ describe("Analytics Query Parser", () => { }); it("should parse equals operator (:) for nested property", () => { - const result = parseFiltersFromQuery("metadata['key']:value"); + const result = queryParser("metadata['key']:value"); expect(result).toEqual([ { operand: "metadata.key", operator: "equals", value: "value" }, ]); }); it("should parse not equals operator for nested property", () => { - const result = parseFiltersFromQuery("metadata['status']!=completed"); + const result = queryParser("metadata['status']!=completed"); expect(result).toEqual([ { operand: "metadata.status", operator: "notEquals", value: "completed" }, ]); }); it("should handle empty query", () => { - const result = parseFiltersFromQuery(""); + const result = queryParser(""); expect(result).toBeUndefined(); }); it("should handle null query", () => { - const result = parseFiltersFromQuery(null as any); + const result = queryParser(null as any); expect(result).toBeUndefined(); }); it("should handle undefined query", () => { - const result = parseFiltersFromQuery(undefined as any); + const result = queryParser(undefined as any); expect(result).toBeUndefined(); }); it("should handle whitespace-only query", () => { - const result = parseFiltersFromQuery(" "); + const result = queryParser(" "); expect(result).toBeUndefined(); }); });