Filter events & analytics by metadata

apps/web/app/(ee)/api/events/route.ts

@@ -10,6 +10,7 @@ import { eventsQuerySchema } from "@/lib/zod/schemas/analytics";
 import { Folder, Link } from "@dub/prisma/client";
 import { NextResponse } from "next/server";
 
+// GET /api/events
 export const GET = withWorkspace(
   async ({ searchParams, workspace, session }) => {
     throwIfClicksUsageExceeded(workspace);

apps/web/lib/analytics/get-analytics.ts

@@ -12,6 +12,7 @@ import {
   DIMENSIONAL_ANALYTICS_FILTERS,
   SINGULAR_ANALYTICS_ENDPOINTS,
 } from "./constants";
+import { parseFiltersFromQuery } from "./query-parser";
 import { AnalyticsFilters } from "./types";
 import { getStartEndDates } from "./utils/get-start-end-dates";
 
@@ -32,6 +33,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
     timezone = "UTC",
     isDeprecatedClicksEndpoint = false,
     dataAvailableFrom,
+    query,
   } = params;
 
   const tagIds = combineTagIds(params);
@@ -100,6 +102,8 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
         : analyticsResponse[groupBy],
   });
 
+  const filters = parseFiltersFromQuery(query);
+
   const response = await pipe({
     ...params,
     ...(UTM_TAGS_PLURAL_LIST.includes(groupBy)
@@ -115,6 +119,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
     timezone,
     country,
     region,
+    filters: filters ? JSON.stringify(filters) : undefined,
   });
 
   if (groupBy === "count") {

apps/web/lib/analytics/get-events.ts

@@ -21,6 +21,7 @@ import {
   saleEventResponseSchema,
   saleEventSchemaTBEndpoint,
 } from "../zod/schemas/sales";
+import { parseFiltersFromQuery } from "./query-parser";
 import { EventsFilters } from "./types";
 import { getStartEndDates } from "./utils/get-start-end-dates";
 
@@ -39,6 +40,7 @@ export const getEvents = async (params: EventsFilters) => {
     order,
     sortOrder,
     dataAvailableFrom,
+    query,
   } = params;
 
   const { startDate, endDate } = getStartEndDates({
@@ -74,6 +76,8 @@ export const getEvents = async (params: EventsFilters) => {
       }[eventType] ?? clickEventSchemaTBEndpoint,
   });
 
+  const filters = parseFiltersFromQuery(query);
+
   const response = await pipe({
     ...params,
     eventType,
@@ -85,6 +89,7 @@ export const getEvents = async (params: EventsFilters) => {
     offset: (params.page - 1) * params.limit,
     start: startDate.toISOString().replace("T", " ").replace("Z", ""),
     end: endDate.toISOString().replace("T", " ").replace("Z", ""),
+    filters: filters ? JSON.stringify(filters) : undefined,
   });
 
   const [linksMap, customersMap] = await Promise.all([

apps/web/lib/analytics/query-parser.ts

@@ -0,0 +1,126 @@
+import { EventsFilters } from "./types";
+
+interface InternalFilter {
+  operand: string;
+  operator:
+    | "equals"
+    | "notEquals"
+    | "greaterThan"
+    | "lessThan"
+    | "greaterThanOrEqual"
+    | "lessThanOrEqual";
+  value: string;
+}
+
+// Query parser that can parse the query string into a list of filters
+export const parseFiltersFromQuery = (
+  query: EventsFilters["query"],
+  allowedOperands = ["metadata"],
+) => {
+  if (!query) {
+    return undefined;
+  }
+
+  const filters: InternalFilter[] = [];
+
+  // Split the query by logical operators (AND/OR) to handle multiple conditions
+  // For now, we'll focus on single conditions, but this structure allows for future expansion
+  const conditions = query.split(/\s+(?:AND|and|OR|or)\s+/);
+
+  for (const condition of conditions) {
+    const trimmedCondition = condition.trim();
+
+    if (!trimmedCondition) {
+      continue;
+    }
+
+    const filter = parseCondition(trimmedCondition);
+
+    if (!filter) {
+      continue;
+    }
+
+    const isAllowed = allowedOperands.some((allowed) => {
+      if (filter.operand === allowed) {
+        return true;
+      }
+
+      if (filter.operand.startsWith(`${allowed}.`)) {
+        return true;
+      }
+
+      return false;
+    });
+
+    if (!isAllowed) {
+      continue;
+    }
+
+    filters.push(filter);
+  }
+
+  return filters.length > 0 ? filters : undefined;
+};
+
+// Parses a single condition in the format: field:value, field>value, or metadata['key']:value
+function parseCondition(condition: string): InternalFilter | null {
+  // This regex captures:
+  // 1. field - either a regular field name OR metadata with bracket notation (supports both single and double quotes)
+  // 2. operator - :, >, <, >=, <=, !=
+  // 3. value - the value after the operator (supports quoted and unquoted values)
+  const unifiedPattern =
+    /^([a-zA-Z_][a-zA-Z0-9_]*|metadata\[['"][^'"]*['"]\](?:\[['"][^'"]*['"]\])*)\s*([:><=!]+)\s*(.+)$/;
+
+  const match = condition.match(unifiedPattern);
+
+  if (!match) {
+    return null;
+  }
+
+  // Extract the matched groups
+  const [, fieldOrMetadata, operator, value] = match;
+
+  let operand: string;
+
+  // Determine the operand based on whether it's metadata or a regular field
+  if (fieldOrMetadata.startsWith("metadata")) {
+    const keyPath = fieldOrMetadata.replace(/^metadata/, "");
+
+    const extractedKey = keyPath
+      .replace(/^\[['"]|['"]\]$/g, "") // Remove leading [' or [" and trailing '] or "]
+      .replace(/\[['"]/g, ".") // Replace [' or [" with .
+      .replace(/['"]\]/g, ""); // Remove trailing '] or "]
+
+    operand = `metadata.${extractedKey}`;
+  } else {
+    operand = fieldOrMetadata;
+  }
+
+  return {
+    operand,
+    operator: mapOperator(operator),
+    value: value.trim().replace(/^['"`]|['"`]$/g, ""),
+  };
+}
+
+// Maps operator strings to our internal operator types
+function mapOperator(operator: string): InternalFilter["operator"] {
+  switch (operator) {
+    case ":":
+    case "=":
+      return "equals";
+    case ">":
+      return "greaterThan";
+    case "<":
+      return "lessThan";
+    case ">=":
+      return "greaterThanOrEqual";
+    case "<=":
+      return "lessThanOrEqual";
+    case "!=":
+      return "notEquals";
+    default:
+      // For unsupported operators, default to equals
+      return "equals";
+  }
+}

apps/web/lib/zod/schemas/analytics.ts

@@ -232,6 +232,15 @@ export const analyticsQuerySchema = z
       .describe(
         "Filter sales by type: 'new' for first-time purchases, 'recurring' for repeat purchases. If undefined, returns both.",
       ),
+    query: z
+      .string()
+      .optional()
+      .describe(
+        "Search the events by a custom metadata value. Only available for lead and sale events.",
+      )
+      .openapi({
+        example: "metadata['key']:'value'",
+      }),
   })
   .merge(utmTagsSchema);
 
@@ -261,6 +270,10 @@ export const analyticsFilterTB = z
       .optional()
       .describe("The folder IDs to retrieve analytics for."),
     isMegaFolder: z.boolean().optional(),
+    filters: z
+      .string()
+      .optional()
+      .describe("The filters to apply to the analytics."),
   })
   .merge(
     analyticsQuerySchema.pick({

apps/web/tests/misc/analytics-query-parser.test.ts

@@ -0,0 +1,78 @@
+import { parseFiltersFromQuery } from "@/lib/analytics/query-parser";
+import { describe, expect, it } from "vitest";
+
+describe("Analytics Query Parser", () => {
+  it("should parse simple nested property", () => {
+    const result = parseFiltersFromQuery("metadata['key']:value");
+    expect(result).toEqual([
+      { operand: "metadata.key", operator: "equals", value: "value" },
+    ]);
+  });
+
+  it("should parse nested property with double quotes", () => {
+    const result = parseFiltersFromQuery('metadata["key"]:"quoted value"');
+    expect(result).toEqual([
+      { operand: "metadata.key", operator: "equals", value: "quoted value" },
+    ]);
+  });
+
+  it("should parse deeply nested property", () => {
+    const result = parseFiltersFromQuery(
+      "metadata['level1']['level2']['level3']:value",
+    );
+    expect(result).toEqual([
+      {
+        operand: "metadata.level1.level2.level3",
+        operator: "equals",
+        value: "value",
+      },
+    ]);
+  });
+
+  it("should parse nested property with complex path", () => {
+    const result = parseFiltersFromQuery(
+      "metadata['user']['preferences']['theme']:dark",
+    );
+    expect(result).toEqual([
+      {
+        operand: "metadata.user.preferences.theme",
+        operator: "equals",
+        value: "dark",
+      },
+    ]);
+  });
+
+  it("should parse equals operator (:) for nested property", () => {
+    const result = parseFiltersFromQuery("metadata['key']:value");
+    expect(result).toEqual([
+      { operand: "metadata.key", operator: "equals", value: "value" },
+    ]);
+  });
+
+  it("should parse not equals operator for nested property", () => {
+    const result = parseFiltersFromQuery("metadata['status']!=completed");
+    expect(result).toEqual([
+      { operand: "metadata.status", operator: "notEquals", value: "completed" },
+    ]);
+  });
+
+  it("should handle empty query", () => {
+    const result = parseFiltersFromQuery("");
+    expect(result).toBeUndefined();
+  });
+
+  it("should handle null query", () => {
+    const result = parseFiltersFromQuery(null as any);
+    expect(result).toBeUndefined();
+  });
+
+  it("should handle undefined query", () => {
+    const result = parseFiltersFromQuery(undefined as any);
+    expect(result).toBeUndefined();
+  });
+
+  it("should handle whitespace-only query", () => {
+    const result = parseFiltersFromQuery("   ");
+    expect(result).toBeUndefined();
+  });
+});

packages/tinybird/pipes/v2_events.pipe

@@ -134,6 +134,31 @@ SQL >
             AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%')
         {% end %}
         {% if defined(url) %} AND url = {{ url }} {% end %}
+
+        {% if defined(filters) %}
+            {% for item in JSON(filters, '[]') %}
+                {% if item.get('operand', '').startswith('metadata.') %}
+                    {% set metadataKey = item.get('operand', '').split('.')[1] %}
+                    {% set operator = item.get('operator', 'equals') %}
+                    {% set value = item.get('value', '') %}
+
+                    {% if operator == 'equals' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }}
+                    {% elif operator == 'notEquals' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }}
+                    {% elif operator == 'greaterThan' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }}
+                    {% elif operator == 'lessThan' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }}
+                    {% elif operator == 'greaterThanOrEqual' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }}
+                    {% elif operator == 'lessThanOrEqual' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }}
+                    {% end %}
+                {% end %}
+            {% end %}
+        {% end %}
+
     ORDER BY timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %}
     LIMIT {{ Int32(limit, 100) }}
     {% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %}
@@ -169,12 +194,48 @@ SQL >
         {% if defined(os) %} AND os = {{ os }} {% end %}
         {% if defined(referer) %} AND referer = {{ referer }} {% end %}
         {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %}
-        {% if defined(utm_source) %} AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %}
-        {% if defined(utm_medium) %} AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %}
-        {% if defined(utm_campaign) %} AND url LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %}
-        {% if defined(utm_term) %} AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %}
-        {% if defined(utm_content) %} AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %}
+        {% if defined(utm_source) %}
+            AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%')
+        {% end %}
+        {% if defined(utm_medium) %}
+            AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%')
+        {% end %}
+        {% if defined(utm_campaign) %}
+            AND url
+            LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%')
+        {% end %}
+        {% if defined(utm_term) %}
+            AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%')
+        {% end %}
+        {% if defined(utm_content) %}
+            AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%')
+        {% end %}
         {% if defined(url) %} AND url = {{ url }} {% end %}
+
+        {% if defined(filters) %}
+            {% for item in JSON(filters, '[]') %}
+                {% if item.get('operand', '').startswith('metadata.') %}
+                    {% set metadataKey = item.get('operand', '').split('.')[1] %}
+                    {% set operator = item.get('operator', 'equals') %}
+                    {% set value = item.get('value', '') %}
+
+                    {% if operator == 'equals' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }}
+                    {% elif operator == 'notEquals' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }}
+                    {% elif operator == 'greaterThan' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }}
+                    {% elif operator == 'lessThan' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }}
+                    {% elif operator == 'greaterThanOrEqual' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }}
+                    {% elif operator == 'lessThanOrEqual' %}
+                        AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }}
+                    {% end %}
+                {% end %}
+            {% end %}
+        {% end %}
+
     ORDER BY timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %}
     LIMIT {{ Int32(limit, 100) }}
     {% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %}