Merge pull request #2 from openstatusHQ/feat/data-table-endpoints

mxkaske · web-flow · commit c50ee2910a6a · 2025-03-30T11:13:22.000+02:00
feat: add data-table pipe endpoints
diff --git a/README.md b/README.md
@@ -19,6 +19,10 @@ Currently supported [notification channels](#notification-channels) are:
 - Campsite
 - Telegram
 
+We also provide a 6 min. [YouTube](https://www.youtube.com/watch?v=cpurWC9Co1U) video for more visual content. If you have any trouble, feel free to open an issue, join our [Discord](https://openstatus.dev/discord) or message us to [ping@openstatus.dev](mailto:ping@openstatus.dev)
+
+If storing the data to [Tinybird](https://www.tinybird.co), you can use the [OpenStatus Light Viewer](https://data-table.openstatus.dev/light) to filter and learn from your pings.
+
 ## Getting Started
 
 To work/test it locally:
@@ -56,28 +60,59 @@ Add `--dry-run` to the commands if you want to first test them:
 ```bash
 tb push tb/datasources/http_ping_responses.datasource
 tb push tb/pipes/endpoint__get_http.pipe
+tb push tb/pipes/endpoint__get_http_stats.pipe
+tb push tb/pipes/endpoint__get_http_facets.pipe
 ```
 
 #### API endpoint
 
-If you've connected your database to Tinybird, we provide you a simple `GET` endpoint to query your periodical pings.
+If you've connected your database to Tinybird, we provide you a some simple `GET` endpoint to query your pings.
+
+The `/api/get` endpoint (`endpoint__get_http.pipe`) accepts the following query parameters:
+
+| **Parameter**    | **Type** | **Description**                                       |
+| ---------------- | -------- | ----------------------------------------------------- |
+| `pageIndex`      | `number` | The page number for pagination (starts at 0)          |
+| `pageSize`       | `number` | Number of results per page (defaults to 100)          |
+| `orderBy`        | `string` | Column to sort by (defaults to 'timestamp')           |
+| `orderDir`       | `string` | Sort direction - 'ASC' or 'DESC' (defaults to 'DESC') |
+| `latencyStart`   | `number` | Filter results with latency >= this value (ms)        |
+| `latencyEnd`     | `number` | Filter results with latency <= this value (ms)        |
+| `statuses`       | `string` | Filter by HTTP status codes (comma-separated list)    |
+| `regions`        | `string` | Filter by edge regions (comma-separated list)         |
+| `methods`        | `string` | Filter by HTTP methods (comma-separated list)         |
+| `url`            | `string` | Filter URLs containing this string                    |
+| `timestampStart` | `number` | Unix timestamp in ms (defaults to 30 days prior)      |
+| `timestampEnd`   | `number` | Unix timestamp in ms (defaults to now)                |
+
+Moreover, to make best use of the [OpenStatus Light Viewer](https://data-table.openstatus.dev/light), we support two more API endpoints. They both extend the above query params.
+
+- `/api/stats` endpoint (`endpoint__get_http_stats.pipe`) for the chart values (incl. `interval` param)
 
-The endpoint accepts the following query parameters:
+| **Parameter** | **Type** | **Description**                                                                     |
+| ------------- | -------- | ----------------------------------------------------------------------------------- |
+| `interval`    | `number` | Interval of grouped data in minutes (defaults to 1_440 minutes equivalent to 1 day) |
 
-| **Parameter**  | **Type** | **Description**                                       |
-| -------------- | -------- | ----------------------------------------------------- |
-| `pageIndex`    | `number` | The page number for pagination (starts at 0)          |
-| `pageSize`     | `number` | Number of results per page (defaults to 100)          |
-| `orderBy`      | `string` | Column to sort by (defaults to 'timestamp')           |
-| `orderDir`     | `string` | Sort direction - 'ASC' or 'DESC' (defaults to 'DESC') |
-| `latencyStart` | `number` | Filter results with latency >= this value (ms)        |
-| `latencyEnd`   | `number` | Filter results with latency <= this value (ms)        |
-| `statuses`     | `string` | Filter by HTTP status codes (comma-separated list)    |
-| `regions`      | `string` | Filter by edge regions (comma-separated list)         |
-| `methods`      | `string` | Filter by HTTP methods (comma-separated list)         |
-| `url`          | `string` | Filter URLs containing this string                    |
+- `/api/facets` endpoint (`endpoint__get_http_facets.pipe`) for the table facets (no additional param)
 
-> Remember that this `/api/get` endpoint will be accessible by anyone if you are not securing it yourself.
+All endpoints are just a layer to access the tinybird pipe responses. They share the following response object example:
+
+```json
+{
+  "meta": [],
+  "data": [],
+  "rows": 20,
+  "rows_before_limit_at_least": 100,
+  "statistics": {
+    "elapsed": 0.004015279,
+    "rows_read": 2845,
+    "bytes_read": 725311
+  }
+}
+```
+
+> [!NOTE]
+> The `/api/get`, `/api/stats`, `/api/facets` endpoints will be accessible by anyone if you are not securing it yourself.
 
 ## Configuration
 
diff --git a/api/_ping.ts b/api/_ping.ts
@@ -82,9 +82,12 @@ async function check(request: PingRequest): Promise<PingResponse> {
   clearTimeout(timeoutId);
 
   const latency = end - start;
-  const body = (await res.text()).slice(0, 1000); // limit to 1000 characters to avoid saving large payloads in tb
+  const text = await res.text();
   const headers = Object.fromEntries(res.headers.entries());
 
+  // NOTE: we limit the body to 1000 characters to avoid saving large payloads in tb
+  const body = text.length > 1000 ? text.slice(0, 1000) + "..." : text;
+
   return {
     url: request.url,
     method: request.method,
diff --git a/api/facets.ts b/api/facets.ts
@@ -0,0 +1,33 @@
+const EVENT_NAME = "endpoint__get_http_facets__v0";
+
+export async function GET(req: Request) {
+  if (!process.env.TINYBIRD_TOKEN) {
+    return new Response("No Connected Database", { status: 200 });
+  }
+
+  const reqUrl = new URL(req.url);
+  const searchParams = reqUrl.searchParams;
+
+  const tbUrl = new URL(
+    `https://api.tinybird.co/v0/pipes/${EVENT_NAME}.json?${searchParams.toString()}`
+  );
+
+  const result = await fetch(tbUrl, {
+    method: "GET",
+    headers: {
+      Authorization: `Bearer ${process.env.TINYBIRD_TOKEN}`,
+    },
+  })
+    .then((r) => r.json())
+    .then((r) => r)
+    .catch((e) => e.toString());
+
+  if (!result?.data) {
+    console.error(`Error with: ${JSON.stringify(result)}`);
+    return new Response("Internal Server Error", { status: 500 });
+  }
+
+  return new Response(JSON.stringify(result), {
+    headers: { "Content-Type": "application/json" },
+  });
+}
diff --git a/api/get.ts b/api/get.ts
@@ -7,31 +7,10 @@ export async function GET(req: Request) {
 
   const reqUrl = new URL(req.url);
   const searchParams = reqUrl.searchParams;
-  const pageIndex = searchParams.get("pageIndex");
-  const pageSize = searchParams.get("pageSize");
-  const orderBy = searchParams.get("orderBy");
-  const orderDir = searchParams.get("orderDir");
-  const latencyStart = searchParams.get("latencyStart");
-  const latencyEnd = searchParams.get("latencyEnd");
-  const url = searchParams.get("url");
-  // REMINDER: can be comma separated list as tb supports arrays.
-  const statuses = searchParams.get("statuses");
-  const methods = searchParams.get("methods");
-  const regions = searchParams.get("regions");
 
-  const tbUrl = new URL(`https://api.tinybird.co/v0/pipes/${EVENT_NAME}.json`);
-
-  // Only set params if they are provided
-  if (pageIndex) tbUrl.searchParams.set("pageIndex", pageIndex);
-  if (pageSize) tbUrl.searchParams.set("pageSize", pageSize);
-  if (orderBy) tbUrl.searchParams.set("orderBy", orderBy);
-  if (orderDir) tbUrl.searchParams.set("orderDir", orderDir);
-  if (latencyStart) tbUrl.searchParams.set("latencyStart", latencyStart);
-  if (latencyEnd) tbUrl.searchParams.set("latencyEnd", latencyEnd);
-  if (statuses) tbUrl.searchParams.set("statuses", statuses);
-  if (methods) tbUrl.searchParams.set("methods", methods);
-  if (regions) tbUrl.searchParams.set("regions", regions);
-  if (url) tbUrl.searchParams.set("url", url);
+  const tbUrl = new URL(
+    `https://api.tinybird.co/v0/pipes/${EVENT_NAME}.json?${searchParams.toString()}`
+  );
 
   const result = await fetch(tbUrl, {
     method: "GET",
@@ -48,7 +27,7 @@ export async function GET(req: Request) {
     return new Response("Internal Server Error", { status: 500 });
   }
 
-  return new Response(JSON.stringify(result.data), {
+  return new Response(JSON.stringify(result), {
     headers: { "Content-Type": "application/json" },
   });
 }
diff --git a/api/index.ts b/api/index.ts
@@ -6,7 +6,7 @@ const region = process.env.VERCEL_REGION || "unknown";
 
 const META_DESCRIPTION =
   "Lightweight one-click solution to monitor your endpoints across multiple regions.";
-const META_TITLE = "OpenStatus | Vercel Edge Ping";
+const META_TITLE = "OpenStatus Light | Vercel Edge Ping";
 const META_URL = "https://light.openstatus.dev";
 const META_OG_IMAGE =
   "https://www.openstatus.dev/api/og?title=Vercel%20Edge%20Ping&description=Lightweight%20one-click%20solution%20to%20monitor%20your%20endpoints%20across%20multiple%20regions.&footer=light.openstatus.dev";
@@ -63,9 +63,13 @@ export async function GET(): Promise<Response> {
       <p>
         Be notified via Slack, Discord, Campsite or Telegram if >50% of the regions are down.
       </p>
-      <p>
-        Get the latest cron requests by visiting <a href="/api/get">/api/get</a>.
-      </p>
+      <p>More endpoints (with search params support):</p>
+      <ul>
+        <li>Get the latest requests by visiting <a href="/api/get">/api/get</a>.</li>
+        <li>Get the latest stats by visiting <a href="/api/stats">/api/stats</a>.</li>
+        <li>Get the latest facets by visiting <a href="/api/facets">/api/facets</a>.</li>
+      </ul>
+      <p>Once deployed, access your data by visiting <a href="https://logs.run/light">logs.run/light</a> and changing the base URL to your endpoint.</p>
       <p>
         Read more on <a href="https://github.com/openstatusHQ/vercel-edge-ping">GitHub</a>
         &#183;
diff --git a/api/stats.ts b/api/stats.ts
@@ -0,0 +1,33 @@
+const EVENT_NAME = "endpoint__get_http_stats__v0";
+
+export async function GET(req: Request) {
+  if (!process.env.TINYBIRD_TOKEN) {
+    return new Response("No Connected Database", { status: 200 });
+  }
+
+  const reqUrl = new URL(req.url);
+  const searchParams = reqUrl.searchParams;
+
+  const tbUrl = new URL(
+    `https://api.tinybird.co/v0/pipes/${EVENT_NAME}.json?${searchParams.toString()}`
+  );
+
+  const result = await fetch(tbUrl, {
+    method: "GET",
+    headers: {
+      Authorization: `Bearer ${process.env.TINYBIRD_TOKEN}`,
+    },
+  })
+    .then((r) => r.json())
+    .then((r) => r)
+    .catch((e) => e.toString());
+
+  if (!result?.data) {
+    console.error(`Error with: ${JSON.stringify(result)}`);
+    return new Response("Internal Server Error", { status: 500 });
+  }
+
+  return new Response(JSON.stringify(result), {
+    headers: { "Content-Type": "application/json" },
+  });
+}
diff --git a/tb/pipes/endpoint__get_http.pipe b/tb/pipes/endpoint__get_http.pipe
@@ -4,7 +4,17 @@ NODE endpoint
 SQL >
 
     %
-    SELECT *
+    SELECT
+        *,
+        multiIf(
+            status >= 200 AND status < 400,
+            'success',
+            status >= 400 AND status < 500,
+            'warning',
+            status >= 500,
+            'error',
+            'unknown'  -- Default case if none of the above match
+        ) as level
     from http_ping_responses__v0
     WHERE
         1 = 1
@@ -16,10 +26,22 @@ SQL >
         {% if defined(url) %} AND url ILIKE concat('%', {{ String(url) }}, '%') {% end %}
         {% if defined(timestampStart) %} AND timestamp >= {{ Int64(timestampStart) }} {% end %}
         {% if defined(timestampEnd) %} AND timestamp <= {{ Int64(timestampEnd) }} {% end %}
+        {% if defined(levels) %}
+            AND multiIf(
+                status >= 200 AND status < 400,
+                'success',
+                status >= 400 AND status < 500,
+                'warning',
+                status >= 500,
+                'error',
+                'unknown'
+            )
+            IN {{ Array(levels, String) }}
+        {% end %}
     ORDER BY
         {{ column(orderBy, 'timestamp') }}
         {% if defined(orderDir) and String(orderDir) == "ASC" %} ASC
         {% else %} DESC
         {% end %}
-    LIMIT {{ Int32(pageSize, 100) }}
+    {% if defined(pageSize) %} LIMIT {{ Int32(pageSize, 100) }} {% end %}
     OFFSET {{ Int32(pageIndex, 0) * Int32(pageSize, 100) }}
diff --git a/tb/pipes/endpoint__get_http_facets.pipe b/tb/pipes/endpoint__get_http_facets.pipe
@@ -0,0 +1,25 @@
+VERSION 0
+
+NODE endpoint
+SQL >
+
+    %
+    SELECT facet, value, COUNT(*) AS count
+    FROM
+        (
+            SELECT
+                arrayJoin(
+                    [
+                        ('method', method),
+                        ('url', url),
+                        ('status', toString(status)),
+                        ('region', region),
+                        ('latency', toString(latency)),
+                        ('level', level)
+                    ]
+                ) AS pair,
+                pair .1 AS facet,
+                pair .2 AS value
+            FROM endpoint__get_http__v0
+        )
+    GROUP BY facet, value
diff --git a/tb/pipes/endpoint__get_http_stats.pipe b/tb/pipes/endpoint__get_http_stats.pipe
@@ -0,0 +1,29 @@
+VERSION 0
+
+NODE endpoint
+SQL >
+
+    %
+    WITH
+        filtered_data AS (
+            SELECT timestamp, status FROM endpoint
+            {% if defined(timestampStart) and defined(timestampEnd) %}
+            UNION ALL
+            SELECT {{ Int64(timestampStart) }} AS timestamp, 0 AS status  -- Start boundary
+            UNION ALL
+            SELECT {{ Int64(timestampEnd) }} AS timestamp, 0 AS status    -- End boundary
+            {% end %}
+        )
+    SELECT
+        toUnixTimestamp(
+            toStartOfInterval(
+                toDateTime(timestamp / 1000), INTERVAL {{ Int32(interval, 1_440) }} MINUTE
+            )
+        ) * 1000 AS timestamp,
+        countIf(status >= 200 AND status < 400) AS success,
+        countIf(status >= 400 AND status < 500) AS warning,
+        countIf(status >= 500) AS error
+    FROM filtered_data
+    GROUP BY timestamp
+    ORDER BY timestamp
+    WITH FILL STEP {{ Int32(interval, 1_440) }} * 60 * 1000
