feat: parse monthlyUsage.dailyServiceUsages[].date as Date (#519)

This field does not end with "At", which means it is ignored by the
existing parseDateFields helper. This PR adjusts this helper to support
parsing other fields as well.

To avoid accidental braking changes by applying the new parser on all
existing data structures, the helper now has a new optional
shouldParseField() param to identify other fields to be parsed. Without
this param, the helper behaves just like before.

The actual user-facing change (`typeof
monthlyUsage.dailyServiceUsages[].date === Date`) was tested manually
and will be covered by the integration tests in `apify-core`.

Author: tobice

src/resource_clients/user.ts

@@ -35,7 +35,10 @@ export class UserClient extends ResourceClient {
         };
         try {
             const response = await this.httpClient.call(requestOpts);
-            return cast(parseDateFields(pluckData(response.data)));
+            return cast(parseDateFields(
+                pluckData(response.data),
+                // Convert  monthlyUsage.dailyServiceUsages[].date to Date (by default it's ignored by parseDateFields)
+                /* shouldParseField = */ (key) => key === 'date'));
         } catch (err) {
             catchNotFoundOrThrow(err as ApifyApiError);
         }

src/utils.ts

@@ -2,6 +2,7 @@ import type { Readable } from 'node:stream';
 import util from 'util';
 import zlib from 'zlib';
 
+import log from '@apify/log';
 import ow from 'ow';
 import type { TypedArray, JsonValue } from 'type-fest';
 
@@ -12,8 +13,6 @@ import {
 } from './resource_clients/request_queue';
 import { WebhookUpdateData } from './resource_clients/webhook';
 
-const PARSE_DATE_FIELDS_MAX_DEPTH = 3; // obj.data.someArrayField.[x].field
-const PARSE_DATE_FIELDS_KEY_SUFFIX = 'At';
 const NOT_FOUND_STATUS_CODE = 404;
 const RECORD_NOT_FOUND_TYPE = 'record-not-found';
 const RECORD_OR_TOKEN_NOT_FOUND_TYPE = 'record-or-token-not-found';
@@ -51,24 +50,37 @@ type ReturnJsonObject = { [Key in string]?: ReturnJsonValue; };
 type ReturnJsonArray = Array<ReturnJsonValue>;
 
 /**
- * Helper function that traverses JSON structure and parses fields such as modifiedAt or createdAt to dates.
+ * Traverses JSON structure and converts fields that end with "At" to a Date object (fields such as "modifiedAt" or
+ * "createdAt").
+ *
+ * If you want parse other fields as well, you can provide a custom matcher function shouldParseField(). This
+ * admittedly awkward approach allows this function to be reused for various purposes without introducing potential
+ * breaking changes.
+ *
+ * If the field cannot be converted to Date, it is left as is.
  */
-export function parseDateFields(input: JsonValue, depth = 0): ReturnJsonValue {
-    if (depth > PARSE_DATE_FIELDS_MAX_DEPTH) return input as ReturnJsonValue;
-    if (Array.isArray(input)) return input.map((child) => parseDateFields(child, depth + 1));
+export function parseDateFields(input: JsonValue, shouldParseField: ((key: string) => boolean) | null = null, depth = 0): ReturnJsonValue {
+    // Don't go too deep to avoid stack overflows (especially if there is a circular reference). The depth of 3
+    // corresponds to obj.data.someArrayField.[x].field and should be generally enough.
+    if (depth > 3) {
+        log.warning('parseDateFields: Maximum depth reached, not parsing further');
+        return input as ReturnJsonValue;
+    }
+
+    if (Array.isArray(input)) return input.map((child) => parseDateFields(child, shouldParseField, depth + 1));
     if (!input || typeof input !== 'object') return input;
 
     return Object.entries(input).reduce((output, [k, v]) => {
         const isValObject = !!v && typeof v === 'object';
-        if (k.endsWith(PARSE_DATE_FIELDS_KEY_SUFFIX)) {
+        if (k.endsWith('At') || (shouldParseField && shouldParseField(k))) {
             if (v) {
                 const d = new Date(v as string);
                 output[k] = Number.isNaN(d.getTime()) ? v as string : d;
             } else {
                 output[k] = v;
             }
         } else if (isValObject || Array.isArray(v)) {
-            output[k] = parseDateFields(v!, depth + 1);
+            output[k] = parseDateFields(v!, shouldParseField, depth + 1);
         } else {
             output[k] = v;
         }

test/utils.test.js

@@ -64,21 +64,54 @@ describe('utils.parseDateFields()', () => {
         expectDatesDame(parsed.data.foo[1].fooAt, date);
     });
 
-    test('doesn\'t parse falsy values', () => {
+    test('does not parse falsy values', () => {
         const original = { fooAt: null, barAt: '' };
         const parsed = utils.parseDateFields(JSON.parse(JSON.stringify(original)));
 
         expect(parsed.fooAt).toEqual(null);
         expect(parsed.barAt).toEqual('');
     });
 
-    test('doesn\'t mangle non-date strings', () => {
+    test('does not mangle non-date strings', () => {
         const original = { fooAt: 'three days ago', barAt: '30+ days' };
         const parsed = utils.parseDateFields(original);
 
         expect(parsed.fooAt).toEqual('three days ago');
         expect(parsed.barAt).toEqual('30+ days');
     });
+
+    test('ignores perfectly fine RFC 3339 date', () => {
+        const original = { fooAt: 'three days ago', date: '2024-02-18T00:00:00.000Z' };
+        const parsed = utils.parseDateFields(original);
+
+        expect(parsed.fooAt).toEqual('three days ago');
+        expect(parsed.date).toEqual('2024-02-18T00:00:00.000Z');
+    });
+
+    test('parses custom date field detected by matcher', () => {
+        const original = { fooAt: 'three days ago', date: '2024-02-18T00:00:00.000Z' };
+
+        const parsed = utils.parseDateFields(original, (key) => key === 'date');
+
+        expect(parsed.fooAt).toEqual('three days ago');
+        expect(parsed.date).toBeInstanceOf(Date);
+    });
+
+    test('parses custom nested date field detected by matcher', () => {
+        const original = { fooAt: 'three days ago', foo: { date: '2024-02-18T00:00:00.000Z' } };
+
+        const parsed = utils.parseDateFields(original, (key) => key === 'date');
+
+        expect(parsed.foo.date).toBeInstanceOf(Date);
+    });
+
+    test('does not mangle non-date strings even when detected by matcher', () => {
+        const original = { fooAt: 'three days ago', date: '30+ days' };
+        const parsed = utils.parseDateFields(original, (key) => key === 'date');
+
+        expect(parsed.fooAt).toEqual('three days ago');
+        expect(parsed.date).toEqual('30+ days');
+    });
 });
 
 describe('utils.stringifyWebhooksToBase64()', () => {