diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json index e7f075516..402eb92b8 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json @@ -5,7 +5,7 @@ "url": "github.com/firebase/extensions.git", "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker" }, - "version": "1.1.2", + "version": "1.1.3", "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports", "main": "./lib/index.js", "scripts": { @@ -28,7 +28,8 @@ "generate-schema": "^2.6.0", "inquirer": "^6.4.0", "lodash": "^4.17.14", - "sql-formatter": "^2.3.3" + "sql-formatter": "^2.3.3", + "traverse": "^0.6.6" }, "devDependencies": { "typescript": "^3.4.5", diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts index 2a81179b2..4125e155b 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts @@ -15,6 +15,8 @@ */ import * as bigquery from "@google-cloud/bigquery"; +import * as firebase from "firebase-admin"; +import * as traverse from "traverse"; import { RawChangelogSchema, RawChangelogViewSchema } from "./schema"; import { latestConsistentSnapshotView } from "./snapshot"; @@ -52,6 +54,7 @@ export class FirestoreBigQueryEventHistoryTracker async record(events: FirestoreDocumentChangeEvent[]) { await this.initialize(); + const rows = events.map((event) => { return { insertId: event.eventId, @@ -60,20 +63,36 @@ export class FirestoreBigQueryEventHistoryTracker event_id: event.eventId, document_name: event.documentName, operation: ChangeType[event.operation], - data: JSON.stringify(event.data), + data: JSON.stringify(this.serializeData(event.data)), }, }; }); await this.insertData(rows); } + serializeData(eventData: any) { + if (typeof eventData === "undefined") { + return undefined; + } + + const data = traverse(eventData).map((property) => { + if (property instanceof firebase.firestore.DocumentReference) { + return property.path; + } + + return property; + }); + + return data; + } + /** * Inserts rows of data into the BigQuery raw change log table. */ private async insertData(rows: bigquery.RowMetadata[]) { const options = { skipInvalidRows: false, - ignoreUnkownValues: false, + ignoreUnknownValues: false, raw: true, }; try { diff --git a/firestore-bigquery-export/package.json b/firestore-bigquery-export/package.json index 9adc172c3..70f3caf22 100644 --- a/firestore-bigquery-export/package.json +++ b/firestore-bigquery-export/package.json @@ -13,7 +13,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.2", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.3", "@google-cloud/bigquery": "^2.1.0", "@types/chai": "^4.1.6", "chai": "^4.2.0",