Skip to content

Commit ed6dae8

Browse files
jmwskijmwski
authored
feat(firestore-bigquery-export): Support explicit BigQuery schemas for raw changelog and schema views (#138)
1 parent 76382f8 commit ed6dae8

20 files changed

Lines changed: 377 additions & 125 deletions

File tree

‎firestore-bigquery-export/firestore-bigquery-change-tracker/package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"url": "github.com/firebase/extensions.git",
66
"directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
77
},
8-
"version": "1.1.1",
8+
"version": "1.1.2",
99
"description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
1010
"main": "./lib/index.js",
1111
"scripts": {
@@ -16,7 +16,8 @@
1616
"prepare": "npm run build"
1717
},
1818
"files": [
19-
"lib"
19+
"lib/*.js",
20+
"lib/bigquery/*.js"
2021
],
2122
"author": "Jan Wyszynski <wyszynski@google.com>",
2223
"license": "Apache-2.0",

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
*/
1616

1717
import * as bigquery from "@google-cloud/bigquery";
18-
import { firestoreToBQTable } from "./schema";
18+
import { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
1919
import { latestConsistentSnapshotView } from "./snapshot";
2020

2121
import {
@@ -25,6 +25,8 @@ import {
2525
} from "../tracker";
2626
import * as logs from "../logs";
2727

28+
export { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
29+
2830
export interface FirestoreBigQueryEventHistoryTrackerConfig {
2931
datasetId: string;
3032
tableId: string;
@@ -134,7 +136,7 @@ export class FirestoreBigQueryEventHistoryTracker
134136
const options = {
135137
// `friendlyName` needs to be here to satisfy TypeScript
136138
friendlyName: changelogName,
137-
schema: firestoreToBQTable(),
139+
schema: RawChangelogSchema,
138140
};
139141
await table.create(options);
140142
logs.bigQueryTableCreated(changelogName);
@@ -164,6 +166,7 @@ export class FirestoreBigQueryEventHistoryTracker
164166
view: latestSnapshot,
165167
};
166168
await view.create(options);
169+
await view.setMetadata({ schema: RawChangelogViewSchema });
167170
logs.bigQueryViewCreated(this.rawLatestView());
168171
}
169172
return view;

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts‎

Lines changed: 80 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -64,23 +64,84 @@ export const timestampField = bigQueryField(
6464
export const latitudeField = bigQueryField("latitude", "NUMERIC");
6565
export const longitudeField = bigQueryField("longitude", "NUMERIC");
6666

67-
/**
68-
* Convert from a list of Firestore field definitions into the schema
69-
* that will be used by the BigQuery `raw` data table.
70-
*
71-
* The `raw` data table schema is:
72-
* - event_id: The event ID of the function trigger invocation responsible for
73-
* the row
74-
* - timestamp: A timestamp to be used for update ordering
75-
* - documentName: Stores the name of the Firestore document
76-
* - operation: The type of operation: CREATE, UPDATE, DELETE
77-
* - data: A record to contain the Firestore document data fields specified
78-
* in the schema
67+
/*
68+
* We cannot specify a schema for view creation, and all view columns default
69+
* to the NULLABLE mode.
7970
*/
80-
export const firestoreToBQTable = (): BigQueryField[] => [
81-
timestampField,
82-
eventIdField,
83-
documentNameField,
84-
operationField,
85-
dataField,
86-
];
71+
export const RawChangelogViewSchema: any = {
72+
fields: [
73+
{
74+
name: "timestamp",
75+
mode: "NULLABLE",
76+
type: "TIMESTAMP",
77+
description:
78+
"The commit timestamp of this change in Cloud Firestore. If the operation is IMPORT, this timestamp is epoch to ensure that any operation on an imported document supersedes the IMPORT.",
79+
},
80+
{
81+
name: "event_id",
82+
mode: "NULLABLE",
83+
type: "STRING",
84+
description:
85+
"The ID of the most-recent document change event that triggered the Cloud Function created by the extension. Empty for imports.",
86+
},
87+
{
88+
name: "document_name",
89+
mode: "NULLABLE",
90+
type: "STRING",
91+
description:
92+
"The full name of the changed document, for example, projects/collection/databases/(default)/documents/users/me).",
93+
},
94+
{
95+
name: "operation",
96+
mode: "NULLABLE",
97+
type: "STRING",
98+
description: "One of CREATE, UPDATE, IMPORT.",
99+
},
100+
{
101+
name: "data",
102+
mode: "NULLABLE",
103+
type: "STRING",
104+
description:
105+
"The full JSON representation of the current document state.",
106+
},
107+
],
108+
};
109+
110+
export const RawChangelogSchema: any = {
111+
fields: [
112+
{
113+
name: "timestamp",
114+
mode: "REQUIRED",
115+
type: "TIMESTAMP",
116+
description:
117+
"The commit timestamp of this change in Cloud Firestore. If the operation is IMPORT, this timestamp is epoch to ensure that any operation on an imported document supersedes the IMPORT.",
118+
},
119+
{
120+
name: "event_id",
121+
mode: "REQUIRED",
122+
type: "STRING",
123+
description:
124+
"The ID of the document change event that triggered the Cloud Function created by the extension. Empty for imports.",
125+
},
126+
{
127+
name: "document_name",
128+
mode: "REQUIRED",
129+
type: "STRING",
130+
description:
131+
"The full name of the changed document, for example, projects/collection/databases/(default)/documents/users/me).",
132+
},
133+
{
134+
name: "operation",
135+
mode: "REQUIRED",
136+
type: "STRING",
137+
description: "One of CREATE, UPDATE, IMPORT, or DELETE.",
138+
},
139+
{
140+
name: "data",
141+
mode: "NULLABLE",
142+
type: "STRING",
143+
description:
144+
"The full JSON representation of the document state after the indicated operation is applied. This field will be null for DELETE operations.",
145+
},
146+
],
147+
};

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/snapshot.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,11 @@
1717
import * as sqlFormatter from "sql-formatter";
1818

1919
import * as logs from "../logs";
20-
import { firestoreToBQTable, timestampField } from "./schema";
20+
import {
21+
RawChangelogSchema,
22+
RawChangelogViewSchema,
23+
timestampField,
24+
} from "./schema";
2125

2226
const excludeFields: string[] = ["document_name"];
2327

@@ -29,9 +33,9 @@ export const latestConsistentSnapshotView = (
2933
datasetId,
3034
tableName,
3135
timestampField.name,
32-
firestoreToBQTable()
36+
RawChangelogViewSchema["fields"]
3337
.map((field) => field.name)
34-
.filter((name) => excludeFields.indexOf(name) == -1)
38+
.filter((name) => excludeFields.indexOf(name) === -1)
3539
),
3640
useLegacySql: false,
3741
});

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414
* limitations under the License.
1515
*/
1616

17-
export { FirestoreBigQueryEventHistoryTracker } from "./bigquery";
17+
export {
18+
FirestoreBigQueryEventHistoryTracker,
19+
RawChangelogSchema,
20+
} from "./bigquery";
1821
export {
1922
ChangeType,
2023
FirestoreDocumentChangeEvent,

‎firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
"noImplicitReturns": true,
66
"outDir": "lib",
77
"sourceMap": false,
8-
"target": "es6"
8+
"target": "es6",
9+
"types": ["node", "jest", "chai"]
910
},
1011
"compileOnSave": true,
1112
"include": ["src"]

‎firestore-bigquery-export/functions/lib/index.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ exports.fsexportbigquery = functions.handler.firestore.document.onWrite((change,
4444
operation: changeType,
4545
documentName: context.resource.name,
4646
eventId: context.eventId,
47-
data: changeType == firestore_bigquery_change_tracker_1.ChangeType.DELETE ? undefined : change.after.data(),
47+
data: changeType === firestore_bigquery_change_tracker_1.ChangeType.DELETE ? undefined : change.after.data(),
4848
},
4949
]);
5050
logs.complete();

‎firestore-bigquery-export/functions/src/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ exports.fsexportbigquery = functions.handler.firestore.document.onWrite(
4545
documentName: context.resource.name,
4646
eventId: context.eventId,
4747
data:
48-
changeType == ChangeType.DELETE ? undefined : change.after.data(),
48+
changeType === ChangeType.DELETE ? undefined : change.after.data(),
4949
},
5050
]);
5151
logs.complete();

‎firestore-bigquery-export/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
"author": "Jan Wyszynski <wyszynski@google.com>",
1414
"license": "Apache-2.0",
1515
"dependencies": {
16-
"@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.1",
16+
"@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.2",
1717
"@google-cloud/bigquery": "^2.1.0",
1818
"@types/chai": "^4.1.6",
1919
"chai": "^4.2.0",

‎firestore-bigquery-export/scripts/gen-schema-view/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@firebaseextensions/fs-bq-schema-views",
3-
"version": "0.1.4",
3+
"version": "0.1.5",
44
"description": "Generate strongly-typed BigQuery Views based on raw JSON",
55
"main": "./lib/index.js",
66
"repository": {

0 commit comments

Comments
 (0)