Merge pull request #248 from firebase/next

04/07/2020 Release

Author: joehan

auth-mailchimp-sync/POSTINSTALL.md

@@ -1,8 +1,8 @@
 ### See it in action
 
-You can test out this extension right away:
+You can test out this extension right away!
 
-1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users).
+1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users) in the Firebase console.
 
 1.  Click **Add User** to add a test user.
 

delete-user-data/POSTINSTALL.md

@@ -1,12 +1,10 @@
 ### See it in action
 
-You can test out this extension right away:
+You can test out this extension right away!
 
-1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users).
+1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users) in the Firebase console.
 
-1.  Click **Add User** to add a test user.
-
-1.  Copy the test user's UID to your clipboard.
+1.  Click **Add User** to add a test user, then copy the test user's UID to your clipboard.
 
 1.  Create a new Cloud Firestore document, a new Realtime Database entry, or upload a new file to Storage - incorporating the user's UID into the path according to the schema that you configured.
 

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,12 @@
+## Version 0.1.3
+
+feature - Interpret data more easily with column descriptions in the exported BigQuery data (#138).
+
+- The raw changelog now includes column descriptions.
+- The schema-views script allows you to specify custom column descriptions.
+
+fixed - Updated `@firebaseextensions/firestore-bigquery-change-tracker` dependency (fixes issues #235 and #225).
+
 ## Version 0.1.2
 
 fixed - Added "IF NOT EXISTS" to safely run `fs-bq-schema-views` script multiple times (PR #193).

firestore-bigquery-export/DEVSETUP.md

@@ -0,0 +1,109 @@
+# Development Setup
+
+The firestore-bigquery-export extension is split out into 4 node.js packages, three of
+which are hosted on [npm](https://www.npmjs.com/search?q=firebaseextensions).
+
+**[firestore-bigquery-change-tracker](https://www.npmjs.com/package/@firebaseextensions/firestore-bigquery-change-tracker)**:
+Contains the core interface defintions for document changes. Also exposes an
+API for uploading changes to BigQuery. The business-logic associated with
+creating the raw changelog and the latest snapshot of live documents in the
+changelog also lives in this package.
+
+**[fs-bq-import-collection](https://www.npmjs.com/package/@firebaseextensions/fs-bq-import-collection)**:
+Contains a script for resumably importing a firestore collection into BigQuery
+using the interface definitions in `firestore-bigquery-change-tracker`.
+
+**[fs-bq-schema-views](https://www.npmjs.com/package/@firebaseextensions/fs-bq-schema-views)**:
+Contains a script for generating BigQuery views that provide typed-checked
+access to the changelog created in `firestore-bigquery-change-tracker`.
+
+**firestore-bigquery-export-functions (not hosted)**: Contains the definition
+for a Google Cloud function that is called on each write to some collection.
+The function constructs the relevant change event and calls the API in
+`firestore-bigquery-change-tracker` to upload the change to BigQuery.
+
+Here are the dependency edges:
+
+1. [fs-bq-import-collection -> firestore-bigquery-change-tracker](https://github.com/firebase/extensions/blob/next/firestore-bigquery-export/scripts/import/package.json#L27)
+1. [firestore-bigquery-export-functions -> firestore-bigquery-change-tracker](https://github.com/firebase/extensions/blob/next/firestore-bigquery-export/package.json#L17)
+
+Note that `fs-bq-schema-views` is a standalone package, and all of these
+packages are included at the root of this extension's folder in the firebase
+extensions repo.
+
+## Building Locally
+
+### Clean up any local changes (optional)
+
+Make sure you've cleaned up and partial builds:
+
+```
+export PKGS="firestore-bigquery-change-tracker scripts/gen-schema-view scripts/import functions ."
+
+for pkg in $PKGS;
+do
+  pushd . && cd $pkg && npm run clean && rm -rf node_modules
+  popd
+done;
+```
+
+### Local package.json file pointers
+
+Npm supports using [local paths as
+dependencies](https://docs.npmjs.com/files/package.json#local-paths) in package.json.
+You'll need to update the following package.json files with local pointers to
+the firestore-bigquery-change-tracker package:
+
+1. firestore-bigquery-export/package.json
+1. firestore-bigquery-export/scripts/import/package.json
+
+This can be done with jq from the root of this extension's folder:
+
+```
+jq '.dependencies."@firebaseextensions/firestore-bigquery-change-tracker" = "file:./firestore-bigquery-change-tracker"' package.json > package.local.json
+jq '.dependencies."@firebaseextensions/firestore-bigquery-change-tracker" = "file:../../firestore-bigquery-change-tracker"' scripts/import/package.json > scripts/import/package.local.json
+
+mv package.json package.json.bak
+mv scripts/import/package.json scripts/import/package.remote.json.bak
+
+mv package.local.json package.json
+mv scripts/import/package.local.json scripts/import/package.json
+```
+
+Now, build the components according to the dependency order.
+
+```
+export PKGS="firestore-bigquery-change-tracker scripts/import . scripts/gen-schema-view"
+
+for pkg in $PKGS;
+do
+ pushd . && cd $pkg && npm install && npm run build
+ popd
+done;
+```
+
+Finally, you can install the extension you just built onto a Firebase-enabled
+GCP project with:
+
+```
+firebase ext:install ./firestore-bigquery-export --project=project-id
+```
+
+## Publishing
+
+_The following instructions are for Firebase team members only._
+
+We publish 3 separate npm packages for this extension. Each follows semver, so
+make sure to update the version numbers and corresponding dependencies.
+
+For each package, `cd` into the appropriate directory, then run:
+
+```
+npm pack
+npm publish
+```
+
+In general, you should publish `firestore-bigquery-change-tracker` first, since
+it doesn't depend on anything else. You should also only publish _after_ your
+changes have been merged into the `next` branch (and before the extension is
+released).

firestore-bigquery-export/POSTINSTALL.md

@@ -1,10 +1,10 @@
 ### See it in action
 
-You can test out this extension right away:
+You can test out this extension right away!
 
-1.  Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data).
+1.  Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data) in the Firebase console.
 
-1.  If it doesn't already exist, create the collection you specified during installation: `${param:COLLECTION_PATH}`.
+1.  If it doesn't already exist, create the collection you specified during installation: `${param:COLLECTION_PATH}`
 
 1.  Create a document in the collection called `bigquery-mirror-test` that contains any fields with any values that you'd like.
 

firestore-bigquery-export/docs/images/firestore-bigquery-export-dep-diagram.svg

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.1.2
+version: 0.1.3
 specVersion: v1beta
 
 displayName: Export Collections to BigQuery

firestore-bigquery-export/extension.yaml

@@ -5,19 +5,19 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.0.1",
+  "version": "1.1.2",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
     "build": "npm run clean && npm run compile",
-    "clean": "rimraf functions/lib",
+    "clean": "rimraf lib",
     "compile": "tsc",
-    "mocha": "nyc mocha -r ts-node/register --opts mocha.opts",
-    "test": "npm run mocha",
+    "test": "jest",
     "prepare": "npm run build"
   },
   "files": [
-    "lib"
+    "lib/*.js",
+    "lib/bigquery/*.js"
   ],
   "author": "Jan Wyszynski <[email protected]>",
   "license": "Apache-2.0",
@@ -31,13 +31,14 @@
     "sql-formatter": "^2.3.3"
   },
   "devDependencies": {
-    "chai": "^4.2.0",
+    "typescript": "^3.4.5",
     "rimraf": "^2.6.3",
-    "mocha": "^5.0.5",
     "nyc": "^14.0.0",
-    "@types/chai": "^4.1.6",
-    "@types/mocha": "^5.2.5",
+    "jest": "^24.9.0",
+    "chai": "^4.2.0",
     "ts-node": "^7.0.1",
-    "typescript": "^3.4.5"
+    "ts-jest": "^24.1.0",
+    "@types/jest": "^24.0.18",
+    "@types/chai": "^4.1.6"
   }
 }

firestore-bigquery-export/firestore-bigquery-change-tracker/mocha.opts

@@ -15,7 +15,7 @@
  */
 
 import * as bigquery from "@google-cloud/bigquery";
-import { firestoreToBQTable } from "./schema";
+import { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
 import { latestConsistentSnapshotView } from "./snapshot";
 
 import {
@@ -25,6 +25,8 @@ import {
 } from "../tracker";
 import * as logs from "../logs";
 
+export { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
+
 export interface FirestoreBigQueryEventHistoryTrackerConfig {
   datasetId: string;
   tableId: string;
@@ -134,7 +136,7 @@ export class FirestoreBigQueryEventHistoryTracker
       const options = {
         // `friendlyName` needs to be here to satisfy TypeScript
         friendlyName: changelogName,
-        schema: firestoreToBQTable(),
+        schema: RawChangelogSchema,
       };
       await table.create(options);
       logs.bigQueryTableCreated(changelogName);
@@ -164,6 +166,7 @@ export class FirestoreBigQueryEventHistoryTracker
         view: latestSnapshot,
       };
       await view.create(options);
+      await view.setMetadata({ schema: RawChangelogViewSchema });
       logs.bigQueryViewCreated(this.rawLatestView());
     }
     return view;