Merge pull request #1874 from firebase/next

release: firestore-bigquery-changetracker

Author: cabljac

.github/workflows/readmes-updated.yml

@@ -0,0 +1,75 @@
+name: Check READMEs are up to date
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - "next"
+      - "master"
+
+concurrency:
+  group:
+    ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: "npm"
+          cache-dependency-path: "**/functions/package-lock.json"
+
+      - name: Set up global dependencies directory
+        id: global-deps-setup
+        run: |
+          mkdir -p ~/.npm-global
+          npm config set prefix '~/.npm-global'
+          echo "::set-output name=dir::$(npm config get prefix)"
+
+      - name: Cache global dependencies
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.global-deps-setup.outputs.dir }}
+          key:
+            ${{ runner.os }}-npm-global-deps-${{
+            hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-global-deps-
+
+      - name: Install Firebase and Lerna
+        run: |
+          echo "${{ steps.global-deps-setup.outputs.dir }}/bin" >> $GITHUB_PATH
+          npm install -g firebase-tools lerna
+
+      - name: Install local dependencies
+        run: npm ci
+
+      - name: Run Lerna generate-readme
+        run: lerna run --parallel generate-readme
+
+      - name: Check READMEs are up to date and push changes if possible.
+        run: |
+          changed_files=$(git status -s -- '**/README.md' | cut -c4-)
+          if [[ ! -z "$changed_files" ]]; then
+            echo "Changes detected in README.md files:"
+            echo "$changed_files"
+            echo "Please run 'lerna run --parallel generate-readme' locally and commit the changes."
+            exit 1
+          fi

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,7 @@
+## Version 0.1.43
+
+fix - correctly partition when only "timestamp" is selected for partition options
+
 ## Version 0.1.42
 
 fix - correctly extract timestamps from firestore fields to partition columns

firestore-bigquery-export/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.1.42
+version: 0.1.43
 specVersion: v1beta
 
 displayName: Stream Firestore to BigQuery

firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.30",
+  "version": "1.1.31",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {

firestore-bigquery-export/firestore-bigquery-change-tracker/package.json

@@ -479,3 +479,53 @@ describe("processing partitions on a new table", () => {
     });
   });
 });
+
+describe("updateTableMetadata", () => {
+  test("updates the table metadata with the timestamp field", async () => {
+    const config: FirestoreBigQueryEventHistoryTrackerConfig = {
+      datasetId: "",
+      tableId: "",
+      datasetLocation: "",
+      timePartitioning: "MONTH",
+      timePartitioningField: "timestamp",
+      timePartitioningFieldType: undefined,
+      timePartitioningFirestoreField: undefined,
+      transformFunction: "",
+      clustering: [],
+      bqProjectId: null,
+    };
+    const options = {};
+
+    const partitioning = new Partitioning(config, table);
+
+    await partitioning.updateTableMetadata(options);
+
+    expect(options).toEqual({
+      timePartitioning: {
+        field: "timestamp",
+        type: "MONTH",
+      },
+    });
+  });
+  test("Should not update if there is a custom option with the timestamp option", async () => {
+    const config: FirestoreBigQueryEventHistoryTrackerConfig = {
+      datasetId: "",
+      tableId: "",
+      datasetLocation: "",
+      timePartitioning: "MONTH",
+      timePartitioningField: "timestamp",
+      timePartitioningFieldType: "DATETIME",
+      timePartitioningFirestoreField: undefined,
+      transformFunction: "",
+      clustering: [],
+      bqProjectId: null,
+    };
+    const options = {};
+
+    const partitioning = new Partitioning(config, table);
+
+    await partitioning.updateTableMetadata(options);
+
+    expect(options).toEqual({});
+  });
+});

firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts

@@ -89,15 +89,19 @@ export class Partitioning {
       !timePartitioningField &&
       !timePartitioningFieldType &&
       !timePartitioningFirestoreField;
-
     /* No custom config has been set, use partition value option only */
     if (hasNoCustomOptions) return true;
 
-    /* check if all options have been provided to be  */
+    /* check if all valid combinations have been provided*/
+    const hasOnlyTimestamp =
+      timePartitioningField === "timestamp" &&
+      !timePartitioningFieldType &&
+      !timePartitioningFirestoreField;
     return (
-      !!timePartitioningField &&
-      !!timePartitioningFieldType &&
-      !!timePartitioningFirestoreField
+      hasOnlyTimestamp ||
+      (!!timePartitioningField &&
+        !!timePartitioningFieldType &&
+        !!timePartitioningFirestoreField)
     );
   }
 
@@ -248,33 +252,31 @@ export class Partitioning {
 
   async addPartitioningToSchema(fields = []): Promise<void> {
     /** check if class has valid table reference */
-    if (!this.hasValidTableReference()) return Promise.resolve();
+    if (!this.hasValidTableReference()) return;
 
     /** return if table is already partitioned **/
-    if (await this.isTablePartitioned()) return Promise.resolve();
+    if (await this.isTablePartitioned()) return;
 
     /** return if an invalid partition type has been requested**/
-    if (!this.hasValidTimePartitionType()) return Promise.resolve();
-
+    if (!this.hasValidTimePartitionType()) return;
     /** Return if invalid partitioning and field type combination */
-    if (this.hasHourAndDatePartitionConfig()) return Promise.resolve();
+    if (this.hasHourAndDatePartitionConfig()) return;
 
     /** return if an invalid partition type has been requested**/
-    if (!this.hasValidCustomPartitionConfig()) return Promise.resolve();
+    if (!this.hasValidCustomPartitionConfig()) return;
 
     /** return if an invalid partition type has been requested**/
-    if (!this.hasValidCustomPartitionConfig()) return Promise.resolve();
+    if (!this.hasValidCustomPartitionConfig()) return;
 
     /** update fields with new schema option ** */
-    if (!this.hasValidTimePartitionOption()) return Promise.resolve();
-
+    if (!this.hasValidTimePartitionOption()) return;
     /* Check if partition field has been provided */
-    if (!this.config.timePartitioningField) return Promise.resolve();
+    if (!this.config.timePartitioningField) return;
 
     // if (await !this.hasExistingSchema) return Promise.resolve();
 
     // Field already exists on schema, skip
-    if (this.customFieldExists(fields)) return Promise.resolve();
+    if (this.customFieldExists(fields)) return;
 
     fields.push(getNewPartitionField(this.config));
 
@@ -284,24 +286,24 @@ export class Partitioning {
       this.config.timePartitioningField
     );
 
-    return Promise.resolve();
+    return;
   }
 
   async updateTableMetadata(options: bigquery.TableMetadata): Promise<void> {
     /** return if table is already partitioned **/
-    if (await this.isTablePartitioned()) return Promise.resolve();
+    if (await this.isTablePartitioned()) return;
 
     /** return if an invalid partition type has been requested**/
-    if (!this.hasValidTimePartitionType()) return Promise.resolve();
+    if (!this.hasValidTimePartitionType()) return;
 
     /** update fields with new schema option ** */
-    if (!this.hasValidTimePartitionOption()) return Promise.resolve();
+    if (!this.hasValidTimePartitionOption()) return;
 
     /** Return if invalid partitioning and field type combination */
-    if (this.hasHourAndDatePartitionConfig()) return Promise.resolve();
+    if (this.hasHourAndDatePartitionConfig()) return;
 
     /** return if an invalid partition type has been requested**/
-    if (!this.hasValidCustomPartitionConfig()) return Promise.resolve();
+    if (!this.hasValidCustomPartitionConfig()) return;
 
     // if (await !this.hasExistingSchema) return Promise.resolve();