WIP - Analyst sees that GTFS-RT feeds are updated via workflows

Author: ohrite

iac/cal-itp-data-infra-staging/gtfs-rt-archiver/us/.terraform.lock.hcl

@@ -0,0 +1,16 @@
+provider "google" {
+  project = "cal-itp-data-infra-staging"
+}
+
+terraform {
+  required_providers {
+    google = {
+      version = "~> 6.41.0"
+    }
+  }
+
+  backend "gcs" {
+    bucket = "calitp-staging-gcp-components-tfstate"
+    prefix = "cal-itp-data-infra-staging/gtfs-rt-archiver"
+  }
+}

iac/cal-itp-data-infra-staging/gtfs-rt-archiver/us/provider.tf

@@ -0,0 +1,12 @@
+locals {
+  workflow_path = "${path.module}/../../../../workflows"
+}
+
+data "terraform_remote_state" "iam" {
+  backend = "gcs"
+
+  config = {
+    bucket = "calitp-staging-gcp-components-tfstate"
+    prefix = "cal-itp-data-infra-staging/iam"
+  }
+}

iac/cal-itp-data-infra-staging/gtfs-rt-archiver/us/variables.tf

@@ -0,0 +1,34 @@
+resource "google_workflows_workflow" "gtfs-rt-archiver" {
+  name            = "gtfs-rt-archiver"
+  region          = "us-west2"
+  project         = "cal-itp-data-infra-staging"
+  description     = "GTFS-RT Archiver"
+  service_account = data.terraform_remote_state.iam.outputs.google_service_account_composer-service-account_email
+  source_contents = templatefile("${local.workflow_path}/gtfs-rt.yaml", {})
+}
+
+resource "google_cloud_scheduler_job" "gtfs-rt-archiver" {
+  project          = "cal-itp-data-infra-staging"
+  name             = "gtfs-rt-archiver"
+  description      = "Cloud Scheduler GTFS-RT Archiver"
+  schedule         = "*/3 * * * *"
+  time_zone        = "America/Los_Angeles"
+  attempt_deadline = "320s"
+  region           = "us-west2"
+
+  http_target {
+    http_method = "POST"
+    uri         = "https://workflowexecutions.googleapis.com/v1/${google_workflows_workflow.gtfs-rt-archiver.id}/executions"
+    body = base64encode(
+      jsonencode({
+        "argument" : "{}",
+        "callLogLevel" : "CALL_LOG_LEVEL_UNSPECIFIED"
+      })
+    )
+
+    oauth_token {
+      service_account_email = data.terraform_remote_state.iam.outputs.google_service_account_composer-service-account_email
+      scope                 = "https://www.googleapis.com/auth/cloud-platform"
+    }
+  }
+}

iac/cal-itp-data-infra-staging/gtfs-rt-archiver/us/workflow.tf

@@ -294,6 +294,15 @@ resource "google_project_iam_member" "composer-service-account" {
   project = "cal-itp-data-infra-staging"
 }
 
+resource "google_project_iam_member" "workflow-service-account" {
+  for_each = toset([
+    "roles/workflows.invoker"
+  ])
+  role    = each.key
+  member  = "serviceAccount:${google_service_account.workflow-service-account.email}"
+  project = "cal-itp-data-infra-staging"
+}
+
 resource "google_project_iam_member" "ms-entra-id-DOT_DDS_Data_Pipeline_and_Warehouse_Users" {
   for_each = toset([
     "roles/viewer",

iac/cal-itp-data-infra-staging/iam/us/project_iam_member.tf

@@ -58,6 +58,14 @@ resource "google_service_account" "composer-service-account" {
   project      = "cal-itp-data-infra-staging"
 }
 
+resource "google_service_account" "workflow-service-account" {
+  account_id   = "workflow-service-account"
+  description  = "Service account for Workflow"
+  disabled     = "false"
+  display_name = "workflow"
+  project      = "cal-itp-data-infra-staging"
+}
+
 resource "google_service_account" "sftp-pod-service-account" {
   account_id   = "sftp-pod-service-account"
   description  = "Service account for sftp server"

iac/cal-itp-data-infra-staging/iam/us/service_account.tf

@@ -0,0 +1,19 @@
+main:
+    steps:
+    - init:
+        assign:
+            - projectId: $${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
+    - runQuery:
+        call: googleapis.bigquery.v2.jobs.query
+        args:
+            projectId: $${projectId}
+            body:
+                useLegacySql: false
+                useQueryCache: true
+                timeoutMs: 30000
+                query: |
+                    SELECT pipeline_url
+                    FROM `cal-itp-data-infra-staging.staging.int_transit_database__gtfs_datasets_dim`
+                    WHERE _is_current = true
+                    LIMIT 1
+        result: queryResult