add alert for oom for services (#109)

applied here,
[link](https://console.cloud.google.com/monitoring/dashboards/builder/f7a35b30-7561-4fbf-bbe3-500d9c9dab52;duration=P1D?project=kleung-chainguard&supportedpurview=project&f.rlabel.service_name=enforce-registry)

---------

Signed-off-by: Kenny Leung <[email protected]>

Author: k4leung4

modules/cloudevent-broker/ingress.tf

@@ -63,9 +63,10 @@ module "http" {
 }
 
 module "resources" {
-  source = "../dashboard/sections/resources"
-  title  = "Resources"
-  filter = ["resource.type=\"cloud_run_revision\""]
+  source        = "../dashboard/sections/resources"
+  title         = "Resources"
+  filter        = ["resource.type=\"cloud_run_revision\"", "resource.labels.service_name=\"${var.name}\""]
+  cloudrun_name = var.name
 }
 
 module "width" { source = "../dashboard/sections/width" }

modules/dashboard/cloudevent-receiver/dashboard.tf

@@ -23,9 +23,10 @@ module "http" {
 }
 
 module "resources" {
-  source = "../sections/resources"
-  title  = "Resources"
-  filter = ["resource.type=\"cloud_run_revision\""]
+  source        = "../sections/resources"
+  title         = "Resources"
+  filter        = ["resource.type=\"cloud_run_revision\"", "resource.labels.service_name=\"${var.service_name}\""]
+  cloudrun_name = var.service_name
 }
 
 module "width" { source = "../sections/width" }

modules/dashboard/job/dashboard.tf

@@ -5,9 +5,10 @@ module "logs" {
 }
 
 module "resources" {
-  source = "../sections/resources"
-  title  = "Resources"
-  filter = ["resource.type=\"cloud_run_job\""]
+  source        = "../sections/resources"
+  title         = "Resources"
+  filter        = ["resource.type=\"cloud_run_job\"", "resource.labels.job_name=\"${var.job_name}\""]
+  cloudrun_name = var.job_name
 }
 
 module "width" { source = "../sections/width" }

modules/dashboard/sections/resources/main.tf

@@ -1,6 +1,11 @@
 variable "title" { type = string }
 variable "filter" { type = list(string) }
+variable "cloudrun_name" { type = string }
 variable "collapsed" { default = false }
+variable "notification_channels" {
+  type    = list(string)
+  default = []
+}
 
 module "width" { source = "../width" }
 
@@ -55,6 +60,38 @@ module "received_bytes" {
   primary_reduce = "REDUCE_NONE"
 }
 
+module "oom_alert" {
+  source     = "../../widgets/alert"
+  title      = google_monitoring_alert_policy.oom.display_name
+  alert_name = google_monitoring_alert_policy.oom.name
+}
+
+resource "google_monitoring_alert_policy" "oom" {
+  # In the absence of data, incident will auto-close in 7 days
+  alert_strategy {
+    auto_close = "604800s"
+
+    notification_rate_limit {
+      period = "86400s" // re-alert once a day if condition still valid.
+    }
+  }
+
+  display_name = "${var.cloudrun_name} OOM Alert"
+  combiner     = "OR"
+
+  conditions {
+    display_name = "${var.cloudrun_name} OOM Alert"
+
+    condition_matched_log {
+      filter = "${join(" AND ", var.filter)} AND logName:\"run.googleapis.com%2Fvarlog%2Fsystem\" AND severity=ERROR AND textPayload:\"Consider increasing the memory limit\""
+    }
+  }
+
+  enabled = "true"
+
+  notification_channels = var.notification_channels
+}
+
 locals {
   columns = 3
   unit    = module.width.size / local.columns
@@ -67,39 +104,46 @@ locals {
     yPos   = 0,
     xPos   = local.col[0],
     height = local.unit,
-    width  = local.unit,
-    widget = module.cpu_utilization.widget,
+    width  = module.width.size,
+    widget = module.oom_alert.widget,
     },
     {
-      yPos   = 0,
+      yPos   = local.unit,
+      xPos   = local.col[0],
+      height = local.unit,
+      width  = local.unit,
+      widget = module.cpu_utilization.widget,
+    },
+    {
+      yPos   = local.unit,
       xPos   = local.col[1],
       height = local.unit,
       width  = local.unit,
       widget = module.memory_utilization.widget,
     },
     {
-      yPos   = 0,
+      yPos   = local.unit,
       xPos   = local.col[2],
       height = local.unit,
       width  = local.unit,
       widget = module.instance_count.widget,
     },
     {
-      yPos   = local.unit,
+      yPos   = local.unit * 2,
       xPos   = local.col[0],
       height = local.unit,
       width  = local.unit,
       widget = module.startup_latency.widget,
     },
     {
-      yPos   = local.unit,
+      yPos   = local.unit * 2,
       xPos   = local.col[1],
       height = local.unit,
       width  = local.unit,
       widget = module.sent_bytes.widget,
     },
     {
-      yPos   = local.unit,
+      yPos   = local.unit * 2,
       xPos   = local.col[2],
       height = local.unit,
       width  = local.unit,

modules/dashboard/service/README.md

@@ -82,6 +82,7 @@ No requirements.
 |------|-------------|------|---------|:--------:|
 | <a name="input_alerts"></a> [alerts](#input\_alerts) | A mapping from alerting policy names to the alert ids to add to the dashboard. | `map(string)` | `{}` | no |
 | <a name="input_labels"></a> [labels](#input\_labels) | Additional labels to apply to the dashboard. | `map` | `{}` | no |
+| <a name="input_notification_channels"></a> [notification\_channels](#input\_notification\_channels) | List of notification channels to alert. | `list(string)` | `[]` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | ID of the GCP project | `string` | n/a | yes |
 | <a name="input_service_name"></a> [service\_name](#input\_service\_name) | Name of the service(s) to monitor | `string` | n/a | yes |
 

modules/dashboard/service/dashboard.tf

@@ -26,9 +26,11 @@ module "grpc" {
 }
 
 module "resources" {
-  source = "../sections/resources"
-  title  = "Resources"
-  filter = ["resource.type=\"cloud_run_revision\""]
+  source                = "../sections/resources"
+  title                 = "Resources"
+  filter                = ["resource.type=\"cloud_run_revision\"", "resource.labels.service_name=\"${var.service_name}\""]
+  cloudrun_name         = var.service_name
+  notification_channels = var.notification_channels
 }
 
 module "alerts" {

modules/dashboard/service/variables.tf

@@ -18,3 +18,9 @@ variable "project_id" {
   description = "ID of the GCP project"
   type        = string
 }
+
+variable "notification_channels" {
+  description = "List of notification channels to alert."
+  type        = list(string)
+  default     = []
+}