feat: add all pod_ranges to cluster firewall rules and add missing shadow rules (#1480)

Co-authored-by: Bharath KKB <[email protected]>

Author: splichy

README.md

@@ -198,6 +198,7 @@ Then perform the following commands on the root folder:
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no |
 | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no |
+| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | <pre>object({<br>    metadata = string<br>  })</pre> | <pre>{<br>  "metadata": "INCLUDE_ALL_METADATA"<br>}</pre> | no |
 | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |

autogen/main/README.md

@@ -218,7 +218,7 @@ The node_pools variable takes the following parameters:
 | name | The name of the node pool |  | Required |
 {% if beta_cluster %}
 | placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional |
-| pod_range |  The ID of the secondary range for pod IPs. |  | Optional |
+| pod_range |  The name of the secondary range for pod IPs. |  | Optional |
 {% endif %}
 | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters |  | Required |
 | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional |

autogen/main/firewall.tf.tmpl

@@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" {
   direction   = "EGRESS"
 
   target_tags = [local.cluster_network_tag]
-  destination_ranges = [
+  destination_ranges = concat([
     local.cluster_endpoint_for_nodes,
     local.cluster_subnet_cidr,
-    local.cluster_alias_ranges_cidr[var.ip_range_pods],
-  ]
+    ],
+    local.pod_all_ip_ranges
+  )
 
   # Allow all possible protocols
   allow { protocol = "tcp" }
@@ -143,7 +144,7 @@ resource "google_compute_firewall" "shadow_allow_pods" {
   priority    = var.shadow_firewall_rules_priority
   direction   = "INGRESS"
 
-  source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]]
+  source_ranges = local.pod_all_ip_ranges
   target_tags   = [local.cluster_network_tag]
 
   # Allow all possible protocols
@@ -154,8 +155,11 @@ resource "google_compute_firewall" "shadow_allow_pods" {
   allow { protocol = "esp" }
   allow { protocol = "ah" }
 
-  log_config {
-    metadata = "INCLUDE_ALL_METADATA"
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
   }
 }
 
@@ -177,8 +181,11 @@ resource "google_compute_firewall" "shadow_allow_master" {
     ports    = ["10250", "443"]
   }
 
-  log_config {
-    metadata = "INCLUDE_ALL_METADATA"
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
   }
 }
 
@@ -209,7 +216,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" {
     ports    = ["1-65535"]
   }
 
-  log_config {
-    metadata = "INCLUDE_ALL_METADATA"
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_inkubelet" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet"
+  description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999
+  direction   = "INGRESS"
+
+  source_ranges = local.pod_all_ip_ranges
+  source_tags   = [local.cluster_network_tag]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "tcp"
+    ports    = ["10255"]
+  }
+
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
+  }
+}
+
+resource "google_compute_firewall" "shadow_deny_exkubelet" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet"
+  description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000
+  direction   = "INGRESS"
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = [local.cluster_network_tag]
+
+  deny {
+    protocol = "tcp"
+    ports    = ["10255"]
+  }
+
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
   }
 }

autogen/main/main.tf.tmpl

@@ -86,6 +86,11 @@ locals {
 
   cluster_subnet_cidr       = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null
   cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
+{% if autopilot_cluster != true %}
+  pod_all_ip_ranges         = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools): local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0] )) : []
+{% else %}
+  pod_all_ip_ranges         = var.add_cluster_firewall_rules ? [local.cluster_alias_ranges_cidr[var.ip_range_pods]] : []
+{% endif %}
 
 {% if autopilot_cluster != true %}
   cluster_network_policy = var.network_policy ? [{

autogen/main/variables.tf.tmpl

@@ -485,9 +485,23 @@ variable "add_shadow_firewall_rules" {
 }
 
 variable "shadow_firewall_rules_priority" {
-  type = number
+  type        = number
   description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000."
-  default = 999
+  default     = 999
+  validation {
+    condition     = var.shadow_firewall_rules_priority < 1000
+    error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)."
+  }
+}
+
+variable "shadow_firewall_rules_log_config" {
+  type = object({
+      metadata = string
+    })
+  description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging."
+  default = {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
 }
 
 {% if beta_cluster %}

docs/private_clusters.md

@@ -20,6 +20,16 @@ If you are using these features with a private cluster, you will need to either:
 
 If you are going to isolate your GKE private clusters from internet access you could check [this guide](https://medium.com/google-cloud/completely-private-gke-clusters-with-no-internet-connectivity-945fffae1ccd) and the associated [repo](https://github.com/andreyk-code/no-inet-gke-cluster).
 
+## Discontiguous multi-Pod CIDR
+If you are going to use [discontiguous multi-Pod CIDR](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr) it can happen that GKE robot will not update `gke-[cluster-name]-[cluster-hash]-all` and other firewall rules automatically when you add a new node pool (as stated in [documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr#modified_firewall_rule)). You can prevent this from happening, by using a workaround with shadow firewall rules:
+```
+module "gke" {
+  ...
+  add_shadow_firewall_rules  = true
+  shadow_firewall_rules_log_config = null # to save some $ on logs
+}
+```
+
 ## Troubleshooting
 
 ### Master Authorized Network

firewall.tf

@@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" {
   direction   = "EGRESS"
 
   target_tags = [local.cluster_network_tag]
-  destination_ranges = [
+  destination_ranges = concat([
     local.cluster_endpoint_for_nodes,
     local.cluster_subnet_cidr,
-    local.cluster_alias_ranges_cidr[var.ip_range_pods],
-  ]
+    ],
+    local.pod_all_ip_ranges
+  )
 
   # Allow all possible protocols
   allow { protocol = "tcp" }
@@ -99,7 +100,7 @@ resource "google_compute_firewall" "shadow_allow_pods" {
   priority    = var.shadow_firewall_rules_priority
   direction   = "INGRESS"
 
-  source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]]
+  source_ranges = local.pod_all_ip_ranges
   target_tags   = [local.cluster_network_tag]
 
   # Allow all possible protocols
@@ -110,8 +111,11 @@ resource "google_compute_firewall" "shadow_allow_pods" {
   allow { protocol = "esp" }
   allow { protocol = "ah" }
 
-  log_config {
-    metadata = "INCLUDE_ALL_METADATA"
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
   }
 }
 
@@ -133,8 +137,11 @@ resource "google_compute_firewall" "shadow_allow_master" {
     ports    = ["10250", "443"]
   }
 
-  log_config {
-    metadata = "INCLUDE_ALL_METADATA"
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
   }
 }
 
@@ -165,7 +172,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" {
     ports    = ["1-65535"]
   }
 
-  log_config {
-    metadata = "INCLUDE_ALL_METADATA"
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_inkubelet" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet"
+  description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999
+  direction   = "INGRESS"
+
+  source_ranges = local.pod_all_ip_ranges
+  source_tags   = [local.cluster_network_tag]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "tcp"
+    ports    = ["10255"]
+  }
+
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
+  }
+}
+
+resource "google_compute_firewall" "shadow_deny_exkubelet" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet"
+  description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000
+  direction   = "INGRESS"
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = [local.cluster_network_tag]
+
+  deny {
+    protocol = "tcp"
+    ports    = ["10255"]
+  }
+
+  dynamic "log_config" {
+    for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
+    content {
+      metadata = log_config.value.metadata
+    }
   }
 }

main.tf

@@ -73,6 +73,7 @@ locals {
 
   cluster_subnet_cidr       = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null
   cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
+  pod_all_ip_ranges         = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : []
 
   cluster_network_policy = var.network_policy ? [{
     enabled  = true

modules/beta-autopilot-private-cluster/README.md

@@ -125,6 +125,7 @@ Then perform the following commands on the root folder:
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no |
 | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no |
+| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | <pre>object({<br>    metadata = string<br>  })</pre> | <pre>{<br>  "metadata": "INCLUDE_ALL_METADATA"<br>}</pre> | no |
 | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |