Drop regional scope, randomize global bucket name. (#1103)

mattmoor · octo-sts[bot] · web-flow · commit 9d1088f81c0f · 2025-09-23T10:48:27.000-07:00
This eliminates the ability to specify a regional scope for workqueue,
regional-go-reconciler, and the associated dashboards.

This also shifts the (now) solve GCS bucket to use a randomized suffix
to reduce the possibility of collisions given the global namespace of
GCS buckets.

This change is technically lossy as it deletes the live GCS bucket
backing the workqueue and replaces it.

However, the nature of workqueues is that you can "resync" (e.g.
backstop) and recover from any lost data (the K8s equivalent is a
controller restart, this is why controllers resync at startup).

I applied this to multiple reconcilers in my dev environment without any
TF issues.

---------

Signed-off-by: Matt Moore &lt;mattmoor@chainguard.dev&gt;
Co-authored-by: octo-sts[bot] &lt;157150467+octo-sts[bot]@users.noreply.github.com&gt;
diff --git a/modules/dashboard/reconciler/README.md b/modules/dashboard/reconciler/README.md
@@ -18,7 +18,6 @@ module "my-reconciler-dashboard" {
   # Workqueue configuration
   max_retry       = 100
   concurrent_work = 20
-  scope           = "global"
 
   # Optional sections
   sections = {
@@ -55,7 +54,6 @@ The dashboard includes:
 | `workqueue_name` | Workqueue name | `${name}-wq` |
 | `max_retry` | Maximum retry attempts for tasks | `100` |
 | `concurrent_work` | Concurrent work items | `20` |
-| `scope` | Workqueue scope (regional/global) | `global` |
 | `sections` | Optional dashboard sections | See variables.tf |
 | `notification_channels` | Alert notification channels | `[]` |
 
@@ -82,7 +80,6 @@ module "my-reconciler-dashboard" {
   name            = "my-reconciler"  # Same base name as the reconciler
   max_retry       = module.my-reconciler.max-retry
   concurrent_work = module.my-reconciler.concurrent-work
-  scope           = "global"
 }
 <!-- BEGIN_TF_DOCS -->
 ## Requirements
@@ -123,7 +120,6 @@ No resources.
 | <a name="input_name"></a> [name](#input\_name) | The name of the reconciler (base name without suffixes) | `string` | n/a | yes |
 | <a name="input_notification_channels"></a> [notification\_channels](#input\_notification\_channels) | List of notification channels for alerts | `list(string)` | `[]` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The GCP project ID | `string` | n/a | yes |
-| <a name="input_scope"></a> [scope](#input\_scope) | The scope of the workqueue (regional or global) | `string` | `"global"` | no |
 | <a name="input_sections"></a> [sections](#input\_sections) | Configure visibility of optional dashboard sections | <pre>object({<br/>    github = optional(bool, false)<br/>  })</pre> | `{}` | no |
 | <a name="input_service_name"></a> [service\_name](#input\_service\_name) | The name of the reconciler service (defaults to name-rec) | `string` | `""` | no |
 | <a name="input_workqueue_name"></a> [workqueue\_name](#input\_workqueue\_name) | The name of the workqueue (defaults to name-wq) | `string` | `""` | no |
diff --git a/modules/dashboard/reconciler/dashboard.tf b/modules/dashboard/reconciler/dashboard.tf
@@ -16,7 +16,6 @@ module "workqueue-state" {
   service_name    = local.workqueue_name
   max_retry       = var.max_retry
   concurrent_work = var.concurrent_work
-  scope           = var.scope
   filter          = []
   collapsed       = false
 }
diff --git a/modules/dashboard/reconciler/variables.tf b/modules/dashboard/reconciler/variables.tf
@@ -38,12 +38,6 @@ variable "concurrent_work" {
   default     = 20
 }
 
-variable "scope" {
-  description = "The scope of the workqueue (regional or global)"
-  type        = string
-  default     = "global"
-}
-
 // Section visibility
 variable "sections" {
   description = "Configure visibility of optional dashboard sections"
diff --git a/modules/dashboard/sections/workqueue/README.md b/modules/dashboard/sections/workqueue/README.md
@@ -81,7 +81,6 @@ No resources.
 | <a name="input_filter"></a> [filter](#input\_filter) | n/a | `list(string)` | `[]` | no |
 | <a name="input_max_retry"></a> [max\_retry](#input\_max\_retry) | n/a | `number` | `0` | no |
 | <a name="input_receiver_name"></a> [receiver\_name](#input\_receiver\_name) | n/a | `string` | `""` | no |
-| <a name="input_scope"></a> [scope](#input\_scope) | n/a | `string` | `"regional"` | no |
 | <a name="input_service_name"></a> [service\_name](#input\_service\_name) | n/a | `string` | n/a | yes |
 | <a name="input_title"></a> [title](#input\_title) | n/a | `string` | `"Workqueue State"` | no |
 
diff --git a/modules/dashboard/sections/workqueue/main.tf b/modules/dashboard/sections/workqueue/main.tf
@@ -28,10 +28,6 @@ variable "max_retry" {
 variable "concurrent_work" {
   type = number
 }
-variable "scope" {
-  type    = string
-  default = "regional"
-}
 
 locals {
   // Use provided names or derive from service_name
@@ -52,7 +48,7 @@ module "work-in-progress" {
     "metric.type=\"prometheus.googleapis.com/workqueue_in_progress_keys/gauge\"",
     "metric.label.\"service_name\"=\"${local.dsp_name}\"",
   ])
-  group_by_fields = var.scope == "regional" ? ["resource.label.\"location\""] : null
+  group_by_fields = null
   primary_align   = "ALIGN_MAX"
   primary_reduce  = "REDUCE_MAX"
   thresholds      = [var.concurrent_work]
@@ -66,7 +62,7 @@ module "work-queued" {
     "metric.type=\"prometheus.googleapis.com/workqueue_queued_keys/gauge\"",
     "metric.label.\"service_name\"=\"${local.dsp_name}\"",
   ])
-  group_by_fields = var.scope == "regional" ? ["resource.label.\"location\""] : null
+  group_by_fields = null
   plot_type       = "STACKED_AREA"
   primary_align   = "ALIGN_MAX"
   primary_reduce  = "REDUCE_MAX"
@@ -105,7 +101,7 @@ module "wait-latency" {
     "metric.type=\"prometheus.googleapis.com/workqueue_wait_latency_seconds/histogram\"",
     "metric.label.\"service_name\"=\"${local.dsp_name}\"",
   ])
-  group_by_fields = var.scope == "regional" ? ["resource.label.\"location\""] : null
+  group_by_fields = null
 }
 
 module "percent-deduped" {
@@ -128,10 +124,10 @@ module "percent-deduped" {
   alignment_period            = "60s"
   thresholds                  = []
   numerator_align             = "ALIGN_RATE"
-  numerator_group_by_fields   = var.scope == "regional" ? ["resource.label.\"location\""] : null
+  numerator_group_by_fields   = null
   numerator_reduce            = "REDUCE_SUM"
   denominator_align           = "ALIGN_RATE"
-  denominator_group_by_fields = var.scope == "regional" ? ["resource.label.\"location\""] : null
+  denominator_group_by_fields = null
   denominator_reduce          = "REDUCE_SUM"
 }
 
@@ -150,7 +146,7 @@ module "max-attempts" {
     "metric.type=\"prometheus.googleapis.com/workqueue_max_attempts/gauge\"",
     "metric.label.\"service_name\"=\"${local.dsp_name}\"",
   ])
-  group_by_fields = var.scope == "regional" ? ["resource.label.\"location\""] : null
+  group_by_fields = null
   primary_align   = "ALIGN_MAX"
   primary_reduce  = "REDUCE_MAX"
   thresholds      = var.max_retry > 0 ? [var.max_retry] : []
@@ -171,7 +167,7 @@ module "dead-letter-queue" {
     "metric.type=\"prometheus.googleapis.com/workqueue_dead_lettered_keys/gauge\"",
     "metric.label.\"service_name\"=\"${local.dsp_name}\"",
   ])
-  group_by_fields = var.scope == "regional" ? ["resource.label.\"location\""] : null
+  group_by_fields = null
   plot_type       = "STACKED_AREA"
   primary_align   = "ALIGN_MAX"
   primary_reduce  = "REDUCE_MAX"
diff --git a/modules/dashboard/workqueue/README.md b/modules/dashboard/workqueue/README.md
@@ -75,7 +75,6 @@ No resources.
 | <a name="input_labels"></a> [labels](#input\_labels) | Additional labels to apply to the dashboard | `map(string)` | `{}` | no |
 | <a name="input_max_retry"></a> [max\_retry](#input\_max\_retry) | The maximum number of retry attempts before a task is moved to the dead letter queue | `number` | `100` | no |
 | <a name="input_name"></a> [name](#input\_name) | Name of the workqueue | `string` | n/a | yes |
-| <a name="input_scope"></a> [scope](#input\_scope) | The scope of the workqueue: 'regional' or 'global' | `string` | `"regional"` | no |
 
 ## Outputs
 
diff --git a/modules/dashboard/workqueue/dashboard.tf b/modules/dashboard/workqueue/dashboard.tf
@@ -5,7 +5,6 @@ module "workqueue-state" {
   service_name    = var.name
   max_retry       = var.max_retry
   concurrent_work = var.concurrent_work
-  scope           = var.scope
   filter          = []
   collapsed       = false
 }
diff --git a/modules/dashboard/workqueue/variables.tf b/modules/dashboard/workqueue/variables.tf
@@ -14,12 +14,6 @@ variable "concurrent_work" {
   type        = number
 }
 
-variable "scope" {
-  description = "The scope of the workqueue: 'regional' or 'global'"
-  type        = string
-  default     = "regional"
-}
-
 variable "labels" {
   description = "Additional labels to apply to the dashboard"
   type        = map(string)
diff --git a/modules/regional-go-reconciler/main.tf b/modules/regional-go-reconciler/main.tf
@@ -31,7 +31,6 @@ module "workqueue" {
   notification_channels = var.notification_channels
   labels                = var.labels
 
-  scope                    = "global"
   multi_regional_location = var.multi_regional_location
 
   depends_on = [module.reconciler]
diff --git a/modules/workqueue/README.md b/modules/workqueue/README.md
@@ -171,22 +171,17 @@ No requirements.
 | [google-beta_google_project_service_identity.pubsub](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_project_service_identity) | resource |
 | [google_cloud_scheduler_job.cron](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_scheduler_job) | resource |
 | [google_pubsub_subscription.global-this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource |
-| [google_pubsub_subscription.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource |
 | [google_pubsub_topic.global-object-change-notifications](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource |
-| [google_pubsub_topic.object-change-notifications](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource |
-| [google_pubsub_topic_iam_binding.gcs-publishes-to-topic](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_binding) | resource |
 | [google_pubsub_topic_iam_binding.global-gcs-publishes-to-topic](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_binding) | resource |
 | [google_service_account.change-trigger](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
 | [google_service_account.cron-trigger](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
 | [google_service_account.dispatcher](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
 | [google_service_account.receiver](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
 | [google_service_account_iam_binding.allow-pubsub-to-mint-tokens](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_binding) | resource |
 | [google_storage_bucket.global-workqueue](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource |
-| [google_storage_bucket.workqueue](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource |
-| [google_storage_bucket_iam_binding.authorize-access](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource |
 | [google_storage_bucket_iam_binding.global-authorize-access](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource |
 | [google_storage_notification.global-object-change-notifications](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_notification) | resource |
-| [google_storage_notification.object-change-notifications](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_notification) | resource |
+| [random_string.bucket_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
 | [random_string.change-trigger](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
 | [random_string.cron-trigger](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
 | [random_string.dispatcher](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
@@ -208,7 +203,7 @@ No requirements.
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | n/a | `string` | n/a | yes |
 | <a name="input_reconciler-service"></a> [reconciler-service](#input\_reconciler-service) | The name of the reconciler service that the workqueue will dispatch work to. | <pre>object({<br/>    name = string<br/>  })</pre> | n/a | yes |
 | <a name="input_regions"></a> [regions](#input\_regions) | A map from region names to a network and subnetwork.  A service will be created in each region configured to egress the specified traffic via the specified subnetwork. | <pre>map(object({<br/>    network = string<br/>    subnet  = string<br/>  }))</pre> | n/a | yes |
-| <a name="input_scope"></a> [scope](#input\_scope) | The scope of the workqueue: 'regional' for region-specific workqueues or 'global' for a single multi-regional workqueue. | `string` | `"global"` | no |
+| <a name="input_scope"></a> [scope](#input\_scope) | The scope of the workqueue. Must be 'global' for a single multi-regional workqueue. | `string` | `"global"` | no |
 | <a name="input_squad"></a> [squad](#input\_squad) | squad label to apply to the service. | `string` | n/a | yes |
 
 ## Outputs
diff --git a/modules/workqueue/dispatcher.tf b/modules/workqueue/dispatcher.tf
@@ -70,10 +70,8 @@ module "dispatcher-service" {
       regional-env = [
         {
           name = "WORKQUEUE_BUCKET"
-          value = var.scope == "global" ? {
+          value = {
             for k, v in var.regions : k => google_storage_bucket.global-workqueue.name
-            } : {
-            for k, v in google_storage_bucket.workqueue : k => v.name
           }
         },
         {
@@ -194,31 +192,6 @@ module "change-trigger-calls-dispatcher" {
   service-account = google_service_account.change-trigger.email
 }
 
-// Configure subscriptions to deliver events to dispatcher service: regional and global
-resource "google_pubsub_subscription" "this" {
-  for_each = var.regions
-
-  name   = "${var.name}-${each.key}"
-  topic  = google_pubsub_topic.object-change-notifications[each.key].id
-  labels = local.merged_labels
-
-  ack_deadline_seconds = 600 // Maximum value
-
-  push_config {
-    push_endpoint = module.change-trigger-calls-dispatcher[each.key].uri
-
-    // Authenticate requests to this service using tokens minted
-    // from the given service account.
-    oidc_token {
-      service_account_email = google_service_account.change-trigger.email
-    }
-  }
-
-  expiration_policy {
-    ttl = "" // This does not expire.
-  }
-}
-
 resource "google_pubsub_subscription" "global-this" {
   for_each = var.regions
 
diff --git a/modules/workqueue/main.tf b/modules/workqueue/main.tf
@@ -17,22 +17,15 @@ locals {
   merged_labels = merge(local.default_labels, local.squad_label, local.product_label, var.labels)
 }
 
-// Storage buckets: regional and global
-resource "google_storage_bucket" "workqueue" {
-  for_each = var.regions
-
-  name          = "${var.name}-${each.key}"
-  project       = var.project_id
-  location      = each.key
-  force_destroy = true
-  labels        = local.merged_labels
-
-  uniform_bucket_level_access = true
-  public_access_prevention    = "enforced"
+resource "random_string" "bucket_suffix" {
+  length  = 6  // Same length as "global"
+  special = false
+  upper   = false
+  numeric = true
 }
 
 resource "google_storage_bucket" "global-workqueue" {
-  name          = "${var.name}-global"
+  name          = "${var.name}-${random_string.bucket_suffix.result}"
   project       = var.project_id
   location      = var.multi_regional_location
   force_destroy = true
@@ -42,18 +35,6 @@ resource "google_storage_bucket" "global-workqueue" {
   public_access_prevention    = "enforced"
 }
 
-// Storage IAM: regional and global
-resource "google_storage_bucket_iam_binding" "authorize-access" {
-  for_each = var.regions
-
-  bucket = google_storage_bucket.workqueue[each.key].name
-  role   = "roles/storage.admin"
-  members = [
-    "serviceAccount:${google_service_account.receiver.email}",
-    "serviceAccount:${google_service_account.dispatcher.email}",
-  ]
-}
-
 resource "google_storage_bucket_iam_binding" "global-authorize-access" {
   bucket = google_storage_bucket.global-workqueue.name
   role   = "roles/storage.admin"
@@ -63,17 +44,6 @@ resource "google_storage_bucket_iam_binding" "global-authorize-access" {
   ]
 }
 
-// Pub/Sub topics: regional and global
-resource "google_pubsub_topic" "object-change-notifications" {
-  for_each = var.regions
-  name     = "${var.name}-${each.key}"
-  labels   = local.merged_labels
-
-  message_storage_policy {
-    allowed_persistence_regions = [each.key]
-  }
-}
-
 resource "google_pubsub_topic" "global-object-change-notifications" {
   for_each = var.regions
 
@@ -87,15 +57,6 @@ resource "google_pubsub_topic" "global-object-change-notifications" {
 
 data "google_storage_project_service_account" "gcs_account" {}
 
-// Pub/Sub IAM: regional and global
-resource "google_pubsub_topic_iam_binding" "gcs-publishes-to-topic" {
-  for_each = var.regions
-
-  topic   = google_pubsub_topic.object-change-notifications[each.key].id
-  role    = "roles/pubsub.publisher"
-  members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"]
-}
-
 resource "google_pubsub_topic_iam_binding" "global-gcs-publishes-to-topic" {
   for_each = var.regions
 
@@ -104,17 +65,6 @@ resource "google_pubsub_topic_iam_binding" "global-gcs-publishes-to-topic" {
   members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"]
 }
 
-// Storage notifications: regional and global
-resource "google_storage_notification" "object-change-notifications" {
-  for_each = var.regions
-
-  depends_on = [google_pubsub_topic_iam_binding.gcs-publishes-to-topic]
-
-  bucket         = google_storage_bucket.workqueue[each.key].name
-  payload_format = "JSON_API_V1"
-  topic          = google_pubsub_topic.object-change-notifications[each.key].id
-}
-
 resource "google_storage_notification" "global-object-change-notifications" {
   for_each = var.regions
 
diff --git a/modules/workqueue/receiver.tf b/modules/workqueue/receiver.tf
@@ -54,10 +54,8 @@ module "receiver-service" {
       regional-env = [
         {
           name = "WORKQUEUE_BUCKET"
-          value = var.scope == "global" ? {
+          value = {
             for k, v in var.regions : k => google_storage_bucket.global-workqueue.name
-            } : {
-            for k, v in google_storage_bucket.workqueue : k => v.name
           }
         },
       ]
diff --git a/modules/workqueue/variables.tf b/modules/workqueue/variables.tf
@@ -61,13 +61,13 @@ variable "product" {
 }
 
 variable "scope" {
-  description = "The scope of the workqueue: 'regional' for region-specific workqueues or 'global' for a single multi-regional workqueue."
+  description = "The scope of the workqueue. Must be 'global' for a single multi-regional workqueue."
   type        = string
   default     = "global"
 
   validation {
-    condition     = contains(["regional", "global"], var.scope)
-    error_message = "scope must be either 'regional' or 'global'."
+    condition     = var.scope == "global"
+    error_message = "scope must be 'global'. Regional scope is no longer supported."
   }
 }
 

Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,6 @@ module "workqueue-state" {`
`16`	`16`	`service_name = local.workqueue_name`
`17`	`17`	`max_retry = var.max_retry`
`18`	`18`	`concurrent_work = var.concurrent_work`
`19`		`- scope = var.scope`
`20`	`19`	`filter = []`
`21`	`20`	`collapsed = false`
`22`	`21`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,6 @@ module "workqueue-state" {`
`5`	`5`	`service_name = var.name`
`6`	`6`	`max_retry = var.max_retry`
`7`	`7`	`concurrent_work = var.concurrent_work`
`8`		`- scope = var.scope`
`9`	`8`	`filter = []`
`10`	`9`	`collapsed = false`
`11`	`10`	`}`