add GitHub API rate limits metrics (#169)

Signed-off-by: Jason Hall <[email protected]>

Author: imjasonh

modules/dashboard/sections/github/README.md

@@ -0,0 +1,38 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+No providers.
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_collapsible"></a> [collapsible](#module\_collapsible) | ../collapsible | n/a |
+| <a name="module_limit"></a> [limit](#module\_limit) | ../../widgets/xy | n/a |
+| <a name="module_remaining"></a> [remaining](#module\_remaining) | ../../widgets/xy | n/a |
+| <a name="module_reset"></a> [reset](#module\_reset) | ../../widgets/xy | n/a |
+| <a name="module_width"></a> [width](#module\_width) | ../width | n/a |
+
+## Resources
+
+No resources.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_cloudrun_name"></a> [cloudrun\_name](#input\_cloudrun\_name) | n/a | `string` | n/a | yes |
+| <a name="input_collapsed"></a> [collapsed](#input\_collapsed) | n/a | `bool` | `false` | no |
+| <a name="input_filter"></a> [filter](#input\_filter) | n/a | `list(string)` | n/a | yes |
+| <a name="input_title"></a> [title](#input\_title) | n/a | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_section"></a> [section](#output\_section) | n/a |
+<!-- END_TF_DOCS -->

modules/dashboard/sections/github/main.tf

@@ -0,0 +1,88 @@
+variable "title" { type = string }
+variable "filter" { type = list(string) }
+variable "cloudrun_name" { type = string }
+variable "collapsed" { default = false }
+
+module "width" { source = "../width" }
+
+module "remaining" {
+  source = "../../widgets/xy"
+  title  = "GitHub API Rate Limit Remaining"
+  filter = concat(var.filter, [
+    "metric.type=\"prometheus.googleapis.com/github_rate_limit_remaining/gauge\"",
+    "metric.label.service_name=\"${var.cloudrun_name}\"",
+  ])
+  group_by_fields = ["resource.label.\"resource\""]
+  primary_align   = "ALIGN_MEAN"
+  primary_reduce  = "REDUCE_SUM"
+  plot_type       = "STACKED_AREA"
+}
+
+module "limit" {
+  source = "../../widgets/xy"
+  title  = "GitHub API Rate Limit"
+  filter = concat(var.filter, [
+    "metric.type=\"prometheus.googleapis.com/github_rate_limit/gauge\"",
+    "metric.label.service_name=\"${var.cloudrun_name}\"",
+  ])
+  group_by_fields = ["resource.label.\"resource\""]
+  primary_align   = "ALIGN_DELTA"
+  primary_reduce  = "REDUCE_MEAN"
+}
+
+module "reset" {
+  source = "../../widgets/xy"
+  title  = "Next GitHub API reset"
+  filter = concat(var.filter, [
+    "metric.type=\"prometheus.googleapis.com/github_rate_limit_remaining_reset/gauge\"",
+    "metric.label.service_name=\"${var.cloudrun_name}\"",
+  ])
+  group_by_fields = ["resource.label.\"resource\""]
+  primary_align   = "ALIGN_DELTA"
+  primary_reduce  = "REDUCE_MEAN"
+}
+
+locals {
+  columns = 3
+  unit    = module.width.size / local.columns
+
+  // https://www.terraform.io/language/functions/range
+  // N columns, unit width each  ([0, unit, 2 * unit, ...])
+  col = range(0, local.columns * local.unit, local.unit)
+
+  tiles = [{
+
+    yPos   = local.unit,
+    xPos   = local.col[0],
+    height = local.unit,
+    width  = local.unit,
+    widget = module.remaining.widget,
+    },
+    {
+      yPos   = local.unit,
+      xPos   = local.col[1],
+      height = local.unit,
+      width  = local.unit,
+      widget = module.limit.widget,
+    },
+    {
+      yPos   = local.unit,
+      xPos   = local.col[2],
+      height = local.unit,
+      width  = local.unit,
+      widget = module.reset.widget,
+    },
+  ]
+}
+
+module "collapsible" {
+  source = "../collapsible"
+
+  title     = var.title
+  tiles     = local.tiles
+  collapsed = var.collapsed
+}
+
+output "section" {
+  value = module.collapsible.section
+}

modules/regional-go-service/README.md

@@ -94,9 +94,9 @@ No requirements.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_containers"></a> [containers](#input\_containers) | The containers to run in the service.  Each container will be run in each region. | <pre>map(object({<br>    source = object({<br>      base_image  = optional(string, "cgr.dev/chainguard/static:latest-glibc")<br>      working_dir = string<br>      importpath  = string<br>    })<br>    args = optional(list(string), [])<br>    ports = optional(list(object({<br>      name           = optional(string, "http1")<br>      container_port = number<br>    })), [])<br>    resources = optional(<br>      object(<br>        {<br>          limits = optional(object(<br>            {<br>              cpu    = string<br>              memory = string<br>            }<br>          ), null)<br>          cpu_idle = optional(bool, true)<br>        }<br>      ),<br>      {<br>        cpu_idle = true<br>      }<br>    )<br>    env = optional(list(object({<br>      name  = string<br>      value = optional(string)<br>      value_source = optional(object({<br>        secret_key_ref = object({<br>          secret  = string<br>          version = string<br>        })<br>      }), null)<br>    })), [])<br>    regional-env = optional(list(object({<br>      name  = string<br>      value = map(string)<br>    })), [])<br>    volume_mounts = optional(list(object({<br>      name       = string<br>      mount_path = string<br>    })), [])<br>  }))</pre> | n/a | yes |
-| <a name="input_egress"></a> [egress](#input\_egress) | The egress mode for the service.  Must be one of ALL\_TRAFFIC, or PRIVATE\_RANGES\_ONLY. Egress traffic is routed through the regional VPC network from var.regions. | `string` | `"ALL_TRAFFIC"` | no |
+| <a name="input_egress"></a> [egress](#input\_egress) | Which type of egress traffic to send through the VPC.<br><br>- ALL\_TRAFFIC sends all traffic through regional VPC network<br>- PRIVATE\_RANGES\_ONLY sends only traffic to private IP addresses through regional VPC network | `string` | `"ALL_TRAFFIC"` | no |
 | <a name="input_execution_environment"></a> [execution\_environment](#input\_execution\_environment) | The execution environment for the service | `string` | `"EXECUTION_ENVIRONMENT_GEN1"` | no |
-| <a name="input_ingress"></a> [ingress](#input\_ingress) | The ingress mode for the service.  Must be one of INGRESS\_TRAFFIC\_ALL, INGRESS\_TRAFFIC\_INTERNAL\_LOAD\_BALANCER, or INGRESS\_TRAFFIC\_INTERNAL\_ONLY. | `string` | `"INGRESS_TRAFFIC_INTERNAL_ONLY"` | no |
+| <a name="input_ingress"></a> [ingress](#input\_ingress) | Which type of ingress traffic to accept for the service.<br><br>- INGRESS\_TRAFFIC\_ALL accepts all traffic, enabling the public .run.app URL for the service<br>- INGRESS\_TRAFFIC\_INTERNAL\_LOAD\_BALANCER accepts traffic only from a load balancer<br>- INGRESS\_TRAFFIC\_INTERNAL\_ONLY accepts internal traffic only | `string` | `"INGRESS_TRAFFIC_INTERNAL_ONLY"` | no |
 | <a name="input_name"></a> [name](#input\_name) | n/a | `string` | n/a | yes |
 | <a name="input_notification_channels"></a> [notification\_channels](#input\_notification\_channels) | List of notification channels to alert. | `list(string)` | n/a | yes |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | n/a | `string` | n/a | yes |

modules/regional-go-service/variables.tf

@@ -16,13 +16,24 @@ variable "regions" {
 
 variable "ingress" {
   type        = string
-  description = "The ingress mode for the service.  Must be one of INGRESS_TRAFFIC_ALL, INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER, or INGRESS_TRAFFIC_INTERNAL_ONLY."
+  description = <<EOD
+Which type of ingress traffic to accept for the service.
+
+- INGRESS_TRAFFIC_ALL accepts all traffic, enabling the public .run.app URL for the service
+- INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER accepts traffic only from a load balancer
+- INGRESS_TRAFFIC_INTERNAL_ONLY accepts internal traffic only
+EOD
   default     = "INGRESS_TRAFFIC_INTERNAL_ONLY"
 }
 
 variable "egress" {
   type        = string
-  description = "The egress mode for the service.  Must be one of ALL_TRAFFIC, or PRIVATE_RANGES_ONLY. Egress traffic is routed through the regional VPC network from var.regions."
+  description = <<EOD
+Which type of egress traffic to send through the VPC.
+
+- ALL_TRAFFIC sends all traffic through regional VPC network
+- PRIVATE_RANGES_ONLY sends only traffic to private IP addresses through regional VPC network
+EOD
   default     = "ALL_TRAFFIC"
 }
 

pkg/httpmetrics/transport.go

@@ -5,6 +5,7 @@ import (
 	"log/slog"
 	"math"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -56,7 +57,8 @@ func WrapTransport(t http.RoundTripper) http.RoundTripper {
 	return instrumentRoundTripperCounter(
 		instrumentRoundTripperInFlight(
 			instrumentRoundTripperDuration(
-				otelhttp.NewTransport(t))))
+				instrumentGitHubRateLimits(
+					otelhttp.NewTransport(t)))))
 }
 
 func mapErrorToLabel(err error) string {
@@ -161,3 +163,60 @@ func bucketize(host string) string {
 	}
 	return "other"
 }
+
+var (
+	mGitHubRateLimitRemaining = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit_remaining",
+			Help: "The number of requests remaining in the current rate limit window",
+		},
+		[]string{"resource"},
+	)
+	mGitHubRateLimit = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit",
+			Help: "The number of requests allowed during the rate limit window",
+		},
+		[]string{"resource"},
+	)
+	mGitHubRateLimitReset = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit_reset",
+			Help: "The timestamp at which the current rate limit window resets",
+		},
+		[]string{"resource"},
+	)
+)
+
+// instrumentGitHubRateLimits is a promhttp.RoundTripperFunc that records GitHub rate limit metrics.
+// See https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28
+func instrumentGitHubRateLimits(next http.RoundTripper) promhttp.RoundTripperFunc {
+	return func(r *http.Request) (*http.Response, error) {
+		resp, err := next.RoundTrip(r)
+		if err != nil {
+			return resp, err
+		}
+		if r.URL.Host == "api.github.com" {
+			resource := resp.Header.Get("X-RateLimit-Resource")
+			if resource == "" {
+				resource = "unknown"
+			}
+
+			set := func(key string, v *prometheus.GaugeVec) {
+				val := resp.Header.Get(key)
+				if val == "" {
+					return
+				}
+				i, err := strconv.Atoi(val)
+				if err != nil {
+					return
+				}
+				v.With(prometheus.Labels{"resource": resource}).Set(float64(i))
+			}
+			set("X-RateLimit-Remaining", mGitHubRateLimitRemaining)
+			set("X-RateLimit-Limit", mGitHubRateLimit)
+			set("X-RateLimit-Reset", mGitHubRateLimitReset)
+		}
+		return resp, err
+	}
+}