runners: Add terraform module for scale-cycle

terraform-aws-github-runner/main.tf

@@ -115,6 +115,10 @@ module "runners" {
 
   retry_scale_up_chron_hud_query_url = var.retry_scale_up_chron_hud_query_url
 
+  enable_scale_cycle               = var.enable_scale_cycle
+  scale_cycle_schedule_expression  = var.scale_cycle_schedule_expression
+  lambda_timeout_scale_cycle       = var.lambda_timeout_scale_cycle
+
   must_have_issues_labels = var.must_have_issues_labels
   cant_have_issues_labels = var.cant_have_issues_labels
 

terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts

@@ -7,11 +7,13 @@ import {
   ScaleUpChronMetrics,
   sendMetricsAtTimeout,
   sendMetricsTimeoutVars,
+  ScaleCycleMetrics,
 } from './scale-runners/metrics';
 import { getDelayWithJitterRetryCount, stochaticRunOvershoot } from './scale-runners/utils';
 import { scaleDown as scaleDownR } from './scale-runners/scale-down';
 import { scaleUpChron as scaleUpChronR } from './scale-runners/scale-up-chron';
 import { sqsSendMessages, sqsDeleteMessageBatch } from './scale-runners/sqs';
+import { scaleCycle as scaleCycleR } from './scale-runners/scale-cycle';
 
 async function sendRetryEvents(evtFailed: Array<[SQSRecord, boolean, number]>, metrics: ScaleUpMetrics) {
   console.error(`Detected ${evtFailed.length} errors when processing messages, will retry relevant messages.`);
@@ -202,3 +204,38 @@ export async function scaleUpChron(event: ScheduledEvent, context: Context, call
   }
   callback(callbackOutput);
 }
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function scaleCycle(event: ScheduledEvent, context: Context, callback: any) {
+  // we mantain open connections to redis, so the event pool is only cleaned when the SIGTERM is sent
+  context.callbackWaitsForEmptyEventLoop = false;
+
+  const metrics = new ScaleCycleMetrics();
+  const sndMetricsTimout: sendMetricsTimeoutVars = {
+    metrics: metrics,
+  };
+  sndMetricsTimout.setTimeout = setTimeout(
+    sendMetricsAtTimeout(sndMetricsTimout),
+    (Config.Instance.lambdaTimeout - 10) * 1000,
+  );
+
+  let callbackOutput: string | null = null;
+
+  try {
+    await scaleCycleR(metrics);
+  } catch (e) {
+    console.error(e);
+    callbackOutput = `Failed to scale cycle: ${e}`;
+  } finally {
+    try {
+      clearTimeout(sndMetricsTimout.setTimeout);
+      sndMetricsTimout.metrics = undefined;
+      sndMetricsTimout.setTimeout = undefined;
+      await metrics.sendMetrics();
+    } catch (e) {
+      callbackOutput = `Error sending metrics: ${e}`;
+    }
+  }
+
+  callback(callbackOutput);
+}

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/metrics.ts

@@ -1813,3 +1813,30 @@ export function sendMetricsAtTimeout(metricsTimeouts: sendMetricsTimeoutVars) {
     }
   };
 }
+
+export class ScaleCycleMetrics extends ScaleUpMetrics {
+  constructor() {
+    super('scaleCycle');
+  }
+
+  scaleCycleRunnerReuseFound(runnerType: string) {
+    const dimensions = new Map([['RunnerType', runnerType]]);
+    this.countEntry('run.scaleCycle.runnerReuse.found', 1, dimensions);
+  }
+
+  scaleCycleRunnerReuseFoundOrg(org: string, runnerType: string) {
+    const dimensions = new Map([
+      ['Org', org],
+      ['RunnerType', runnerType],
+    ]);
+    this.countEntry('run.scaleCycle.runnerReuse.found.org', 1, dimensions);
+  }
+
+  scaleCycleRunnerReuseFoundRepo(repo: string, runnerType: string) {
+    const dimensions = new Map([
+      ['Repo', repo],
+      ['RunnerType', runnerType],
+    ]);
+    this.countEntry('run.scaleCycle.runnerReuse.found.repo', 1, dimensions);
+  }
+}

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-cycle.ts

@@ -0,0 +1,77 @@
+import { Config } from './config';
+import { listRunners, RunnerInputParameters, tryReuseRunner } from './runners';
+import { getRepo, getRepoKey } from './utils';
+import { ScaleCycleMetrics } from './metrics';
+import { getRunnerTypes } from './gh-runners';
+import { createRunnerConfigArgument } from './scale-up';
+
+export async function scaleCycle(metrics: ScaleCycleMetrics) {
+  // Get runner types configuration first
+  const scaleConfigRepo = getRepo(Config.Instance.scaleConfigOrg, Config.Instance.scaleConfigRepo);
+  const runnerTypes = await getRunnerTypes(scaleConfigRepo, metrics);
+
+  // Get all valid runner type names for filtering
+  const validRunnerTypeNames = Array.from(runnerTypes.keys());
+
+  // Make separate calls for each runner type to filter at EC2 level
+  const allRunners = await Promise.all(
+    validRunnerTypeNames.map((runnerTypeName) =>
+      listRunners(metrics, {
+        containsTags: ['GithubRunnerID', 'EphemeralRunnerFinished', 'RunnerType'],
+        runnerType: runnerTypeName,
+      }),
+    ),
+  );
+
+  // Flatten the results
+  const runners = allRunners.flat();
+
+  for (const runner of runners) {
+    // Skip if required fields are missing (org/repo still need to be checked)
+    if (!runner.runnerType || !runner.org || !runner.repo) {
+      console.warn(`Skipping runner ${runner.instanceId} due to missing required tags`);
+      continue;
+    }
+
+    // Get the RunnerType object from the string (we know it exists since we filtered by it)
+    const runnerType = runnerTypes.get(runner.runnerType);
+    if (!runnerType) {
+      console.warn(`Unknown runner type: ${runner.runnerType}, skipping`);
+      continue;
+    }
+
+    // Create repo object
+    const repo = getRepo(runner.org, runner.repo);
+
+    // For each runner send an EBS volume replacement task
+    const runnerInputParameters: RunnerInputParameters = {
+      runnerConfig: (awsRegion: string, experimentalRunner: boolean) => {
+        return createRunnerConfigArgument(
+          runnerType,
+          repo,
+          // NOTE: installationId can actually be undefined here but this may incur lower rate limits
+          // TODO: figure out if we need to pass an actual installationId here
+          undefined,
+          metrics,
+          awsRegion,
+          experimentalRunner,
+        );
+      },
+      environment: Config.Instance.environment,
+      runnerType: runnerType,
+    };
+
+    // Set orgName or repoName based on configuration
+    if (Config.Instance.enableOrganizationRunners) {
+      runnerInputParameters.orgName = runner.org;
+      metrics.scaleCycleRunnerReuseFoundOrg(runner.org, runner.runnerType);
+      console.info(`Reusing runner ${runner.instanceId} for ${runner.org}`);
+    } else {
+      runnerInputParameters.repoName = getRepoKey(repo);
+      metrics.scaleCycleRunnerReuseFoundRepo(getRepoKey(repo), runner.runnerType);
+      console.info(`Reusing runner ${runner.instanceId} for ${getRepoKey(repo)}`);
+    }
+
+    await tryReuseRunner(runnerInputParameters, metrics);
+  }
+}

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts

@@ -163,7 +163,7 @@ export async function scaleUp(
   }
 }
 
-async function createRunnerConfigArgument(
+export async function createRunnerConfigArgument(
   runnerType: RunnerType,
   repo: Repo,
   installationId: number | undefined,

terraform-aws-github-runner/modules/runners/policies/lambda-scale-cycle.json

@@ -0,0 +1,44 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Action": [
+          "ec2:DescribeInstances",
+          "ec2:DescribeTags",
+          "ec2:RunInstances",
+          "ec2:CreateNetworkInterface",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DeleteNetworkInterface",
+          "ec2:DescribeImages",
+          "ec2:CreateTags",
+          "ec2:DeleteTags",
+          "ec2:CreateReplaceRootVolumeTask",
+          "ec2:DescribeReplaceRootVolumeTasks"
+        ],
+        "Resource": ["*"]
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "ec2:CreateTags"
+        ],
+        "Resource": ["*"],
+        "Condition": {
+          "StringEquals": {
+            "ec2:CreateAction" : "RunInstances"
+          }
+        }
+      },
+      {
+        "Effect": "Allow",
+        "Action": "iam:PassRole",
+        "Resource": "${arn_runner_instance_role}"
+      },
+      {
+        "Effect": "Allow",
+        "Action": ["ssm:PutParameter", "ssm:GetParameter", "ssm:DeleteParameter"],
+        "Resource": "*"
+      }
+    ]
+  }