Update GitLab webhooks

Author: danielcampagnolitg

package.json

@@ -90,8 +90,8 @@
     "@fastify/type-provider-typebox": "^4.0.0",
     "@firebase/rules-unit-testing": "^4.0.1",
     "@ghostery/adblocker-puppeteer": "2.11.3",
-    "@gitbeaker/core": "43.4.0",
-    "@gitbeaker/rest": "43.4.0",
+    "@gitbeaker/core": "43.5.0",
+    "@gitbeaker/rest": "43.5.0",
     "@google-cloud/aiplatform": "^5.7.0",
     "@google-cloud/bigquery": "^8.1.1",
     "@google-cloud/discoveryengine": "^2.4.0",

pnpm-lock.yaml

@@ -59,6 +59,7 @@ export const testUser: User = {
 	name: 'John Doe',
 	email: 'test@example.com',
 	enabled: true,
+	admin: false,
 	createdAt: new Date(Date.now() - 86400000), // Yesterday
 	lastLoginAt: new Date(),
 	hilBudget: 1.5,
@@ -73,6 +74,7 @@ export const otherUser: User = {
 	name: 'John Doe',
 	email: 'other@example.com',
 	enabled: true,
+	admin: false,
 	createdAt: new Date(Date.now() - 172800000), // Day before yesterday
 	lastLoginAt: new Date(Date.now() - 3600000), // Hour ago
 	hilBudget: 0.5,

src/agent/agentContextService/agentContextService.test.ts

@@ -9,6 +9,7 @@ import { appContext } from '#app/applicationContext';
 import { FUNC_SEP } from '#functionSchema/functions';
 import { Git } from '#functions/scm/git';
 import { GitHub } from '#functions/scm/github';
+import { GitLab } from '#functions/scm/gitlab';
 import { logger } from '#o11y/logger';
 import type { AgentContext } from '#shared/agent/agent.model';
 import type { FunctionCallResult } from '#shared/llm/llm.model';
@@ -53,15 +54,7 @@ async function _startAgent(agent: AgentContext): Promise<AgentExecution> {
 
 	await checkRepoHomeAndWorkingDirectory(agent);
 
-	const metadata = agent.metadata ?? {};
-	const githubProject = metadata.github?.repository;
-	if (githubProject) {
-		runAsUser(agent.user, async () => {
-			const repoPath = await new GitHub().cloneProject(githubProject);
-			agent.fileSystem!.setWorkingDirectory(repoPath);
-			if (metadata.github.branch) await new Git().switchToBranch(metadata.github.branch);
-		});
-	}
+	await initialiseMetadataRepository(agent);
 
 	switch (agent.subtype) {
 		case 'xml':
@@ -81,6 +74,48 @@ async function _startAgent(agent: AgentContext): Promise<AgentExecution> {
 	return execution;
 }
 
+async function initialiseMetadataRepository(agent: AgentContext) {
+	const metadata = agent.metadata ?? {};
+	let hasRepo = false;
+	let branch: string | undefined;
+
+	let gitProject = metadata.github?.repository;
+	if (gitProject) {
+		hasRepo = true;
+		await runAsUser(agent.user, async () => {
+			branch = metadata.github.branch;
+			const repoPath = await new GitHub().cloneProject(gitProject, branch);
+			agent.fileSystem!.setWorkingDirectory(repoPath);
+		});
+	}
+	gitProject = metadata.gitlab?.projectPath;
+	if (gitProject) {
+		hasRepo = true;
+		await runAsUser(agent.user, async () => {
+			branch = metadata.gitlab.branch;
+			const repoPath = await new GitLab().cloneProject(gitProject, branch);
+			agent.fileSystem!.setWorkingDirectory(repoPath);
+		});
+	}
+	// If an agent has switched a shared repo from the main/master branch, then switch it back
+	if (agent.useSharedRepos && hasRepo && !branch) {
+		const git = new Git();
+		const currentBranch = await git.getBranchName();
+		if (currentBranch !== 'main' && currentBranch !== 'master') {
+			logger.warn(`Shared repo ${gitProject} is not on branch main or master, switching back`);
+			try {
+				await git.switchToBranch('main');
+			} catch (e) {
+				try {
+					await git.switchToBranch('master');
+				} catch (e) {
+					logger.warn({ metadata }, 'Couldnt restore branch to `main` or `master`', e);
+				}
+			}
+		}
+	}
+}
+
 export async function startAgentAndWaitForCompletion(config: RunAgentConfig): Promise<string> {
 	const agentExecution = await startAgent(config);
 

src/agent/autonomous/autonomousAgentRunner.ts

@@ -6,7 +6,8 @@ import type { User } from '#shared/user/user.model';
 import { runAsUser } from '#user/userContext';
 
 export const SINGLE_USER: User = {
-	enabled: false,
+	enabled: true,
+	admin: false,
 	hilBudget: 0,
 	hilCount: 0,
 	llmConfig: {},
@@ -27,6 +28,7 @@ export const USER_A: User = {
 	name: 'User A',
 	email: 'usera@example.com',
 	enabled: true,
+	admin: false,
 	hilBudget: 0,
 	hilCount: 0,
 	llmConfig: {},
@@ -44,6 +46,7 @@ export const USER_B: User = {
 	name: 'User B',
 	email: 'userb@example.com',
 	enabled: true,
+	admin: false,
 	hilBudget: 0,
 	hilCount: 0,
 	llmConfig: {},

src/chat/chatService.test.ts

@@ -0,0 +1,27 @@
+export function buildJobUrl(job: JobResult): string {
+	if (job.host.includes('github.com')) return `https://${job.host}/${job.project}/actions/runs/${job.buildId}`;
+
+	return `https://${job.host}/${job.project}/-/jobs/${job.buildId}`;
+}
+
+export interface JobResult {
+	buildId: number;
+	project: string;
+	status: string;
+	jobName: string;
+	stage: string;
+	startedAt: string;
+	duration: number;
+	pipeline: number;
+	host: string;
+	/** build_failure_reason field */
+	failureReason?: string;
+	/** Our classification of the failure type */
+	failureType?: string;
+}
+
+export interface CICDStatsService {
+	saveJobResult(jobResult: JobResult): Promise<void>;
+
+	getRecentSuccessfulJobs(project: string, jobName: string): Promise<JobResult[]>;
+}

src/functions/scm/cicdStatsService.ts

@@ -286,7 +286,7 @@ export class GitLab extends AbstractSCM implements SourceControlManagement {
 	 * @param mergeRequestIId The merge request IID. Can be found in the URL to a pipeline
 	 */
 	@func()
-	async getLatestMergeRequestPipeline(gitlabProjectId: string | number, mergeRequestIId: number): Promise<PipelineWithJobs | null> {
+	async getMergeRequestLatestPipeline(gitlabProjectId: string | number, mergeRequestIId: number): Promise<PipelineWithJobs | null> {
 		// allPipelines<E extends boolean = false>(projectId: string | number, mergerequestIId: number, options?: Sudo & ShowExpanded<E>): Promise<GitlabAPIResponse<Pick<PipelineSchema, 'id' | 'sha' | 'ref' | 'status'>[], C, E, void>>;
 		const pipelines = await this.api().MergeRequests.allPipelines(gitlabProjectId, mergeRequestIId);
 		if (pipelines.length === 0) return null;
@@ -330,28 +330,23 @@ export class GitLab extends AbstractSCM implements SourceControlManagement {
 	/**
 	 * Gets the logs from the jobs which have failed in a pipeline
 	 * @param gitlabProjectId GitLab project full path or the numeric id
-	 * @param mergeRequestIId The merge request IID. Can get this from the URL of the merge request. https://<gitlab-host>/<group>/[<sub-group>/]<project>/-/merge_requests/<mergeRequestIId>
+	 * @param pipelineId The pipelineId. Can be determined from the URL of a pipeline. https://<gitlab-host>/<group>/[<sub-group>/]<project>/-/pipelines/<pipelineId>
 	 * @returns A Record with the job name as the key and the logs as the value.
 	 */
 	@func()
-	async getMergeRequestPipelineFailedJobLogs(gitlabProjectId: string | number, mergeRequestIId: number): Promise<string> {
-		logger.info({ gitlabProjectId, mergeRequestIId }, 'Getting pipelines');
-		const pipelines = await this.api().MergeRequests.allPipelines(gitlabProjectId, mergeRequestIId);
-		if (pipelines.length === 0) throw new Error('No pipelines for the merge request');
+	async getPipelineFailedJobLogs(gitlabProjectId: string | number, pipelineId: number): Promise<string> {
+		logger.info({ gitlabProjectId, pipelineId }, 'Getting pipeline');
+		const pipeline = await this.api().Pipelines.show(gitlabProjectId, pipelineId);
 
-		// pipelines.sort((a, b) => (Date.parse(a.created_at) < Date.parse(b.created_at) ? 1 : -1));
-		const latestPipeline = pipelines.at(0);
-		if (!latestPipeline) throw new Error('No pipelines for the merge request');
-
-		if (latestPipeline.status !== 'failed' && latestPipeline.status !== 'blocked') throw new Error('Pipeline is not failed or blocked');
+		if (pipeline.status !== 'failed' && pipeline.status !== 'blocked') throw new Error(`Pipeline status is not failed or blocked. Status: ${pipeline.status}`);
 
-		logger.info({ gitlabProjectId, pipelineId: latestPipeline.id }, 'Getting jobs');
-		const jobs: JobSchema[] = await this.api().Jobs.all(gitlabProjectId, { pipelineId: latestPipeline.id });
+		logger.info({ gitlabProjectId, pipelineId: pipeline.id }, 'Getting jobs');
+		const jobs: JobSchema[] = await this.api().Jobs.all(gitlabProjectId, { pipelineId: pipeline.id });
 		const failedJobs = jobs.filter((job) => job.status === 'failed' && job.allow_failure === false);
 
 		let jobLogs = '';
 		for (const job of failedJobs) {
-			logger.info({ gitlabProjectId, pipelineId: latestPipeline.id, jobId: job.id }, 'Getting job logs');
+			logger.info({ gitlabProjectId, pipelineId: pipeline.id, jobId: job.id }, 'Getting job logs');
 			let logs = await this.getJobLogs(gitlabProjectId, job.id.toString());
 
 			// If the logs are longer than ~12,000 tokens, truncate them.
@@ -365,6 +360,25 @@ export class GitLab extends AbstractSCM implements SourceControlManagement {
 		return jobLogs;
 	}
 
+	/**
+	 * Gets the logs from the jobs which have failed in the latest pipeline of a merge request
+	 * @param gitlabProjectId GitLab project full path or the numeric id
+	 * @param mergeRequestIId The merge request IID. Can get this from the URL of the merge request. https://<gitlab-host>/<group>/[<sub-group>/]<project>/-/merge_requests/<mergeRequestIId>
+	 * @returns A Record with the job name as the key and the logs as the value.
+	 */
+	@func()
+	async getMergeRequestPipelineFailedJobLogs(gitlabProjectId: string | number, mergeRequestIId: number): Promise<string> {
+		logger.info({ gitlabProjectId, mergeRequestIId }, 'Getting pipelines');
+		const pipelines = await this.api().MergeRequests.allPipelines(gitlabProjectId, mergeRequestIId);
+		if (pipelines.length === 0) throw new Error('No pipelines for the merge request');
+
+		// pipelines.sort((a, b) => (Date.parse(a.created_at) < Date.parse(b.created_at) ? 1 : -1));
+		const latestPipeline = pipelines.at(0);
+		if (!latestPipeline) throw new Error('No pipelines for the merge request');
+
+		return this.getPipelineFailedJobLogs(gitlabProjectId, latestPipeline.id);
+	}
+
 	/**
 	 * @returns the diffs for a merge request
 	 */

src/functions/scm/gitlab.ts

@@ -0,0 +1,39 @@
+import { randomUUID } from 'node:crypto';
+import type { Firestore } from '@google-cloud/firestore';
+import { CICDStatsService, JobResult } from '#functions/scm/cicdStatsService';
+import { logger } from '#o11y/logger';
+import { span } from '#o11y/trace';
+import { currentUser } from '#user/userContext';
+import { firestoreDb } from './firestore';
+
+export class FirestoreCICDStatsService implements CICDStatsService {
+	private db: Firestore;
+
+	constructor() {
+		this.db = firestoreDb();
+	}
+
+	@span()
+	async saveJobResult(jobResult: JobResult): Promise<void> {
+		const docRef = this.db.collection('CICDStats').doc(randomUUID());
+		await docRef.set(jobResult);
+	}
+
+	@span()
+	async getRecentSuccessfulJobs(project: string, jobName: string): Promise<JobResult[]> {
+		const query = this.db
+			.collection('CICDStats')
+			.where('project', '==', project)
+			.where('jobName', '==', jobName)
+			.where('status', '==', 'success')
+			.orderBy('startedAt', 'desc')
+			.limit(20);
+		return query.get().then((querySnapshot) => {
+			const jobResults: JobResult[] = [];
+			querySnapshot.forEach((doc) => {
+				jobResults.push(doc.data() as JobResult);
+			});
+			return jobResults;
+		});
+	}
+}