Feat/git autodeploy

apps/api/src/api/github/index.ts

@@ -151,24 +151,9 @@ export const githubRoutes = new Elysia({ prefix: "/github" })
 			set.status = 401;
 			return { error: "Not authenticated" };
 		}
-		const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&type=all", token);
-		const user = await fetchGitHub("/user", token) as { login: string };
-		const orgs = await fetchGitHub("/user/orgs", token) as { login: string }[];
+		const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&affiliation=owner", token);
 		const allRepos = Array.isArray(repos) ? repos : [];
-		const orgRepos: any[] = [];
-		for (const org of Array.isArray(orgs) ? orgs : []) {
-			try {
-				const orgR = await fetchGitHub(`/orgs/${org.login}/repos?per_page=100&sort=updated&type=all`, token);
-				if (Array.isArray(orgR)) orgRepos.push(...orgR);
-			} catch {}
-		}
-		const seen = new Set<number>();
-		const merged = [...allRepos, ...orgRepos].filter((r) => {
-			if (seen.has(r.id)) return false;
-			seen.add(r.id);
-			return true;
-		});
-		return merged.map((r: any) => ({
+		return allRepos.map((r: any) => ({
 			id: r.id,
 			name: r.name,
 			fullName: r.full_name,

apps/api/src/databases/manager.ts

@@ -1,6 +1,7 @@
 import { spawn } from 'node:child_process';
 import { config } from '../utils/config';
 import { dockerBin } from '../utils/docker-bin';
+import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels';
 import type { Database } from '../types';
 import { updateDatabaseStatus } from '../db/repo';
 
@@ -52,6 +53,7 @@ export const provisionDatabase = async (dbRecord: Database): Promise<void> => {
     '--name', containerName,
     '--network', config.dockerNetwork,
     '--network-alias', containerName,
+    '-l', DEQUEL_MANAGED_LABEL,
     ...(dbRecord.cpuLimit ? ['--cpus', String(dbRecord.cpuLimit)] : []),
     ...(dbRecord.memoryLimitMb ? ['--memory', `${Math.round(dbRecord.memoryLimitMb)}m`] : []),
     '-v', `${volumeName}:/var/lib/${dbRecord.type === 'mysql' ? 'mysql' : 'postgresql/data'}`,

apps/api/src/index.ts

@@ -9,11 +9,11 @@ import { config } from './utils/config';
 import { getDb } from './db/client';
 import { scalingEngine } from './scaling/engine';
 import { serverManager } from './servers/manager';
-import { startGitWatcher } from './git/watcher';
 import { startDomainPolling } from './utils/domain-verifier';
 import { alertEvaluator } from './monitoring/evaluator';
 import { loadOrCreateJwtSecret } from './utils/secrets';
 import { initAuth, cleanupExpiredTokens } from './utils/auth';
+import { startBuildCleanup } from './orchestrator/cleanup';
 const bootstrap = async () => {
   await mkdir(dirname(config.databasePath), { recursive: true });
   await mkdir(config.workspaceRoot, { recursive: true });
@@ -27,9 +27,9 @@ const bootstrap = async () => {
   orchestrator.startWorker();
   scalingEngine.start();
   serverManager.start();
-  startGitWatcher();
   startDomainPolling();
   alertEvaluator.start();
+  startBuildCleanup();
   setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000);
 
   const metrics = {

apps/api/src/orchestrator/__tests__/cleanup.test.ts

@@ -0,0 +1,86 @@
+import { describe, it, expect, mock, beforeEach } from 'bun:test';
+
+const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString();
+
+const tryRunCalls: { cmd: string; args: string[] }[] = [];
+const mockTryRun = mock(async (cmd: string, args: string[]) => {
+  tryRunCalls.push({ cmd, args });
+});
+const mockRedisDel = mock(async () => {});
+const mockRedisLlLen = mock(async () => 0);
+const mockRedisQuit = mock(async () => {});
+
+beforeEach(() => {
+  tryRunCalls.length = 0;
+  mockTryRun.mockClear();
+  mockRedisDel.mockClear();
+  mockRedisLlLen.mockClear();
+  mockRedisQuit.mockClear();
+});
+
+mock.module(fileUrl('../runtime'), () => ({
+  tryRun: mockTryRun,
+}));
+
+mock.module(fileUrl('../utils/config'), () => ({
+  config: {
+    redisUrl: 'redis://localhost:6379',
+  },
+}));
+
+mock.module(fileUrl('../utils/docker-bin'), () => ({
+  dockerBin: '/usr/bin/docker',
+}));
+
+mock.module('ioredis', () => ({
+  default: class FakeRedis {
+    llen = mockRedisLlLen;
+    del = mockRedisDel;
+    quit = mockRedisQuit;
+  },
+}));
+
+const { pruneDocker, pruneDlq } = await import('../cleanup');
+
+describe('pruneDocker', () => {
+  beforeEach(() => {
+    tryRunCalls.length = 0;
+  });
+
+  it('prunes containers with dequel label filter only', async () => {
+    await pruneDocker();
+
+    const containerPrune = tryRunCalls.find(
+      c => c.args[0] === 'container' && c.args[1] === 'prune',
+    );
+    expect(containerPrune).toBeDefined();
+    expect(containerPrune!.args).toContain('-f');
+    expect(containerPrune!.args).toContain('--filter');
+    expect(containerPrune!.args.some(a => a.includes('com.dequel.managed'))).toBe(true);
+  });
+
+  it('prunes dangling images without -a flag (preserves cache)', async () => {
+    await pruneDocker();
+
+    const imagePrune = tryRunCalls.find(
+      c => c.args[0] === 'image' && c.args[1] === 'prune',
+    );
+    expect(imagePrune).toBeDefined();
+    expect(imagePrune!.args).not.toContain('-a');
+
+    const buildxPrune = tryRunCalls.find(
+      c => c.args[0] === 'buildx' && c.args[1] === 'prune',
+    );
+    expect(buildxPrune).toBeDefined();
+    expect(buildxPrune!.args).not.toContain('-a');
+  });
+});
+
+describe('pruneDlq', () => {
+  it('does not need a Redis connection per tick (shared connection)', async () => {
+    await pruneDlq();
+
+    expect(mockRedisLlLen).toHaveBeenCalled();
+    expect(mockRedisQuit).not.toHaveBeenCalled();
+  });
+});

apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts

@@ -0,0 +1,124 @@
+import { describe, it, expect, mock, beforeEach } from 'bun:test';
+
+const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString();
+
+let cleanupFailedDeploymentCalled = false;
+let buildShouldFail = false;
+
+beforeEach(() => {
+  cleanupFailedDeploymentCalled = false;
+  buildShouldFail = false;
+});
+
+const mockDb = {
+  getDeploymentById: mock(() => Promise.resolve({
+    id: 'dep-1',
+    projectId: 'proj-1',
+    sourceType: 'git',
+    sourceRef: 'https://github.com/test/repo.git',
+    branch: 'main',
+    status: 'pending',
+    commitSha: null,
+    clearCache: false,
+    environment: null,
+    imageTag: null,
+    containerName: null,
+  })),
+  getProjectById: mock(() => Promise.resolve({
+    id: 'proj-1',
+    name: 'Test Project',
+    sourceDir: null,
+    port: 3000,
+    cpuLimit: null,
+    memoryLimitMb: null,
+    repoUrl: 'https://github.com/test/repo.git',
+  })),
+  updateDeploymentStatus: mock(() => Promise.resolve()),
+  updateDeploymentCommitSha: mock(() => Promise.resolve()),
+  appendLog: mock(async (_depId: string, stage: string, message: string) => {
+    if (stage === 'system' && message === 'Deployment is running') {
+      throw new Error('Simulated DB write failure');
+    }
+    return { sequence: 1 };
+  }),
+  listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])),
+  listVolumes: mock(() => Promise.resolve([])),
+  listDeployments: mock(() => Promise.resolve([])),
+  listAllDatabases: mock(() => Promise.resolve([])),
+  deleteDeploymentAndLogs: mock(() => Promise.resolve()),
+  getScalingPolicy: mock(() => Promise.resolve(null)),
+  updateDomainValidation: mock(() => Promise.resolve()),
+  listDomains: mock(() => Promise.resolve([])),
+};
+
+mock.module(fileUrl('../../db/repo'), () => mockDb);
+
+mock.module(fileUrl('../runtime'), () => ({
+  deployContainer: mock(() => Promise.resolve({
+    containerName: 'test-project-abc12345',
+    liveUrl: 'http://test-project.localhost',
+  })),
+  cleanupFailedDeployment: mock(async () => {
+    cleanupFailedDeploymentCalled = true;
+  }),
+  ensureContainerRunning: mock(() => Promise.resolve()),
+  reloadCaddy: mock(() => Promise.resolve()),
+  tryRun: mock(() => Promise.resolve('')),
+}));
+
+mock.module(fileUrl('../railpack'), () => ({
+  buildWithRailpack: mock(async (
+    _workspace: string,
+    _imageTag: string,
+    _onLog: (line: string) => Promise<void>,
+    _opts?: unknown,
+  ) => {
+    if (buildShouldFail) throw new Error('Build failed');
+  }),
+  CancelledError: class CancelledError extends Error {
+    constructor() { super('Cancelled'); }
+  },
+}));
+
+mock.module(fileUrl('../source'), () => ({
+  prepareSourceWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')),
+  prepareUploadWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')),
+  cleanupWorkspace: mock(() => Promise.resolve()),
+  getHeadSha: mock(() => Promise.resolve('abc123def456')),
+}));
+
+mock.module(fileUrl('../../utils/grafana'), () => ({
+  ensureProjectDashboard: mock(() => Promise.resolve()),
+}));
+
+mock.module('ioredis', () => ({
+  default: class FakeRedis {
+    queue: string[] = [];
+    rpush = mock(async (_key: string, ...values: string[]) => { this.queue.push(...values); });
+    blpop = mock(async () => { const v = this.queue.shift(); return v ? ['queue', v] : null; });
+    lrem = mock(async () => {});
+    zadd = mock(async () => {});
+    zrem = mock(async () => {});
+    zrangebyscore = mock(async () => []);
+    quit = mock(async () => {});
+  },
+}));
+
+const { PipelineOrchestrator } = await import('../pipeline');
+
+describe('runDeployment cleanup behavior', () => {
+  it('does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds', async () => {
+    const orchestrator = new PipelineOrchestrator();
+    await (orchestrator as any).runDeployment('dep-1');
+
+    expect(cleanupFailedDeploymentCalled).toBe(false);
+  });
+
+  it('calls cleanupFailedDeployment when failure occurs during build (before deployContainer)', async () => {
+    buildShouldFail = true;
+    const orchestrator = new PipelineOrchestrator();
+    await (orchestrator as any).runDeployment('dep-1');
+
+    expect(cleanupFailedDeploymentCalled).toBe(true);
+  });
+});

apps/api/src/orchestrator/cleanup.ts

@@ -0,0 +1,49 @@
+import Redis from 'ioredis';
+import { config } from '../utils/config';
+import { dockerBin } from '../utils/docker-bin';
+import { tryRun } from './runtime';
+import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels';
+
+const DLQ_KEY = 'dequel:deploy:dlq';
+const GC_INTERVAL_MS = 1_800_000;
+
+let interval: ReturnType<typeof setInterval> | null = null;
+let redis: Redis | null = null;
+
+const getRedis = () => {
+  if (!redis) redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null });
+  return redis;
+};
+
+export const pruneDocker = async () => {
+  await tryRun(dockerBin, ['container', 'prune', '-f', '--filter', `label=${DEQUEL_MANAGED_LABEL}`]);
+  await tryRun(dockerBin, ['image', 'prune', '-f']);
+  await tryRun(dockerBin, ['buildx', 'prune', '-f']);
+};
+
+export const pruneDlq = async () => {
+  try {
+    const r = getRedis();
+    const size = await r.llen(DLQ_KEY);
+    if (size > 0) {
+      await r.del(DLQ_KEY);
+      console.log(`[Cleanup] Purged ${size} items from dead letter queue`);
+    }
+  } catch (e) {
+    console.warn('[Cleanup] DLQ prune failed:', e);
+  }
+};
+
+export const startBuildCleanup = () => {
+  if (interval) return;
+  console.log('[Cleanup] Docker garbage collector started (every 30min)');
+  interval = setInterval(async () => {
+    await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e));
+    await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e));
+  }, GC_INTERVAL_MS);
+};
+
+export const stopBuildCleanup = () => {
+  if (interval) { clearInterval(interval); interval = null; }
+  if (redis) { redis.quit(); redis = null; }
+};