From 8501dd369305dcef7da1e07e143ced0e3fc031c9 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 14 Jul 2026 23:58:37 +0100 Subject: [PATCH 1/8] feat(orchestrator): implement cleanup system for builds and containers - Add `DEQUEL_MANAGED_LABEL` for Docker resource tracking - Implement periodic Docker and dead-letter queue garbage collection - Add `cleanupFailedDeployment` to remove artifacts on pipeline failure - Improve worker connection management in deployment queue --- apps/api/src/databases/manager.ts | 2 + apps/api/src/git/watcher.ts | 57 -------- apps/api/src/index.ts | 4 +- .../orchestrator/__tests__/cleanup.test.ts | 86 ++++++++++++ .../__tests__/pipeline-cleanup.test.ts | 124 ++++++++++++++++++ apps/api/src/orchestrator/cleanup.ts | 49 +++++++ apps/api/src/orchestrator/pipeline.ts | 30 ++++- apps/api/src/orchestrator/queue.ts | 37 ++++-- apps/api/src/orchestrator/runtime.ts | 21 +++ apps/api/src/scaling/engine.ts | 2 + apps/api/src/utils/dequel-labels.ts | 1 + 11 files changed, 338 insertions(+), 75 deletions(-) delete mode 100644 apps/api/src/git/watcher.ts create mode 100644 apps/api/src/orchestrator/__tests__/cleanup.test.ts create mode 100644 apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts create mode 100644 apps/api/src/orchestrator/cleanup.ts create mode 100644 apps/api/src/utils/dequel-labels.ts diff --git a/apps/api/src/databases/manager.ts b/apps/api/src/databases/manager.ts index 6821bda..8c02b04 100644 --- a/apps/api/src/databases/manager.ts +++ b/apps/api/src/databases/manager.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import type { Database } from '../types'; import { updateDatabaseStatus } from '../db/repo'; @@ -52,6 +53,7 @@ export const provisionDatabase = async (dbRecord: Database): Promise => { '--name', containerName, '--network', config.dockerNetwork, '--network-alias', containerName, + '-l', DEQUEL_MANAGED_LABEL, ...(dbRecord.cpuLimit ? ['--cpus', String(dbRecord.cpuLimit)] : []), ...(dbRecord.memoryLimitMb ? ['--memory', `${Math.round(dbRecord.memoryLimitMb)}m`] : []), '-v', `${volumeName}:/var/lib/${dbRecord.type === 'mysql' ? 'mysql' : 'postgresql/data'}`, diff --git a/apps/api/src/git/watcher.ts b/apps/api/src/git/watcher.ts deleted file mode 100644 index a6da875..0000000 --- a/apps/api/src/git/watcher.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { getDb } from '../db/client'; -import { listProjects } from '../db/repo'; -import { createDeployment } from '../db/repo'; -import { orchestrator } from '../orchestrator'; -import { getRemoteSha } from '../orchestrator/source'; - -const POLL_INTERVAL_MS = 60_000; - -let interval: ReturnType | null = null; - -export const startGitWatcher = () => { - if (interval) return; - interval = setInterval(poll, POLL_INTERVAL_MS); - console.log('[GitWatcher] Started (poll every 60s)'); -}; - -export const stopGitWatcher = () => { - if (interval) { clearInterval(interval); interval = null; } -}; - -async function poll() { - try { - const db = await getDb(); - const projects = db.query( - "SELECT * FROM projects WHERE repo_url IS NOT NULL AND repo_url != ''", - ).all() as any[]; - - for (const project of projects) { - const branch = project.repo_branch || 'main'; - - const latest = db.query( - 'SELECT * FROM deployments WHERE project_id = ? AND source_type = ? ORDER BY created_at DESC LIMIT 1', - ).get(project.id, 'git') as any; - - if (!latest?.commit_sha) continue; - - const remoteSha = await getRemoteSha(project.repo_url, branch); - if (!remoteSha) continue; - - if (remoteSha === latest.commit_sha) continue; - - console.log(`[GitWatcher] New commit detected for ${project.name} (${branch}): ${remoteSha.slice(0, 7)}`); - - const dep = await createDeployment({ - projectId: project.id, - sourceType: 'git', - sourceRef: project.repo_url, - branch, - commitSha: remoteSha, - }); - - orchestrator.enqueue(dep.id); - } - } catch (err) { - console.error('[GitWatcher] Poll error:', err); - } -} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 5fad630..e63a333 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,11 +9,11 @@ import { config } from './utils/config'; import { getDb } from './db/client'; import { scalingEngine } from './scaling/engine'; import { serverManager } from './servers/manager'; -import { startGitWatcher } from './git/watcher'; import { startDomainPolling } from './utils/domain-verifier'; import { alertEvaluator } from './monitoring/evaluator'; import { loadOrCreateJwtSecret } from './utils/secrets'; import { initAuth, cleanupExpiredTokens } from './utils/auth'; +import { startBuildCleanup } from './orchestrator/cleanup'; const bootstrap = async () => { await mkdir(dirname(config.databasePath), { recursive: true }); await mkdir(config.workspaceRoot, { recursive: true }); @@ -27,9 +27,9 @@ const bootstrap = async () => { orchestrator.startWorker(); scalingEngine.start(); serverManager.start(); - startGitWatcher(); startDomainPolling(); alertEvaluator.start(); + startBuildCleanup(); setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); const metrics = { diff --git a/apps/api/src/orchestrator/__tests__/cleanup.test.ts b/apps/api/src/orchestrator/__tests__/cleanup.test.ts new file mode 100644 index 0000000..0986179 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/cleanup.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +const tryRunCalls: { cmd: string; args: string[] }[] = []; +const mockTryRun = mock(async (cmd: string, args: string[]) => { + tryRunCalls.push({ cmd, args }); +}); +const mockRedisDel = mock(async () => {}); +const mockRedisLlLen = mock(async () => 0); +const mockRedisQuit = mock(async () => {}); + +beforeEach(() => { + tryRunCalls.length = 0; + mockTryRun.mockClear(); + mockRedisDel.mockClear(); + mockRedisLlLen.mockClear(); + mockRedisQuit.mockClear(); +}); + +mock.module(fileUrl('../runtime'), () => ({ + tryRun: mockTryRun, +})); + +mock.module(fileUrl('../utils/config'), () => ({ + config: { + redisUrl: 'redis://localhost:6379', + }, +})); + +mock.module(fileUrl('../utils/docker-bin'), () => ({ + dockerBin: '/usr/bin/docker', +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + llen = mockRedisLlLen; + del = mockRedisDel; + quit = mockRedisQuit; + }, +})); + +const { pruneDocker, pruneDlq } = await import('../cleanup'); + +describe('pruneDocker', () => { + beforeEach(() => { + tryRunCalls.length = 0; + }); + + it('prunes containers with dequel label filter only', async () => { + await pruneDocker(); + + const containerPrune = tryRunCalls.find( + c => c.args[0] === 'container' && c.args[1] === 'prune', + ); + expect(containerPrune).toBeDefined(); + expect(containerPrune!.args).toContain('-f'); + expect(containerPrune!.args).toContain('--filter'); + expect(containerPrune!.args.some(a => a.includes('com.dequel.managed'))).toBe(true); + }); + + it('prunes dangling images without -a flag (preserves cache)', async () => { + await pruneDocker(); + + const imagePrune = tryRunCalls.find( + c => c.args[0] === 'image' && c.args[1] === 'prune', + ); + expect(imagePrune).toBeDefined(); + expect(imagePrune!.args).not.toContain('-a'); + + const buildxPrune = tryRunCalls.find( + c => c.args[0] === 'buildx' && c.args[1] === 'prune', + ); + expect(buildxPrune).toBeDefined(); + expect(buildxPrune!.args).not.toContain('-a'); + }); +}); + +describe('pruneDlq', () => { + it('does not need a Redis connection per tick (shared connection)', async () => { + await pruneDlq(); + + expect(mockRedisLlLen).toHaveBeenCalled(); + expect(mockRedisQuit).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts new file mode 100644 index 0000000..0e8a0b9 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +let cleanupFailedDeploymentCalled = false; +let buildShouldFail = false; + +beforeEach(() => { + cleanupFailedDeploymentCalled = false; + buildShouldFail = false; +}); + +const mockDb = { + getDeploymentById: mock(() => Promise.resolve({ + id: 'dep-1', + projectId: 'proj-1', + sourceType: 'git', + sourceRef: 'https://github.com/test/repo.git', + branch: 'main', + status: 'pending', + commitSha: null, + clearCache: false, + environment: null, + imageTag: null, + containerName: null, + })), + getProjectById: mock(() => Promise.resolve({ + id: 'proj-1', + name: 'Test Project', + sourceDir: null, + port: 3000, + cpuLimit: null, + memoryLimitMb: null, + repoUrl: 'https://github.com/test/repo.git', + })), + updateDeploymentStatus: mock(() => Promise.resolve()), + updateDeploymentCommitSha: mock(() => Promise.resolve()), + appendLog: mock(async (_depId: string, stage: string, message: string) => { + if (stage === 'system' && message === 'Deployment is running') { + throw new Error('Simulated DB write failure'); + } + return { sequence: 1 }; + }), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), + listVolumes: mock(() => Promise.resolve([])), + listDeployments: mock(() => Promise.resolve([])), + listAllDatabases: mock(() => Promise.resolve([])), + deleteDeploymentAndLogs: mock(() => Promise.resolve()), + getScalingPolicy: mock(() => Promise.resolve(null)), + updateDomainValidation: mock(() => Promise.resolve()), + listDomains: mock(() => Promise.resolve([])), +}; + +mock.module(fileUrl('../../db/repo'), () => mockDb); + +mock.module(fileUrl('../runtime'), () => ({ + deployContainer: mock(() => Promise.resolve({ + containerName: 'test-project-abc12345', + liveUrl: 'http://test-project.localhost', + })), + cleanupFailedDeployment: mock(async () => { + cleanupFailedDeploymentCalled = true; + }), + ensureContainerRunning: mock(() => Promise.resolve()), + reloadCaddy: mock(() => Promise.resolve()), + tryRun: mock(() => Promise.resolve('')), +})); + +mock.module(fileUrl('../railpack'), () => ({ + buildWithRailpack: mock(async ( + _workspace: string, + _imageTag: string, + _onLog: (line: string) => Promise, + _opts?: unknown, + ) => { + if (buildShouldFail) throw new Error('Build failed'); + }), + CancelledError: class CancelledError extends Error { + constructor() { super('Cancelled'); } + }, +})); + +mock.module(fileUrl('../source'), () => ({ + prepareSourceWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + prepareUploadWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + cleanupWorkspace: mock(() => Promise.resolve()), + getHeadSha: mock(() => Promise.resolve('abc123def456')), +})); + +mock.module(fileUrl('../../utils/grafana'), () => ({ + ensureProjectDashboard: mock(() => Promise.resolve()), +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + queue: string[] = []; + rpush = mock(async (_key: string, ...values: string[]) => { this.queue.push(...values); }); + blpop = mock(async () => { const v = this.queue.shift(); return v ? ['queue', v] : null; }); + lrem = mock(async () => {}); + zadd = mock(async () => {}); + zrem = mock(async () => {}); + zrangebyscore = mock(async () => []); + quit = mock(async () => {}); + }, +})); + +const { PipelineOrchestrator } = await import('../pipeline'); + +describe('runDeployment cleanup behavior', () => { + it('does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds', async () => { + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(false); + }); + + it('calls cleanupFailedDeployment when failure occurs during build (before deployContainer)', async () => { + buildShouldFail = true; + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(true); + }); +}); diff --git a/apps/api/src/orchestrator/cleanup.ts b/apps/api/src/orchestrator/cleanup.ts new file mode 100644 index 0000000..6bf3adc --- /dev/null +++ b/apps/api/src/orchestrator/cleanup.ts @@ -0,0 +1,49 @@ +import Redis from 'ioredis'; +import { config } from '../utils/config'; +import { dockerBin } from '../utils/docker-bin'; +import { tryRun } from './runtime'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; + +const DLQ_KEY = 'dequel:deploy:dlq'; +const GC_INTERVAL_MS = 1_800_000; + +let interval: ReturnType | null = null; +let redis: Redis | null = null; + +const getRedis = () => { + if (!redis) redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + return redis; +}; + +export const pruneDocker = async () => { + await tryRun(dockerBin, ['container', 'prune', '-f', '--filter', `label=${DEQUEL_MANAGED_LABEL}`]); + await tryRun(dockerBin, ['image', 'prune', '-f']); + await tryRun(dockerBin, ['buildx', 'prune', '-f']); +}; + +export const pruneDlq = async () => { + try { + const r = getRedis(); + const size = await r.llen(DLQ_KEY); + if (size > 0) { + await r.del(DLQ_KEY); + console.log(`[Cleanup] Purged ${size} items from dead letter queue`); + } + } catch (e) { + console.warn('[Cleanup] DLQ prune failed:', e); + } +}; + +export const startBuildCleanup = () => { + if (interval) return; + console.log('[Cleanup] Docker garbage collector started (every 30min)'); + interval = setInterval(async () => { + await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e)); + await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e)); + }, GC_INTERVAL_MS); +}; + +export const stopBuildCleanup = () => { + if (interval) { clearInterval(interval); interval = null; } + if (redis) { redis.quit(); redis = null; } +}; diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 3e75590..91e8b77 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -21,6 +21,7 @@ import { getHeadSha, } from "./source"; import { + cleanupFailedDeployment, deployContainer, ensureContainerRunning, reloadCaddy, @@ -158,6 +159,10 @@ export class PipelineOrchestrator { ]); } + if (deployment.imageTag) { + await tryRun("docker", ["rmi", "-f", deployment.imageTag]); + } + await deleteDeploymentAndLogs( deploymentId, ); @@ -290,6 +295,9 @@ export class PipelineOrchestrator { let workspacePath = ""; let uploadedArchivePath: string | null = null; + let imageTag: string | undefined; + let projectName: string | undefined; + let deployed = false; try { await emitLog( @@ -298,7 +306,7 @@ export class PipelineOrchestrator { "Deployment enqueued", ); - const imageTag = + imageTag = await this.resolveImageTag( deployment, ); @@ -484,7 +492,6 @@ export class PipelineOrchestrator { await this.checkCancelled(deploymentId); - let projectName: string | undefined; let cpuLimit: | number | null @@ -546,6 +553,8 @@ export class PipelineOrchestrator { }, ); + deployed = true; + if (projectName) { const dashSlug = projectName .toLowerCase() @@ -622,6 +631,23 @@ export class PipelineOrchestrator { "failed", { failureReason: message }, ); + + if (!deployed) { + await emitLog( + deploymentId, + "system", + "Cleaning up Docker resources from failed deployment", + ); + await cleanupFailedDeployment( + deploymentId, + imageTag, + projectName, + deployment.projectId, + ).catch(e => + console.warn(`[Cleanup] Failed to clean deployment ${deploymentId}:`, e), + ); + } + if (deployment.projectId) { try { const dbs = diff --git a/apps/api/src/orchestrator/queue.ts b/apps/api/src/orchestrator/queue.ts index 3fdfe51..2b8ecb3 100644 --- a/apps/api/src/orchestrator/queue.ts +++ b/apps/api/src/orchestrator/queue.ts @@ -22,12 +22,14 @@ const decodeJob = (raw: string): JobPayload | null => { } }; +const createRedis = () => new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + export class DeploymentQueue { private redis: Redis; private shuttingDown = false; constructor() { - this.redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + this.redis = createRedis(); } async enqueue(deploymentId: string) { @@ -41,7 +43,9 @@ export class DeploymentQueue { encodeJob({ id: deploymentId, attempt: i }), ); await this.redis.zrem(RETRY_KEY, ...allAttempts); - await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: 0 })); + for (let i = 0; i <= config.queueRetryMax + 1; i++) { + await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: i })); + } } async start(handler: (deploymentId: string) => Promise) { @@ -55,21 +59,26 @@ export class DeploymentQueue { } private async runWorker(workerId: number, handler: (deploymentId: string) => Promise) { - while (!this.shuttingDown) { - await this.requeueDueJobs(); + const workerRedis = createRedis(); + try { + while (!this.shuttingDown) { + await this.requeueDueJobs(); - const item = await this.redis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); - if (!item) continue; - const job = decodeJob(item[1]); - if (!job) continue; + const item = await workerRedis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); + if (!item) continue; + const job = decodeJob(item[1]); + if (!job) continue; - try { - const ok = await handler(job.id); - if (!ok) await this.retryOrDlq(job); - } catch (err) { - console.error(`[Queue] Worker ${workerId} handler error:`, err); - await this.retryOrDlq(job); + try { + const ok = await handler(job.id); + if (!ok) await this.retryOrDlq(job); + } catch (err) { + console.error(`[Queue] Worker ${workerId} handler error:`, err); + await this.retryOrDlq(job); + } } + } finally { + await workerRedis.quit(); } } diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index 8370f09..f1b82e1 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; const slugify = (s: string) => s.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 63); @@ -99,6 +100,25 @@ export const reloadCaddy = async () => { await run(dockerBin, ['exec', caddyContainer, 'caddy', 'reload', '--config', '/etc/caddy/Caddyfile']); }; +export const getContainerName = (deploymentId: string, projectName?: string, projectId?: string) => { + const slug = slugify(projectName || projectId || deploymentId); + return `${slug}-${deploymentId.slice(0, 8)}`; +}; + +export const cleanupFailedDeployment = async ( + deploymentId: string, + imageTag?: string | null, + projectName?: string, + projectId?: string, +) => { + const containerName = getContainerName(deploymentId, projectName, projectId); + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); + await tryRun(dockerBin, ['rm', '-f', containerName]); + if (imageTag) { + await tryRun(dockerBin, ['rmi', '-f', imageTag]); + } +}; + export const deployContainer = async ( deploymentId: string, imageTag: string, @@ -117,6 +137,7 @@ export const deployContainer = async ( 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, '-e', `PORT=${opts.appPort ?? config.appInternalPort}`, ]; diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 16e544b..afdc173 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import Redis from 'ioredis'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import { run, tryRun } from './docker-utils'; import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy } from '../db/repo'; @@ -201,6 +202,7 @@ class ScalingEngine { await run(dockerBin, [ 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, ...volumes, ...envVars, imageTag, ]); diff --git a/apps/api/src/utils/dequel-labels.ts b/apps/api/src/utils/dequel-labels.ts new file mode 100644 index 0000000..cac7a70 --- /dev/null +++ b/apps/api/src/utils/dequel-labels.ts @@ -0,0 +1 @@ +export const DEQUEL_MANAGED_LABEL = 'com.dequel.managed=true'; From 3b73df998a25231e4ae678dad3955738320ee662 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:14:29 +0100 Subject: [PATCH 2/8] feat(web): add mobile-responsive navigation and layouts - Implement mobile sidebar toggle - Make dashboard tables and tabs scrollable on small screens - Set restart policies for docker services - Improve CLI installation and path resolution --- apps/web/src/components/Layout.tsx | 14 ++- apps/web/src/components/layout/Header.tsx | 14 ++- .../components/layout/NotificationBanner.tsx | 2 +- apps/web/src/components/layout/Sidebar.tsx | 95 ++++++++++++------- .../deployments/deployment-history.tsx | 4 +- .../components/project/domains/DomainsTab.tsx | 4 +- .../components/project/envtab/EnvVarTable.tsx | 4 +- .../components/project/volumes/VolumesTab.tsx | 4 +- apps/web/src/routes/ProjectDetail.tsx | 20 ++-- apps/web/src/routes/Settings.tsx | 10 +- docker-compose.yml | 11 +++ scripts/dequel | 12 ++- scripts/install.sh | 13 +++ 13 files changed, 143 insertions(+), 64 deletions(-) diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index d47c4a9..29c4c82 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -33,6 +33,11 @@ export function Layout({ children }: { children: React.ReactNode }) { const { data: projects = [] } = useProjects({ enabled: !!me?.authenticated && location.pathname !== "/login" }); const [projectSelectorOpen, setProjectSelectorOpen] = useState(false); + const [sidebarOpen, setSidebarOpen] = useState(false); + + useEffect(() => { + setSidebarOpen(false); + }, [location.pathname, location.search]); const { data: metricsText } = useQuery({ queryKey: ["metrics"], @@ -100,7 +105,7 @@ export function Layout({ children }: { children: React.ReactNode }) { } return ( -
+
-
+
setNotification(null)} /> -
{children}
+
{children}
); diff --git a/apps/web/src/components/layout/Header.tsx b/apps/web/src/components/layout/Header.tsx index 7b686c5..cce94ce 100644 --- a/apps/web/src/components/layout/Header.tsx +++ b/apps/web/src/components/layout/Header.tsx @@ -1,19 +1,29 @@ import { Link } from "@tanstack/react-router"; -import { ChevronRight } from "lucide-react"; +import { ChevronRight, Menu } from "lucide-react"; interface HeaderProps { currentProject: { name: string } | undefined; currentProjectId: string | null; location: { pathname: string; search: any }; + setSidebarOpen: (open: boolean) => void; } export function Header({ currentProject, currentProjectId, location, + setSidebarOpen, }: HeaderProps) { return ( -
+
+ + Workspace diff --git a/apps/web/src/components/layout/NotificationBanner.tsx b/apps/web/src/components/layout/NotificationBanner.tsx index 35755d6..cb1c6ec 100644 --- a/apps/web/src/components/layout/NotificationBanner.tsx +++ b/apps/web/src/components/layout/NotificationBanner.tsx @@ -18,7 +18,7 @@ export function NotificationBanner({ const Icon = notification.type === "success" ? CheckCircle2 : AlertCircle; return ( -
+
void; + sidebarOpen: boolean; + setSidebarOpen: (open: boolean) => void; } export function Sidebar({ @@ -29,43 +33,70 @@ export function Sidebar({ metrics, location, navigate, + sidebarOpen, + setSidebarOpen, }: SidebarProps) { return ( - + ); } diff --git a/apps/web/src/components/project/deployments/deployment-history.tsx b/apps/web/src/components/project/deployments/deployment-history.tsx index 49afcf0..d3e8d3c 100644 --- a/apps/web/src/components/project/deployments/deployment-history.tsx +++ b/apps/web/src/components/project/deployments/deployment-history.tsx @@ -51,8 +51,8 @@ export function DeploymentHistory({ return ( <> -
- +
+
Status diff --git a/apps/web/src/components/project/domains/DomainsTab.tsx b/apps/web/src/components/project/domains/DomainsTab.tsx index 3cef41e..c2ab828 100644 --- a/apps/web/src/components/project/domains/DomainsTab.tsx +++ b/apps/web/src/components/project/domains/DomainsTab.tsx @@ -366,8 +366,8 @@ export function DomainsTab({ )} -
-
+
+
diff --git a/apps/web/src/components/project/envtab/EnvVarTable.tsx b/apps/web/src/components/project/envtab/EnvVarTable.tsx index f6c4314..2c5dd58 100644 --- a/apps/web/src/components/project/envtab/EnvVarTable.tsx +++ b/apps/web/src/components/project/envtab/EnvVarTable.tsx @@ -77,8 +77,8 @@ export function EnvVarTable({ -
-
+
+
diff --git a/apps/web/src/components/project/volumes/VolumesTab.tsx b/apps/web/src/components/project/volumes/VolumesTab.tsx index d269a75..f67d4d4 100644 --- a/apps/web/src/components/project/volumes/VolumesTab.tsx +++ b/apps/web/src/components/project/volumes/VolumesTab.tsx @@ -233,8 +233,8 @@ export function VolumesTab({ -
-
+
+
diff --git a/apps/web/src/routes/ProjectDetail.tsx b/apps/web/src/routes/ProjectDetail.tsx index f1e69db..ee878f3 100644 --- a/apps/web/src/routes/ProjectDetail.tsx +++ b/apps/web/src/routes/ProjectDetail.tsx @@ -106,32 +106,32 @@ export function ProjectDetail({ navigate({ search: { tab: val } as any })} className="space-y-4"> - - + + Deployments - + Env Vars - + Volumes - + Databases - + Domains - + Scaling - + Alerts - + Observability - + Logs diff --git a/apps/web/src/routes/Settings.tsx b/apps/web/src/routes/Settings.tsx index fb16b15..466b44c 100644 --- a/apps/web/src/routes/Settings.tsx +++ b/apps/web/src/routes/Settings.tsx @@ -72,8 +72,8 @@ function ServersSection() { {servers.length > 0 && ( -
-
+
+
NameHostStatus @@ -322,7 +322,7 @@ function ApiKeysSection() { {newKey} )} - +
setName(e.target.value)} className="w-56" /> @@ -330,8 +330,8 @@ function ApiKeysSection() { {apiKeys.length > 0 && ( -
-
+
+
NameKey HashCreated diff --git a/docker-compose.yml b/docker-compose.yml index f63cf1d..0f22018 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: buildkit: image: moby/buildkit:latest + restart: unless-stopped command: [ "--addr", @@ -28,6 +29,7 @@ services: api: # image: ghcr.io/lftobs/dequel/api:latest + restart: unless-stopped build: context: . dockerfile: ./apps/api/Dockerfile @@ -73,6 +75,7 @@ services: pam-auth: image: python:3-slim + restart: unless-stopped entrypoint: [] command: - sh @@ -103,6 +106,7 @@ services: web: # image: ghcr.io/lftobs/dequel/web:latest + restart: unless-stopped build: context: ./apps/web healthcheck: @@ -124,6 +128,7 @@ services: caddy: image: caddy:2.8-alpine + restart: unless-stopped command: [ "caddy", @@ -153,6 +158,7 @@ services: redis: image: redis:7-alpine + restart: unless-stopped command: redis-server --appendonly yes volumes: - redis-data:/data @@ -168,6 +174,7 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest + restart: unless-stopped volumes: - /:/rootfs:ro - /var/run:/var/run:ro @@ -197,6 +204,7 @@ services: prometheus: image: prom/prometheus:latest + restart: unless-stopped stop_grace_period: 60s entrypoint: [] command: @@ -229,6 +237,7 @@ services: loki: image: grafana/loki:3.0.0 + restart: unless-stopped command: - -config.file=/etc/loki/loki-config.yml volumes: @@ -254,6 +263,7 @@ services: promtail: image: grafana/promtail:3.0.0 + restart: unless-stopped command: - -config.file=/etc/promtail/promtail-config.yml volumes: @@ -269,6 +279,7 @@ services: grafana: image: grafana/grafana:latest + restart: unless-stopped environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin diff --git a/scripts/dequel b/scripts/dequel index 811a722..3e67244 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -DEQUEL_HOME="${DEQUEL_HOME:-$HOME/.dequel}" +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" +DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" VERSION="0.1.0" @@ -27,7 +28,12 @@ cmd() { cmd_start() { header "Starting Dequel" cmd up -d - success "Dequel is running at http://localhost" + local domain="${CADDY_BASE_DOMAIN:-localhost}" + if [ "$domain" = "localhost" ]; then + success "Dequel is running at http://localhost" + else + success "Dequel is running at https://$domain" + fi } cmd_stop() { @@ -130,7 +136,7 @@ cmd_help() { echo " --help Show this help" echo "" echo "Environment:" - echo " DEQUEL_HOME Config directory (default: \$HOME/.dequel)" + echo " DEQUEL_HOME Config directory (default: auto-detected from install path)" } case "${1:-}" in diff --git a/scripts/install.sh b/scripts/install.sh index dd9508a..d40e504 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -40,6 +40,19 @@ check_prerequisites() { fail "Docker is not installed. See https://docs.docker.com/engine/install/" fi + if docker info >/dev/null 2>&1; then + success "Docker socket accessible" + else + warn "Docker socket not accessible — attempting to add user to docker group" + if sudo usermod -aG docker "$USER" 2>/dev/null; then + success "Added $USER to docker group" + warn "Log out and back in (or run 'newgrp docker') for the change to take effect" + else + warn "Could not add user to docker group automatically" + warn "Run this manually: sudo usermod -aG docker $USER && newgrp docker" + fi + fi + if docker compose version >/dev/null 2>&1; then success "Docker Compose found: $(docker compose version)" COMPOSE_CMD="docker compose" From dfb54e5550f90c7ddf8c4f6c51bd2a1e4ec317da Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:28:49 +0100 Subject: [PATCH 3/8] feat(api): simplify repository fetching to owner affiliation --- apps/api/src/api/github/index.ts | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index b2b6e83..f295140 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -151,24 +151,9 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 401; return { error: "Not authenticated" }; } - const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&type=all", token); - const user = await fetchGitHub("/user", token) as { login: string }; - const orgs = await fetchGitHub("/user/orgs", token) as { login: string }[]; + const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&affiliation=owner", token); const allRepos = Array.isArray(repos) ? repos : []; - const orgRepos: any[] = []; - for (const org of Array.isArray(orgs) ? orgs : []) { - try { - const orgR = await fetchGitHub(`/orgs/${org.login}/repos?per_page=100&sort=updated&type=all`, token); - if (Array.isArray(orgR)) orgRepos.push(...orgR); - } catch {} - } - const seen = new Set(); - const merged = [...allRepos, ...orgRepos].filter((r) => { - if (seen.has(r.id)) return false; - seen.add(r.id); - return true; - }); - return merged.map((r: any) => ({ + return allRepos.map((r: any) => ({ id: r.id, name: r.name, fullName: r.full_name, From e6d0e69df2d9e638be09d7b6237d1798301ed2b6 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:36:33 +0100 Subject: [PATCH 4/8] feat(scripts): implement update cmd for dequel --- scripts/dequel | 78 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/scripts/dequel b/scripts/dequel index 3e67244..fc3942d 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,10 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" -DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" +SCRIPT_PATH="$(readlink -f "$0" 2>/dev/null || realpath "$0" 2>/dev/null || echo "$0")" +SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" +DEQUEL_HOME="${DEQUEL_HOME:-$SCRIPT_DIR}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" -VERSION="0.1.0" +INSTALLED_VERSION="$(cat "$DEQUEL_HOME/VERSION" 2>/dev/null || echo "0.1.0")" BOLD='\033[1m' DIM='\033[2m' @@ -59,9 +60,74 @@ cmd_logs() { cmd_update() { header "Updating Dequel" + + local repo="Lftobs/dequel" + local tag="" + local base_url="" + + info "Checking for latest release..." + tag=$(curl -fsSL "https://api.github.com/repos/$repo/releases/latest" \ + | grep '"tag_name"' | head -1 | sed -E 's/.*"([^"]+)".*/\1/') || true + + if [ -z "$tag" ]; then + warn "Could not determine latest release, using main branch" + base_url="https://raw.githubusercontent.com/$repo/main" + else + base_url="https://raw.githubusercontent.com/$repo/$tag" + success "Latest release: $tag" + fi + + local tmp_dir + tmp_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$tmp_dir'" EXIT + + info "Downloading updated configuration..." + curl -fsSL "$base_url/docker-compose.yml" -o "$tmp_dir/docker-compose.yml" + curl -fsSL "$base_url/infra/caddy/Caddyfile" -o "$tmp_dir/Caddyfile" + curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" + curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/$f" + done + + for f in loki.yml prometheus.yml; do + curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/$f" + done + + for f in dashboards.yml deployed-apps.json; do + curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/$f" + done + + mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" + mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" + mkdir -p "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ + "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/$f" + done + for f in loki.yml prometheus.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" + done + for f in dashboards.yml deployed-apps.json; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" + done + + mv "$tmp_dir/VERSION" "$DEQUEL_HOME/VERSION" + chmod +x "$tmp_dir/dequel" + mv "$tmp_dir/dequel" "$SCRIPT_PATH" + + success "Configuration updated" + + header "Pulling Docker images" cmd pull + + header "Recreating services" cmd up -d - success "Dequel updated." + + success "Dequel updated to $tag" } cmd_uninstall() { @@ -116,7 +182,7 @@ cmd_uninstall() { } cmd_version() { - echo "dequel v$VERSION" + echo "dequel v$INSTALLED_VERSION" } cmd_help() { @@ -130,7 +196,7 @@ cmd_help() { echo " restart Restart all Dequel services" echo " status Show service status" echo " logs Follow service logs" - echo " update Pull latest images and recreate services" + echo " update Download latest config, pull images, and recreate services" echo " uninstall Remove Dequel completely (config, images, volumes)" echo " --version Show version" echo " --help Show this help" From 052b3ab57e6254d7248ebedac53455f8b87976f3 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 14 Jul 2026 23:58:37 +0100 Subject: [PATCH 5/8] feat(orchestrator): implement cleanup system for builds and containers - Add `DEQUEL_MANAGED_LABEL` for Docker resource tracking - Implement periodic Docker and dead-letter queue garbage collection - Add `cleanupFailedDeployment` to remove artifacts on pipeline failure - Improve worker connection management in deployment queue --- apps/api/src/databases/manager.ts | 2 + apps/api/src/git/watcher.ts | 57 -------- apps/api/src/index.ts | 4 +- .../orchestrator/__tests__/cleanup.test.ts | 86 ++++++++++++ .../__tests__/pipeline-cleanup.test.ts | 124 ++++++++++++++++++ apps/api/src/orchestrator/cleanup.ts | 49 +++++++ apps/api/src/orchestrator/pipeline.ts | 30 ++++- apps/api/src/orchestrator/queue.ts | 37 ++++-- apps/api/src/orchestrator/runtime.ts | 21 +++ apps/api/src/scaling/engine.ts | 2 + apps/api/src/utils/dequel-labels.ts | 1 + 11 files changed, 338 insertions(+), 75 deletions(-) delete mode 100644 apps/api/src/git/watcher.ts create mode 100644 apps/api/src/orchestrator/__tests__/cleanup.test.ts create mode 100644 apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts create mode 100644 apps/api/src/orchestrator/cleanup.ts create mode 100644 apps/api/src/utils/dequel-labels.ts diff --git a/apps/api/src/databases/manager.ts b/apps/api/src/databases/manager.ts index 6821bda..8c02b04 100644 --- a/apps/api/src/databases/manager.ts +++ b/apps/api/src/databases/manager.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import type { Database } from '../types'; import { updateDatabaseStatus } from '../db/repo'; @@ -52,6 +53,7 @@ export const provisionDatabase = async (dbRecord: Database): Promise => { '--name', containerName, '--network', config.dockerNetwork, '--network-alias', containerName, + '-l', DEQUEL_MANAGED_LABEL, ...(dbRecord.cpuLimit ? ['--cpus', String(dbRecord.cpuLimit)] : []), ...(dbRecord.memoryLimitMb ? ['--memory', `${Math.round(dbRecord.memoryLimitMb)}m`] : []), '-v', `${volumeName}:/var/lib/${dbRecord.type === 'mysql' ? 'mysql' : 'postgresql/data'}`, diff --git a/apps/api/src/git/watcher.ts b/apps/api/src/git/watcher.ts deleted file mode 100644 index a6da875..0000000 --- a/apps/api/src/git/watcher.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { getDb } from '../db/client'; -import { listProjects } from '../db/repo'; -import { createDeployment } from '../db/repo'; -import { orchestrator } from '../orchestrator'; -import { getRemoteSha } from '../orchestrator/source'; - -const POLL_INTERVAL_MS = 60_000; - -let interval: ReturnType | null = null; - -export const startGitWatcher = () => { - if (interval) return; - interval = setInterval(poll, POLL_INTERVAL_MS); - console.log('[GitWatcher] Started (poll every 60s)'); -}; - -export const stopGitWatcher = () => { - if (interval) { clearInterval(interval); interval = null; } -}; - -async function poll() { - try { - const db = await getDb(); - const projects = db.query( - "SELECT * FROM projects WHERE repo_url IS NOT NULL AND repo_url != ''", - ).all() as any[]; - - for (const project of projects) { - const branch = project.repo_branch || 'main'; - - const latest = db.query( - 'SELECT * FROM deployments WHERE project_id = ? AND source_type = ? ORDER BY created_at DESC LIMIT 1', - ).get(project.id, 'git') as any; - - if (!latest?.commit_sha) continue; - - const remoteSha = await getRemoteSha(project.repo_url, branch); - if (!remoteSha) continue; - - if (remoteSha === latest.commit_sha) continue; - - console.log(`[GitWatcher] New commit detected for ${project.name} (${branch}): ${remoteSha.slice(0, 7)}`); - - const dep = await createDeployment({ - projectId: project.id, - sourceType: 'git', - sourceRef: project.repo_url, - branch, - commitSha: remoteSha, - }); - - orchestrator.enqueue(dep.id); - } - } catch (err) { - console.error('[GitWatcher] Poll error:', err); - } -} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 5fad630..e63a333 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,11 +9,11 @@ import { config } from './utils/config'; import { getDb } from './db/client'; import { scalingEngine } from './scaling/engine'; import { serverManager } from './servers/manager'; -import { startGitWatcher } from './git/watcher'; import { startDomainPolling } from './utils/domain-verifier'; import { alertEvaluator } from './monitoring/evaluator'; import { loadOrCreateJwtSecret } from './utils/secrets'; import { initAuth, cleanupExpiredTokens } from './utils/auth'; +import { startBuildCleanup } from './orchestrator/cleanup'; const bootstrap = async () => { await mkdir(dirname(config.databasePath), { recursive: true }); await mkdir(config.workspaceRoot, { recursive: true }); @@ -27,9 +27,9 @@ const bootstrap = async () => { orchestrator.startWorker(); scalingEngine.start(); serverManager.start(); - startGitWatcher(); startDomainPolling(); alertEvaluator.start(); + startBuildCleanup(); setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); const metrics = { diff --git a/apps/api/src/orchestrator/__tests__/cleanup.test.ts b/apps/api/src/orchestrator/__tests__/cleanup.test.ts new file mode 100644 index 0000000..0986179 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/cleanup.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +const tryRunCalls: { cmd: string; args: string[] }[] = []; +const mockTryRun = mock(async (cmd: string, args: string[]) => { + tryRunCalls.push({ cmd, args }); +}); +const mockRedisDel = mock(async () => {}); +const mockRedisLlLen = mock(async () => 0); +const mockRedisQuit = mock(async () => {}); + +beforeEach(() => { + tryRunCalls.length = 0; + mockTryRun.mockClear(); + mockRedisDel.mockClear(); + mockRedisLlLen.mockClear(); + mockRedisQuit.mockClear(); +}); + +mock.module(fileUrl('../runtime'), () => ({ + tryRun: mockTryRun, +})); + +mock.module(fileUrl('../utils/config'), () => ({ + config: { + redisUrl: 'redis://localhost:6379', + }, +})); + +mock.module(fileUrl('../utils/docker-bin'), () => ({ + dockerBin: '/usr/bin/docker', +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + llen = mockRedisLlLen; + del = mockRedisDel; + quit = mockRedisQuit; + }, +})); + +const { pruneDocker, pruneDlq } = await import('../cleanup'); + +describe('pruneDocker', () => { + beforeEach(() => { + tryRunCalls.length = 0; + }); + + it('prunes containers with dequel label filter only', async () => { + await pruneDocker(); + + const containerPrune = tryRunCalls.find( + c => c.args[0] === 'container' && c.args[1] === 'prune', + ); + expect(containerPrune).toBeDefined(); + expect(containerPrune!.args).toContain('-f'); + expect(containerPrune!.args).toContain('--filter'); + expect(containerPrune!.args.some(a => a.includes('com.dequel.managed'))).toBe(true); + }); + + it('prunes dangling images without -a flag (preserves cache)', async () => { + await pruneDocker(); + + const imagePrune = tryRunCalls.find( + c => c.args[0] === 'image' && c.args[1] === 'prune', + ); + expect(imagePrune).toBeDefined(); + expect(imagePrune!.args).not.toContain('-a'); + + const buildxPrune = tryRunCalls.find( + c => c.args[0] === 'buildx' && c.args[1] === 'prune', + ); + expect(buildxPrune).toBeDefined(); + expect(buildxPrune!.args).not.toContain('-a'); + }); +}); + +describe('pruneDlq', () => { + it('does not need a Redis connection per tick (shared connection)', async () => { + await pruneDlq(); + + expect(mockRedisLlLen).toHaveBeenCalled(); + expect(mockRedisQuit).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts new file mode 100644 index 0000000..0e8a0b9 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +let cleanupFailedDeploymentCalled = false; +let buildShouldFail = false; + +beforeEach(() => { + cleanupFailedDeploymentCalled = false; + buildShouldFail = false; +}); + +const mockDb = { + getDeploymentById: mock(() => Promise.resolve({ + id: 'dep-1', + projectId: 'proj-1', + sourceType: 'git', + sourceRef: 'https://github.com/test/repo.git', + branch: 'main', + status: 'pending', + commitSha: null, + clearCache: false, + environment: null, + imageTag: null, + containerName: null, + })), + getProjectById: mock(() => Promise.resolve({ + id: 'proj-1', + name: 'Test Project', + sourceDir: null, + port: 3000, + cpuLimit: null, + memoryLimitMb: null, + repoUrl: 'https://github.com/test/repo.git', + })), + updateDeploymentStatus: mock(() => Promise.resolve()), + updateDeploymentCommitSha: mock(() => Promise.resolve()), + appendLog: mock(async (_depId: string, stage: string, message: string) => { + if (stage === 'system' && message === 'Deployment is running') { + throw new Error('Simulated DB write failure'); + } + return { sequence: 1 }; + }), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), + listVolumes: mock(() => Promise.resolve([])), + listDeployments: mock(() => Promise.resolve([])), + listAllDatabases: mock(() => Promise.resolve([])), + deleteDeploymentAndLogs: mock(() => Promise.resolve()), + getScalingPolicy: mock(() => Promise.resolve(null)), + updateDomainValidation: mock(() => Promise.resolve()), + listDomains: mock(() => Promise.resolve([])), +}; + +mock.module(fileUrl('../../db/repo'), () => mockDb); + +mock.module(fileUrl('../runtime'), () => ({ + deployContainer: mock(() => Promise.resolve({ + containerName: 'test-project-abc12345', + liveUrl: 'http://test-project.localhost', + })), + cleanupFailedDeployment: mock(async () => { + cleanupFailedDeploymentCalled = true; + }), + ensureContainerRunning: mock(() => Promise.resolve()), + reloadCaddy: mock(() => Promise.resolve()), + tryRun: mock(() => Promise.resolve('')), +})); + +mock.module(fileUrl('../railpack'), () => ({ + buildWithRailpack: mock(async ( + _workspace: string, + _imageTag: string, + _onLog: (line: string) => Promise, + _opts?: unknown, + ) => { + if (buildShouldFail) throw new Error('Build failed'); + }), + CancelledError: class CancelledError extends Error { + constructor() { super('Cancelled'); } + }, +})); + +mock.module(fileUrl('../source'), () => ({ + prepareSourceWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + prepareUploadWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + cleanupWorkspace: mock(() => Promise.resolve()), + getHeadSha: mock(() => Promise.resolve('abc123def456')), +})); + +mock.module(fileUrl('../../utils/grafana'), () => ({ + ensureProjectDashboard: mock(() => Promise.resolve()), +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + queue: string[] = []; + rpush = mock(async (_key: string, ...values: string[]) => { this.queue.push(...values); }); + blpop = mock(async () => { const v = this.queue.shift(); return v ? ['queue', v] : null; }); + lrem = mock(async () => {}); + zadd = mock(async () => {}); + zrem = mock(async () => {}); + zrangebyscore = mock(async () => []); + quit = mock(async () => {}); + }, +})); + +const { PipelineOrchestrator } = await import('../pipeline'); + +describe('runDeployment cleanup behavior', () => { + it('does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds', async () => { + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(false); + }); + + it('calls cleanupFailedDeployment when failure occurs during build (before deployContainer)', async () => { + buildShouldFail = true; + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(true); + }); +}); diff --git a/apps/api/src/orchestrator/cleanup.ts b/apps/api/src/orchestrator/cleanup.ts new file mode 100644 index 0000000..6bf3adc --- /dev/null +++ b/apps/api/src/orchestrator/cleanup.ts @@ -0,0 +1,49 @@ +import Redis from 'ioredis'; +import { config } from '../utils/config'; +import { dockerBin } from '../utils/docker-bin'; +import { tryRun } from './runtime'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; + +const DLQ_KEY = 'dequel:deploy:dlq'; +const GC_INTERVAL_MS = 1_800_000; + +let interval: ReturnType | null = null; +let redis: Redis | null = null; + +const getRedis = () => { + if (!redis) redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + return redis; +}; + +export const pruneDocker = async () => { + await tryRun(dockerBin, ['container', 'prune', '-f', '--filter', `label=${DEQUEL_MANAGED_LABEL}`]); + await tryRun(dockerBin, ['image', 'prune', '-f']); + await tryRun(dockerBin, ['buildx', 'prune', '-f']); +}; + +export const pruneDlq = async () => { + try { + const r = getRedis(); + const size = await r.llen(DLQ_KEY); + if (size > 0) { + await r.del(DLQ_KEY); + console.log(`[Cleanup] Purged ${size} items from dead letter queue`); + } + } catch (e) { + console.warn('[Cleanup] DLQ prune failed:', e); + } +}; + +export const startBuildCleanup = () => { + if (interval) return; + console.log('[Cleanup] Docker garbage collector started (every 30min)'); + interval = setInterval(async () => { + await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e)); + await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e)); + }, GC_INTERVAL_MS); +}; + +export const stopBuildCleanup = () => { + if (interval) { clearInterval(interval); interval = null; } + if (redis) { redis.quit(); redis = null; } +}; diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 6cc4ec8..c672e3d 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -21,6 +21,7 @@ import { getHeadSha, } from "./source"; import { + cleanupFailedDeployment, deployContainer, ensureContainerRunning, reloadCaddy, @@ -158,6 +159,10 @@ export class PipelineOrchestrator { ]); } + if (deployment.imageTag) { + await tryRun("docker", ["rmi", "-f", deployment.imageTag]); + } + await deleteDeploymentAndLogs( deploymentId, ); @@ -290,6 +295,9 @@ export class PipelineOrchestrator { let workspacePath = ""; let uploadedArchivePath: string | null = null; + let imageTag: string | undefined; + let projectName: string | undefined; + let deployed = false; try { await emitLog( @@ -298,7 +306,7 @@ export class PipelineOrchestrator { "Deployment enqueued", ); - const imageTag = + imageTag = await this.resolveImageTag( deployment, ); @@ -484,7 +492,6 @@ export class PipelineOrchestrator { await this.checkCancelled(deploymentId); - let projectName: string | undefined; let cpuLimit: | number | null @@ -546,6 +553,8 @@ export class PipelineOrchestrator { }, ); + deployed = true; + if (projectName) { const dashSlug = projectName .toLowerCase() @@ -623,6 +632,23 @@ export class PipelineOrchestrator { "failed", { failureReason: message }, ); + + if (!deployed) { + await emitLog( + deploymentId, + "system", + "Cleaning up Docker resources from failed deployment", + ); + await cleanupFailedDeployment( + deploymentId, + imageTag, + projectName, + deployment.projectId, + ).catch(e => + console.warn(`[Cleanup] Failed to clean deployment ${deploymentId}:`, e), + ); + } + if (deployment.projectId) { try { const dbs = diff --git a/apps/api/src/orchestrator/queue.ts b/apps/api/src/orchestrator/queue.ts index 3fdfe51..2b8ecb3 100644 --- a/apps/api/src/orchestrator/queue.ts +++ b/apps/api/src/orchestrator/queue.ts @@ -22,12 +22,14 @@ const decodeJob = (raw: string): JobPayload | null => { } }; +const createRedis = () => new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + export class DeploymentQueue { private redis: Redis; private shuttingDown = false; constructor() { - this.redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + this.redis = createRedis(); } async enqueue(deploymentId: string) { @@ -41,7 +43,9 @@ export class DeploymentQueue { encodeJob({ id: deploymentId, attempt: i }), ); await this.redis.zrem(RETRY_KEY, ...allAttempts); - await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: 0 })); + for (let i = 0; i <= config.queueRetryMax + 1; i++) { + await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: i })); + } } async start(handler: (deploymentId: string) => Promise) { @@ -55,21 +59,26 @@ export class DeploymentQueue { } private async runWorker(workerId: number, handler: (deploymentId: string) => Promise) { - while (!this.shuttingDown) { - await this.requeueDueJobs(); + const workerRedis = createRedis(); + try { + while (!this.shuttingDown) { + await this.requeueDueJobs(); - const item = await this.redis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); - if (!item) continue; - const job = decodeJob(item[1]); - if (!job) continue; + const item = await workerRedis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); + if (!item) continue; + const job = decodeJob(item[1]); + if (!job) continue; - try { - const ok = await handler(job.id); - if (!ok) await this.retryOrDlq(job); - } catch (err) { - console.error(`[Queue] Worker ${workerId} handler error:`, err); - await this.retryOrDlq(job); + try { + const ok = await handler(job.id); + if (!ok) await this.retryOrDlq(job); + } catch (err) { + console.error(`[Queue] Worker ${workerId} handler error:`, err); + await this.retryOrDlq(job); + } } + } finally { + await workerRedis.quit(); } } diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index 8370f09..f1b82e1 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; const slugify = (s: string) => s.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 63); @@ -99,6 +100,25 @@ export const reloadCaddy = async () => { await run(dockerBin, ['exec', caddyContainer, 'caddy', 'reload', '--config', '/etc/caddy/Caddyfile']); }; +export const getContainerName = (deploymentId: string, projectName?: string, projectId?: string) => { + const slug = slugify(projectName || projectId || deploymentId); + return `${slug}-${deploymentId.slice(0, 8)}`; +}; + +export const cleanupFailedDeployment = async ( + deploymentId: string, + imageTag?: string | null, + projectName?: string, + projectId?: string, +) => { + const containerName = getContainerName(deploymentId, projectName, projectId); + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); + await tryRun(dockerBin, ['rm', '-f', containerName]); + if (imageTag) { + await tryRun(dockerBin, ['rmi', '-f', imageTag]); + } +}; + export const deployContainer = async ( deploymentId: string, imageTag: string, @@ -117,6 +137,7 @@ export const deployContainer = async ( 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, '-e', `PORT=${opts.appPort ?? config.appInternalPort}`, ]; diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 16e544b..afdc173 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import Redis from 'ioredis'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import { run, tryRun } from './docker-utils'; import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy } from '../db/repo'; @@ -201,6 +202,7 @@ class ScalingEngine { await run(dockerBin, [ 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, ...volumes, ...envVars, imageTag, ]); diff --git a/apps/api/src/utils/dequel-labels.ts b/apps/api/src/utils/dequel-labels.ts new file mode 100644 index 0000000..cac7a70 --- /dev/null +++ b/apps/api/src/utils/dequel-labels.ts @@ -0,0 +1 @@ +export const DEQUEL_MANAGED_LABEL = 'com.dequel.managed=true'; From b9d0a965f893789819e22dcd47d482922ed73091 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:14:29 +0100 Subject: [PATCH 6/8] feat(web): add mobile-responsive navigation and layouts - Implement mobile sidebar toggle - Make dashboard tables and tabs scrollable on small screens - Set restart policies for docker services - Improve CLI installation and path resolution --- apps/web/src/components/Layout.tsx | 14 ++- apps/web/src/components/layout/Header.tsx | 14 ++- .../components/layout/NotificationBanner.tsx | 2 +- apps/web/src/components/layout/Sidebar.tsx | 95 ++++++++++++------- .../deployments/deployment-history.tsx | 4 +- .../components/project/domains/DomainsTab.tsx | 4 +- .../components/project/envtab/EnvVarTable.tsx | 4 +- .../components/project/volumes/VolumesTab.tsx | 4 +- apps/web/src/routes/ProjectDetail.tsx | 20 ++-- apps/web/src/routes/Settings.tsx | 10 +- docker-compose.yml | 11 +++ scripts/dequel | 12 ++- scripts/install.sh | 13 +++ 13 files changed, 143 insertions(+), 64 deletions(-) diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index d47c4a9..29c4c82 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -33,6 +33,11 @@ export function Layout({ children }: { children: React.ReactNode }) { const { data: projects = [] } = useProjects({ enabled: !!me?.authenticated && location.pathname !== "/login" }); const [projectSelectorOpen, setProjectSelectorOpen] = useState(false); + const [sidebarOpen, setSidebarOpen] = useState(false); + + useEffect(() => { + setSidebarOpen(false); + }, [location.pathname, location.search]); const { data: metricsText } = useQuery({ queryKey: ["metrics"], @@ -100,7 +105,7 @@ export function Layout({ children }: { children: React.ReactNode }) { } return ( -
+
-
+
setNotification(null)} /> -
{children}
+
{children}
); diff --git a/apps/web/src/components/layout/Header.tsx b/apps/web/src/components/layout/Header.tsx index 7b686c5..cce94ce 100644 --- a/apps/web/src/components/layout/Header.tsx +++ b/apps/web/src/components/layout/Header.tsx @@ -1,19 +1,29 @@ import { Link } from "@tanstack/react-router"; -import { ChevronRight } from "lucide-react"; +import { ChevronRight, Menu } from "lucide-react"; interface HeaderProps { currentProject: { name: string } | undefined; currentProjectId: string | null; location: { pathname: string; search: any }; + setSidebarOpen: (open: boolean) => void; } export function Header({ currentProject, currentProjectId, location, + setSidebarOpen, }: HeaderProps) { return ( -
+
+ + Workspace diff --git a/apps/web/src/components/layout/NotificationBanner.tsx b/apps/web/src/components/layout/NotificationBanner.tsx index 35755d6..cb1c6ec 100644 --- a/apps/web/src/components/layout/NotificationBanner.tsx +++ b/apps/web/src/components/layout/NotificationBanner.tsx @@ -18,7 +18,7 @@ export function NotificationBanner({ const Icon = notification.type === "success" ? CheckCircle2 : AlertCircle; return ( -
+
void; + sidebarOpen: boolean; + setSidebarOpen: (open: boolean) => void; } export function Sidebar({ @@ -29,43 +33,70 @@ export function Sidebar({ metrics, location, navigate, + sidebarOpen, + setSidebarOpen, }: SidebarProps) { return ( - + ); } diff --git a/apps/web/src/components/project/deployments/deployment-history.tsx b/apps/web/src/components/project/deployments/deployment-history.tsx index 49afcf0..d3e8d3c 100644 --- a/apps/web/src/components/project/deployments/deployment-history.tsx +++ b/apps/web/src/components/project/deployments/deployment-history.tsx @@ -51,8 +51,8 @@ export function DeploymentHistory({ return ( <> -
-
+
+
Status diff --git a/apps/web/src/components/project/domains/DomainsTab.tsx b/apps/web/src/components/project/domains/DomainsTab.tsx index 3cef41e..c2ab828 100644 --- a/apps/web/src/components/project/domains/DomainsTab.tsx +++ b/apps/web/src/components/project/domains/DomainsTab.tsx @@ -366,8 +366,8 @@ export function DomainsTab({ )} -
-
+
+
diff --git a/apps/web/src/components/project/envtab/EnvVarTable.tsx b/apps/web/src/components/project/envtab/EnvVarTable.tsx index f6c4314..2c5dd58 100644 --- a/apps/web/src/components/project/envtab/EnvVarTable.tsx +++ b/apps/web/src/components/project/envtab/EnvVarTable.tsx @@ -77,8 +77,8 @@ export function EnvVarTable({ -
-
+
+
diff --git a/apps/web/src/components/project/volumes/VolumesTab.tsx b/apps/web/src/components/project/volumes/VolumesTab.tsx index d269a75..f67d4d4 100644 --- a/apps/web/src/components/project/volumes/VolumesTab.tsx +++ b/apps/web/src/components/project/volumes/VolumesTab.tsx @@ -233,8 +233,8 @@ export function VolumesTab({ -
-
+
+
diff --git a/apps/web/src/routes/ProjectDetail.tsx b/apps/web/src/routes/ProjectDetail.tsx index f1e69db..ee878f3 100644 --- a/apps/web/src/routes/ProjectDetail.tsx +++ b/apps/web/src/routes/ProjectDetail.tsx @@ -106,32 +106,32 @@ export function ProjectDetail({ navigate({ search: { tab: val } as any })} className="space-y-4"> - - + + Deployments - + Env Vars - + Volumes - + Databases - + Domains - + Scaling - + Alerts - + Observability - + Logs diff --git a/apps/web/src/routes/Settings.tsx b/apps/web/src/routes/Settings.tsx index fb16b15..466b44c 100644 --- a/apps/web/src/routes/Settings.tsx +++ b/apps/web/src/routes/Settings.tsx @@ -72,8 +72,8 @@ function ServersSection() { {servers.length > 0 && ( -
-
+
+
NameHostStatus @@ -322,7 +322,7 @@ function ApiKeysSection() { {newKey} )} - +
setName(e.target.value)} className="w-56" /> @@ -330,8 +330,8 @@ function ApiKeysSection() { {apiKeys.length > 0 && ( -
-
+
+
NameKey HashCreated diff --git a/docker-compose.yml b/docker-compose.yml index f63cf1d..0f22018 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: buildkit: image: moby/buildkit:latest + restart: unless-stopped command: [ "--addr", @@ -28,6 +29,7 @@ services: api: # image: ghcr.io/lftobs/dequel/api:latest + restart: unless-stopped build: context: . dockerfile: ./apps/api/Dockerfile @@ -73,6 +75,7 @@ services: pam-auth: image: python:3-slim + restart: unless-stopped entrypoint: [] command: - sh @@ -103,6 +106,7 @@ services: web: # image: ghcr.io/lftobs/dequel/web:latest + restart: unless-stopped build: context: ./apps/web healthcheck: @@ -124,6 +128,7 @@ services: caddy: image: caddy:2.8-alpine + restart: unless-stopped command: [ "caddy", @@ -153,6 +158,7 @@ services: redis: image: redis:7-alpine + restart: unless-stopped command: redis-server --appendonly yes volumes: - redis-data:/data @@ -168,6 +174,7 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest + restart: unless-stopped volumes: - /:/rootfs:ro - /var/run:/var/run:ro @@ -197,6 +204,7 @@ services: prometheus: image: prom/prometheus:latest + restart: unless-stopped stop_grace_period: 60s entrypoint: [] command: @@ -229,6 +237,7 @@ services: loki: image: grafana/loki:3.0.0 + restart: unless-stopped command: - -config.file=/etc/loki/loki-config.yml volumes: @@ -254,6 +263,7 @@ services: promtail: image: grafana/promtail:3.0.0 + restart: unless-stopped command: - -config.file=/etc/promtail/promtail-config.yml volumes: @@ -269,6 +279,7 @@ services: grafana: image: grafana/grafana:latest + restart: unless-stopped environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin diff --git a/scripts/dequel b/scripts/dequel index 811a722..3e67244 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -DEQUEL_HOME="${DEQUEL_HOME:-$HOME/.dequel}" +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" +DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" VERSION="0.1.0" @@ -27,7 +28,12 @@ cmd() { cmd_start() { header "Starting Dequel" cmd up -d - success "Dequel is running at http://localhost" + local domain="${CADDY_BASE_DOMAIN:-localhost}" + if [ "$domain" = "localhost" ]; then + success "Dequel is running at http://localhost" + else + success "Dequel is running at https://$domain" + fi } cmd_stop() { @@ -130,7 +136,7 @@ cmd_help() { echo " --help Show this help" echo "" echo "Environment:" - echo " DEQUEL_HOME Config directory (default: \$HOME/.dequel)" + echo " DEQUEL_HOME Config directory (default: auto-detected from install path)" } case "${1:-}" in diff --git a/scripts/install.sh b/scripts/install.sh index dd9508a..d40e504 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -40,6 +40,19 @@ check_prerequisites() { fail "Docker is not installed. See https://docs.docker.com/engine/install/" fi + if docker info >/dev/null 2>&1; then + success "Docker socket accessible" + else + warn "Docker socket not accessible — attempting to add user to docker group" + if sudo usermod -aG docker "$USER" 2>/dev/null; then + success "Added $USER to docker group" + warn "Log out and back in (or run 'newgrp docker') for the change to take effect" + else + warn "Could not add user to docker group automatically" + warn "Run this manually: sudo usermod -aG docker $USER && newgrp docker" + fi + fi + if docker compose version >/dev/null 2>&1; then success "Docker Compose found: $(docker compose version)" COMPOSE_CMD="docker compose" From a3037ad6d92723c9eb0df84c2d873205daa93d93 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:28:49 +0100 Subject: [PATCH 7/8] feat(api): simplify repository fetching to owner affiliation --- apps/api/src/api/github/index.ts | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index b2b6e83..f295140 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -151,24 +151,9 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 401; return { error: "Not authenticated" }; } - const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&type=all", token); - const user = await fetchGitHub("/user", token) as { login: string }; - const orgs = await fetchGitHub("/user/orgs", token) as { login: string }[]; + const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&affiliation=owner", token); const allRepos = Array.isArray(repos) ? repos : []; - const orgRepos: any[] = []; - for (const org of Array.isArray(orgs) ? orgs : []) { - try { - const orgR = await fetchGitHub(`/orgs/${org.login}/repos?per_page=100&sort=updated&type=all`, token); - if (Array.isArray(orgR)) orgRepos.push(...orgR); - } catch {} - } - const seen = new Set(); - const merged = [...allRepos, ...orgRepos].filter((r) => { - if (seen.has(r.id)) return false; - seen.add(r.id); - return true; - }); - return merged.map((r: any) => ({ + return allRepos.map((r: any) => ({ id: r.id, name: r.name, fullName: r.full_name, From 875eabd186ee2e68f2821598e739fb9dd1398bfa Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:36:33 +0100 Subject: [PATCH 8/8] feat(scripts): implement update cmd for dequel --- scripts/dequel | 78 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/scripts/dequel b/scripts/dequel index 3e67244..fc3942d 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,10 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" -DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" +SCRIPT_PATH="$(readlink -f "$0" 2>/dev/null || realpath "$0" 2>/dev/null || echo "$0")" +SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" +DEQUEL_HOME="${DEQUEL_HOME:-$SCRIPT_DIR}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" -VERSION="0.1.0" +INSTALLED_VERSION="$(cat "$DEQUEL_HOME/VERSION" 2>/dev/null || echo "0.1.0")" BOLD='\033[1m' DIM='\033[2m' @@ -59,9 +60,74 @@ cmd_logs() { cmd_update() { header "Updating Dequel" + + local repo="Lftobs/dequel" + local tag="" + local base_url="" + + info "Checking for latest release..." + tag=$(curl -fsSL "https://api.github.com/repos/$repo/releases/latest" \ + | grep '"tag_name"' | head -1 | sed -E 's/.*"([^"]+)".*/\1/') || true + + if [ -z "$tag" ]; then + warn "Could not determine latest release, using main branch" + base_url="https://raw.githubusercontent.com/$repo/main" + else + base_url="https://raw.githubusercontent.com/$repo/$tag" + success "Latest release: $tag" + fi + + local tmp_dir + tmp_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$tmp_dir'" EXIT + + info "Downloading updated configuration..." + curl -fsSL "$base_url/docker-compose.yml" -o "$tmp_dir/docker-compose.yml" + curl -fsSL "$base_url/infra/caddy/Caddyfile" -o "$tmp_dir/Caddyfile" + curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" + curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/$f" + done + + for f in loki.yml prometheus.yml; do + curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/$f" + done + + for f in dashboards.yml deployed-apps.json; do + curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/$f" + done + + mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" + mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" + mkdir -p "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ + "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/$f" + done + for f in loki.yml prometheus.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" + done + for f in dashboards.yml deployed-apps.json; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" + done + + mv "$tmp_dir/VERSION" "$DEQUEL_HOME/VERSION" + chmod +x "$tmp_dir/dequel" + mv "$tmp_dir/dequel" "$SCRIPT_PATH" + + success "Configuration updated" + + header "Pulling Docker images" cmd pull + + header "Recreating services" cmd up -d - success "Dequel updated." + + success "Dequel updated to $tag" } cmd_uninstall() { @@ -116,7 +182,7 @@ cmd_uninstall() { } cmd_version() { - echo "dequel v$VERSION" + echo "dequel v$INSTALLED_VERSION" } cmd_help() { @@ -130,7 +196,7 @@ cmd_help() { echo " restart Restart all Dequel services" echo " status Show service status" echo " logs Follow service logs" - echo " update Pull latest images and recreate services" + echo " update Download latest config, pull images, and recreate services" echo " uninstall Remove Dequel completely (config, images, volumes)" echo " --version Show version" echo " --help Show this help"