Checkmate Improvements

client/package.json

@@ -66,6 +66,6 @@
 		"lint-staged": "^16.2.7",
 		"prettier": "^3.3.3",
 		"typescript": "5.9.3",
-		"vite": "6.4.1"
+		"vite": "^6.4.2"
 	}
 }

client/src/Hooks/useMonitorForm.ts

@@ -12,6 +12,9 @@ const getBaseDefaults = (data?: Monitor | null) => ({
 	description: data?.description || "",
 	interval: data?.interval || 60000,
 	notifications: data?.notifications || [],
+	escalationEnabled: !!data?.escalation,
+	escalationDelay: data?.escalation?.delayMinutes ?? 1,
+	escalationChannelId: data?.escalation?.channelId ?? "",
 	statusWindowSize: data?.statusWindowSize || 5,
 	statusWindowThreshold: data?.statusWindowThreshold || 60,
 	geoCheckEnabled: data?.geoCheckEnabled ?? false,

client/src/Pages/CreateMonitor/index.tsx

@@ -251,12 +251,31 @@ const CreateMonitorPage = () => {
 		setIsDeleteDialogOpen(false);
 	};
 
-	const onSubmit = async (data: MonitorFormData) => {
+	const onSubmit = async (data: any) => {
+		const {
+			escalationEnabled,
+			escalationDelay,
+			escalationChannelId,
+			...rest
+		} = data;
+
+		const payload = {
+			...rest,
+			escalation: escalationEnabled
+				? {
+						delayMinutes: Number(escalationDelay),
+						channelId: escalationChannelId,
+					}
+				: null,
+		};
+
+		console.log("FINAL PAYLOAD", payload);
+
 		let result;
 		if (isEditMode && monitorId) {
-			result = await patch(`/monitors/${monitorId}`, data);
+			result = await patch(`/monitors/${monitorId}`, payload);
 		} else {
-			result = await post("/monitors", data);
+			result = await post("/monitors", payload);
 		}
 
 		if (result?.success) {
@@ -765,6 +784,70 @@ const CreateMonitorPage = () => {
 				}
 			/>
 
+			<ConfigBox
+				title="Escalation"
+				subtitle="Send additional alert if issue persists"
+				rightContent={
+					<Stack spacing={theme.spacing(LAYOUT.MD)}>
+
+						<Controller
+							name="escalationEnabled"
+							control={control}
+							render={({ field }) => (
+								<Stack direction="row" alignItems="center" spacing={2}>
+									<Switch
+										checked={field.value ?? false}
+										onChange={(e) => field.onChange(e.target.checked)}
+									/>
+									<Typography>Enable Escalation</Typography>
+								</Stack>
+							)}
+						/>
+
+						<Controller
+							name="escalationDelay"
+							control={control}
+							render={({ field }) => (
+								<TextField
+									{...field}
+									type="number"
+									fieldLabel="Delay (minutes)"
+									placeholder="Enter delay time"
+									fullWidth
+								/>
+							)}
+						/>
+
+						<Controller
+							name="escalationChannelId"
+							control={control}
+							render={({ field }) => {
+								const notificationOptions = (notifications ?? []).map((n) => ({
+									...n,
+									name: n.notificationName,
+								}));
+
+								return (
+									<Select
+										{...field}
+										value={field.value ?? ""}
+										fieldLabel="Escalation Channel"
+									>
+										<MenuItem value="">Select channel</MenuItem>
+										{notificationOptions.map((n) => (
+											<MenuItem key={n.id} value={n.id}>
+												{n.notificationName}
+											</MenuItem>
+										))}
+									</Select>
+								);
+							}}
+						/>
+
+					</Stack>
+				}
+			/>
+
 			{(watchedType === "http" ||
 				watchedType === "grpc" ||
 				watchedType === "websocket") && (

client/src/Types/Monitor.ts

@@ -38,6 +38,11 @@ export type MonitorStatus = (typeof MonitorStatuses)[number];
 
 export type MonitorMatchMethod = "equal" | "include" | "regex" | "";
 
+export type MonitorEscalation = {
+	delayMinutes: number;
+	channelId: string;
+};
+
 export interface Monitor {
 	id: string;
 	userId: string;
@@ -60,6 +65,7 @@ export interface Monitor {
 	interval: number;
 	uptimePercentage?: number;
 	notifications: string[];
+	escalation?: MonitorEscalation | null;
 	secret?: string;
 	cpuAlertThreshold: number;
 	cpuAlertCounter: number;

client/src/Validation/monitor.ts

@@ -13,6 +13,12 @@ const baseSchema = z.object({
 	description: z.string().optional(),
 	interval: z.number().min(15000, "Interval must be at least 15 seconds"),
 	notifications: z.array(z.string()),
+	escalationEnabled: z.boolean().optional(),
+	escalationDelay: z.coerce
+		.number()
+		.min(1, "Escalation delay must be at least 1 minute")
+		.optional(),
+	escalationChannelId: z.string().optional(),
 	statusWindowSize: z
 		.number({ message: "Status window size is required" })
 		.min(1, "Status window size must be at least 1")

server/src/db/models/Monitor.ts

@@ -284,6 +284,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
 				ref: "Notification",
 			},
 		],
+		escalation: {
+			delayMinutes: { type: Number },
+			channelId: { type: Schema.Types.ObjectId, ref: "Notification" },
+		},
 		secret: {
 			type: String,
 		},

server/src/repositories/monitors/MongoMonitorsRepository.ts

@@ -8,7 +8,18 @@ import { AppError } from "@/utils/AppError.js";
 
 class MongoMonitorsRepository implements IMonitorsRepository {
 	create = async (monitor: Monitor, teamId: string, userId: string) => {
-		const monitorModel = new MonitorModel({ ...monitor, teamId, userId });
+		const monitorModel = new MonitorModel({
+			...monitor,
+			teamId,
+			userId,
+			escalation: monitor.escalation
+				? {
+						delayMinutes: monitor.escalation.delayMinutes,
+						channelId: new mongoose.Types.ObjectId(monitor.escalation.channelId),
+					}
+				: null,
+		});
+
 		const saved = await monitorModel.save();
 		return this.toEntity(saved);
 	};
@@ -167,15 +178,27 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 	};
 
 	updateById = async (monitorId: string, teamId: string, patch: Partial<Monitor>) => {
+		const updatePayload = {
+			...patch,
+			escalation:
+				patch.escalation === null
+					? null
+					: patch.escalation
+						? {
+								delayMinutes: patch.escalation.delayMinutes,
+								channelId: new mongoose.Types.ObjectId(patch.escalation.channelId),
+							}
+						: undefined,
+		};
+
 		const updatedMonitor = await MonitorModel.findOneAndUpdate(
 			{ _id: monitorId, teamId },
 			{
-				$set: {
-					...patch,
-				},
+				$set: updatePayload,
 			},
 			{ new: true, runValidators: true }
 		);
+
 		if (!updatedMonitor) {
 			throw new AppError({ message: `Failed to update monitor with id ${monitorId}`, status: 500 });
 		}
@@ -374,6 +397,12 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			interval: doc.interval,
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
+			escalation: doc.escalation
+				? {
+						delayMinutes: doc.escalation.delayMinutes,
+						channelId: toStringId(doc.escalation.channelId),
+					}
+				: null,
 			secret: doc.secret ?? undefined,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
 			cpuAlertCounter: doc.cpuAlertCounter,
@@ -433,6 +462,12 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			interval: doc.interval,
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
+			escalation: doc.escalation
+				? {
+						delayMinutes: doc.escalation.delayMinutes,
+						channelId: toStringId(doc.escalation.channelId),
+					}
+				: null,
 			secret: doc.secret ?? undefined,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
 			cpuAlertCounter: doc.cpuAlertCounter,

server/src/service/business/monitorService.ts

@@ -166,6 +166,8 @@ export class MonitorService implements IMonitorService {
 	};
 
 	createMonitor = async (teamId: string, userId: string, body: Monitor): Promise<void> => {
+		console.log("CREATE BODY", JSON.stringify(body, null, 2));
+
 		const monitor = await this.monitorsRepository.create(body, teamId, userId);
 		if (!monitor) {
 			throw new AppError({ message: "Failed to create monitor", status: 500, service: SERVICE_NAME, method: "createMonitor" });
@@ -437,6 +439,8 @@ export class MonitorService implements IMonitorService {
 	};
 
 	editMonitor = async ({ teamId, monitorId, body }: { teamId: string; monitorId: string; body: Partial<Monitor> }) => {
+		console.log("EDIT BODY", JSON.stringify(body, null, 2));
+
 		const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body);
 		await this.jobQueue.updateJob(editedMonitor);
 		return editedMonitor;

server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts

@@ -158,14 +158,19 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
 
 				// Step 6. Handle notifications (best effort, continue even in event of failure, don't wait)
 				if (decision.shouldSendNotification) {
-					this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: unknown) => {
-						this.logger.error({
-							message: `Error sending notifications for job ${statusChangeResult.monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`,
-							service: SERVICE_NAME,
-							method: "getMonitorJob",
-							stack: error instanceof Error ? error.stack : undefined,
+					this.notificationsService
+						.handleNotifications(statusChangeResult.monitor, status, decision)
+						.catch((error: unknown) => {
+							this.logger.error({
+								message: `Error sending notifications for job ${statusChangeResult.monitor.id}: ${
+									error instanceof Error ? error.message : "Unknown error"
+								}`,
+								service: SERVICE_NAME,
+								method: "getMonitorJob",
+							});
 						});
-					});
+
+					this.scheduleEscalation(statusChangeResult.monitor);
 				}
 
 				// Step 7. Handle incidents (best effort, don't wait)
@@ -455,4 +460,39 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
 
 		return decision;
 	}
+
+	private scheduleEscalation(monitor: Monitor) {
+		if (!monitor.escalation?.channelId || !monitor.escalation?.delayMinutes) return;
+
+		setTimeout(async () => {
+			try {
+				const freshMonitor = await this.monitorsRepository.findById(monitor.id, monitor.teamId);
+
+				const stillDown =
+					freshMonitor.status === "down" || freshMonitor.status === "breached";
+
+				if (!stillDown) return;
+
+				const escalationMonitor: Monitor = {
+					...freshMonitor,
+					notifications: [monitor.escalation!.channelId],
+				};
+
+				await this.notificationsService.handleNotifications(
+					escalationMonitor,
+					{} as any,
+					{
+						shouldSendNotification: true,
+						notificationReason: "status_change",
+					} as any
+				);
+			} catch (err) {
+				this.logger.error({
+					message: "Escalation failed",
+					service: SERVICE_NAME,
+					method: "scheduleEscalation",
+				});
+			}
+		}, monitor.escalation.delayMinutes * 60 * 1000);
+	}
 }

server/src/service/infrastructure/SuperSimpleQueue/chat.json

@@ -0,0 +1,5 @@
+{
+  "responderUsername": "GitHub Copilot",
+  "initialLocation": "panel",
+  "requests": []
+}

server/src/types/monitor.ts

@@ -15,6 +15,11 @@ export type MonitorStatus = (typeof MonitorStatuses)[number];
 export const MonitorMatchMethods = ["equal", "include", "regex"] as const;
 export type MonitorMatchMethod = (typeof MonitorMatchMethods)[number] | "";
 
+export type MonitorEscalation = {
+	delayMinutes: number;
+	channelId: string;
+};
+
 export interface Monitor {
 	id: string;
 	userId: string;
@@ -37,6 +42,7 @@ export interface Monitor {
 	interval: number;
 	uptimePercentage?: number;
 	notifications: string[];
+	escalation?: MonitorEscalation | null;
 	secret?: string;
 	cpuAlertThreshold: number;
 	cpuAlertCounter: number;

server/src/validation/monitorValidation.ts

@@ -67,6 +67,7 @@ export const createMonitorBodyValidation = z.object({
 	diskAlertThreshold: z.number().optional(),
 	tempAlertThreshold: z.number().optional(),
 	notifications: z.array(z.string()).optional(),
+	escalation: z.object({delayMinutes: z.number().min(1),channelId: z.string().min(1),}).nullable().optional(),
 	secret: z.string().optional(),
 	jsonPath: z.union([z.string(), z.literal("")]).optional(),
 	expectedValue: z.union([z.string(), z.literal("")]).optional(),
@@ -89,6 +90,7 @@ export const editMonitorBodyValidation = z.object({
 	description: z.union([z.string(), z.literal("")]).optional(),
 	interval: z.number().optional(),
 	notifications: z.array(z.string()).optional(),
+	escalation: z.object({delayMinutes: z.number().min(1),channelId: z.string().min(1),}).nullable().optional(),
 	secret: z.string().optional(),
 	ignoreTlsErrors: z.boolean().optional(),
 	useAdvancedMatching: z.boolean().optional(),