Created Escalate Notifications using GitHub Copilot

client/src/Hooks/useMonitorForm.ts

@@ -17,6 +17,7 @@ const getBaseDefaults = (data?: Monitor | null) => ({
 	geoCheckEnabled: data?.geoCheckEnabled ?? false,
 	geoCheckLocations: data?.geoCheckLocations || [],
 	geoCheckInterval: data?.geoCheckInterval || 300000,
+	escalation: data?.escalation,
 });
 
 export const useMonitorForm = ({

client/src/Pages/CreateMonitor/index.tsx

@@ -40,6 +40,8 @@ import {
 } from "@/Types/Monitor";
 import type { Notification } from "@/Types/Notification";
 import type { MonitorFormData } from "@/Validation/monitor";
+import { parse } from "zod";
+import { es } from "zod/v4/locales";
 
 interface GeneralSettingsConfig {
 	urlLabel: string;
@@ -252,11 +254,16 @@ const CreateMonitorPage = () => {
 	};
 
 	const onSubmit = async (data: MonitorFormData) => {
+		const escalation = data.escalation && data.escalation.escalationDelay > 0 && data.escalation.channelID ? data.escalation : undefined;
+		const payload = {
+			...data,
+			escalation,
+		}
 		let result;
 		if (isEditMode && monitorId) {
-			result = await patch(`/monitors/${monitorId}`, data);
+			result = await patch(`/monitors/${monitorId}`, payload);
 		} else {
-			result = await post("/monitors", data);
+			result = await post("/monitors", payload);
 		}
 
 		if (result?.success) {
@@ -765,6 +772,69 @@ const CreateMonitorPage = () => {
 				}
 			/>
 
+			<ConfigBox
+				title={t("pages.createMonitor.form.escalation.title")}
+				subtitle={t("pages.createMonitor.form.escalation.description")}
+				rightContent={
+					<Controller
+						name="escalation"
+						control={control}
+						render={({ field }) => {
+							const notify = (notifications ?? []).map((n) => ({
+								...n,
+								name: n.notificationName,
+							}));
+							const currEscalation = field.value;
+							const selectedChannel = notify.find(
+								(n) => n.id === currEscalation?.channelID
+							);
+							return (
+								<Stack spacing={theme.spacing(LAYOUT.MD)}>
+									<TextField
+										type="number"
+										fieldLabel={t("pages.createMonitor.form.escalation.escalationDelay.label")}
+										inputProps={{ min: 1 }}
+										placeholder={t("pages.createMonitor.form.escalation.escalationDelay.placeholder")}
+										value={currEscalation?.escalationDelay ?? ""}
+										onChange={(e) => {
+											const escalationDelay = parseInt(e.target.value, 10) || 0;
+											field.onChange({
+												escalationDelay,
+												channelID: currEscalation?.channelID || "",
+											});
+										}}
+										fullWidth
+									/>
+									<Select
+										value={currEscalation?.channelID ?? ""}
+										fieldLabel={t("pages.createMonitor.form.escalation.channelID.label")}
+										onChange={(e) => {
+											const channelID = e.target.value;
+											field.onChange({
+												escalationDelay: currEscalation?.escalationDelay || 0,
+												channelID: e.target.value,
+											});
+										}}
+									>
+										<MenuItem value="">
+											{t("pages.createMonitor.form.escalation.channelID.placeholder")}
+										</MenuItem>
+										{notify.map((notification) => (
+											<MenuItem
+												key={notification.id}
+												value={notification.id}
+											>
+												{notification.notificationName}
+											</MenuItem>
+										))}
+									</Select>
+								</Stack>
+							);
+						}}
+					/>
+				}
+			/>
+
 			{(watchedType === "http" ||
 				watchedType === "grpc" ||
 				watchedType === "websocket") && (

client/src/Types/Monitor.ts

@@ -76,6 +76,10 @@ export interface Monitor {
 	geoCheckEnabled?: boolean;
 	geoCheckLocations?: GeoContinent[];
 	geoCheckInterval?: number;
+	escalation?: {
+		escalationDelay: number;
+		channelID: string;
+	};
 	recentChecks: CheckSnapshot[];
 	createdAt: string;
 	updatedAt: string;

client/src/Validation/monitor.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { GeoContinents } from "@/Types/GeoCheck";
+import { es } from "zod/v4/locales";
 
 // URL schema with custom error message
 const urlSchema = z.url({ message: "Please enter a valid URL" });
@@ -27,6 +28,11 @@ const baseSchema = z.object({
 		.number()
 		.min(300000, "Interval must be at least 5 minutes")
 		.optional(),
+		escalation: z.object({
+			escalationDelay: z.number().min(0),
+			channelID: z.string().min(1),})
+		.nullable()
+		.optional(),
 });
 
 // HTTP monitor schema

client/src/locales/en.json

@@ -543,6 +543,18 @@
 					"description": "Select the notification channels you want to use",
 					"title": "Notifications"
 				},
+				"escalation":{
+					"description": "If the monitor stays down for the specified time, notify additional channels.",
+					"title": "Escalation Rules",
+					"escalationDelay": {
+						"label": "Escalate after (minutes)",
+						"placeholder": "e.g. 3"
+					},
+					"channelID": {
+						"label": "Escalation notification channels",
+						"placeholder": "Type to search"
+					}
+				},
 				"type": {
 					"description": "Select the type of check to perform",
 					"optionDockerDescription": "Use Docker to monitor if a container is running.",

server/src/db/models/Incident.ts

@@ -1,12 +1,13 @@
 import { Schema, model, type Types } from "mongoose";
 import { IncidentResolutionTypes, type Incident } from "@/types/incident.js";
 
-type IncidentDocumentBase = Omit<Incident, "id" | "monitorId" | "teamId" | "resolvedBy" | "startTime" | "endTime" | "createdAt" | "updatedAt"> & {
+type IncidentDocumentBase = Omit<Incident, "id" | "monitorId" | "teamId" | "resolvedBy" | "startTime" | "escalationTime" | "endTime" | "createdAt" | "updatedAt"> & {
 	monitorId: Types.ObjectId;
 	teamId: Types.ObjectId;
 	resolvedBy?: Types.ObjectId | null;
 	startTime: Date;
 	endTime: Date | null;
+	escalationTime: Date | null;
 	createdAt: Date;
 	updatedAt: Date;
 };
@@ -72,6 +73,10 @@ const IncidentSchema = new Schema<IncidentDocument>(
 			type: String,
 			default: null,
 		},
+		escalationTime:{
+			type: Date,
+			default: null,
+		}
 	},
 	{ timestamps: true }
 );

server/src/db/models/Monitor.ts

@@ -351,6 +351,16 @@ const MonitorSchema = new Schema<MonitorDocument>(
 			type: Number,
 			default: 300000,
 		},
+		escalation: {
+			escalationDelay:{
+				type: Number,
+				required: true,
+			},
+			channelID:{
+				type: String,
+				required: true,
+			},
+		},
 		recentChecks: {
 			type: [checkSnapshotSchema],
 			default: [],

server/src/repositories/monitors/MongoMonitorsRepository.ts

@@ -374,6 +374,12 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			interval: doc.interval,
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
+			escalation: doc.escalation
+				? {
+						escalationDelay: doc.escalation.escalationDelay,
+						channelID: toStringId(doc.escalation.channelID),
+					}
+				: undefined,
 			secret: doc.secret ?? undefined,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
 			cpuAlertCounter: doc.cpuAlertCounter,
@@ -433,6 +439,12 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			interval: doc.interval,
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
+			escalation: doc.escalation
+				? {
+						escalationDelay: doc.escalation.escalationDelay,
+						channelID: toStringId(doc.escalation.channelID),
+					}
+				: undefined,
 			secret: doc.secret ?? undefined,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
 			cpuAlertCounter: doc.cpuAlertCounter,

server/src/service/business/incidentService.ts

@@ -64,7 +64,7 @@ export class IncidentService implements IIncidentService {
 		decision: MonitorActionDecision,
 		monitorStatusResponse?: MonitorStatusResponse
 	): Promise<Incident | null> => {
-		if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) {
+		if (!decision.shouldCreateIncident && !decision.shouldResolveIncident && !decision.shouldEscalateIncident) {
 			return null;
 		}
 
@@ -105,6 +105,11 @@ export class IncidentService implements IIncidentService {
 			return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
 		}
 
+		// Handle incident escalation
+		if(decision.shouldEscalateIncident && activeIncident && !activeIncident.escalationTime) {
+			activeIncident.escalationTime = new Date().toISOString();
+			return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
+		}
 		return null;
 	};
 

server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts

@@ -45,6 +45,8 @@ export interface MonitorActionDecision {
 		disk?: boolean;
 		temp?: boolean;
 	};
+	shouldEscalateIncident?: boolean;
+	escalationChannelID?: string | null;
 }
 
 export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
@@ -156,8 +158,28 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
 				// Step 5.  Get decisions
 				const decision = this.evaluateMonitorAction(statusChangeResult);
 
+				if(statusChangeResult.monitor.escalation && statusChangeResult.monitor.status === "down"){
+					const activeIncident = await this.incidentsRepository.findActiveByMonitorId(monitorId, teamId);
+					if(activeIncident){
+						const incidentStart = new Date(activeIncident.createdAt).getTime();
+						const nowNow = new Date().getTime();
+						const incidentDuration = nowNow - incidentStart;
+						const escalationDelay = statusChangeResult.monitor.escalation.escalationDelay * 60 * 1000; // Convert minutes to milliseconds
+
+						if (incidentDuration >= escalationDelay && !activeIncident.escalationTime) {
+							decision.shouldEscalateIncident = true;
+							decision.escalationChannelID = statusChangeResult.monitor.escalation.channelID;
+							this.logger.debug({
+								message: `Incident for monitor ${monitorId} is eligible for escalation`,
+								service: SERVICE_NAME,
+								method: "getHeartbeatJob",
+							});
+						}
+					}
+				}
+
 				// Step 6. Handle notifications (best effort, continue even in event of failure, don't wait)
-				if (decision.shouldSendNotification) {
+				if (decision.shouldSendNotification || decision.shouldEscalateIncident) {
 					this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: unknown) => {
 						this.logger.error({
 							message: `Error sending notifications for job ${statusChangeResult.monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`,

server/src/service/infrastructure/notificationMessageBuilder.ts

@@ -33,6 +33,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
 		const severity = this.determineSeverity(type);
 		const content = this.buildContent(type, monitor, monitorStatusResponse);
 
+		if (decision.shouldEscalateIncident) {
+			content.title = `[ESCALATION] ${content.title}`;
+			content.summary = `[ESCALATION] ${content.summary}`;
+		}
 		return {
 			type,
 			severity,

server/src/service/infrastructure/notificationsService.ts

@@ -132,13 +132,68 @@ export class NotificationsService implements INotificationsService {
 		return succeeded === notifications.length;
 	};
 
-	handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
-		if (!decision.shouldSendNotification) {
+	private sendNotifyEscalation = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision, escalationChannelID: string | null | undefined) => {
+		if (!escalationChannelID) {
+			this.logger.warn({
+				message: "Escalation channel ID not provided for escalation notification",
+				service: SERVICE_NAME,
+				method: "sendNotifyEscalation",
+			});
+			return false;
+		}
+
+		const notification = await this.notificationsRepository.findById(escalationChannelID, monitor.teamId);
+		if (!notification) {
+			this.logger.warn({
+				message: `Escalation notification with ID ${escalationChannelID} not found`,
+				service: SERVICE_NAME,
+				method: "sendNotifyEscalation",
+			});
 			return false;
 		}
 
+		// Build notification message for escalation
+		const settings = this.settingsService.getSettings();
+		const clientHost = settings.clientHost || "Host not defined";
+		const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost);
+
+		this.logger.debug({
+			message: `Sending escalation notification to channel ID ${escalationChannelID}`,
+			service: SERVICE_NAME,
+			method: "sendNotifyEscalation",
+		});
+
+		const result = await this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage);
+
+		if (result){
+			try{
+				const activeIncident = await this.monitorsRepository.findById(monitor.id, monitor.teamId);
+				this.logger.debug({
+					message: `Marking incident as escalated for monitor ${monitor.id}`,
+					service: SERVICE_NAME,
+					method: "sendNotifyEscalation",
+				});
+			} catch (error: unknown) {
+				this.logger.warn({
+					message: `Failed to mark incident as escalated for monitor ${monitor.id}: ${(error as Error).message}`,
+					service: SERVICE_NAME,
+					method: "sendNotifyEscalation",
+				});
+			}
+		}
 		// Send notifications based on decision
-		return await this.sendNotifications(monitor, monitorStatusResponse, decision);
+		return result;
+	};
+
+	handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
+		if(decision.shouldEscalateIncident && decision.escalationChannelID){
+			await this.sendNotifyEscalation(monitor, monitorStatusResponse, decision, decision.escalationChannelID);
+		}
+		if (decision.shouldSendNotification) {
+			return await this.sendNotifications(monitor, monitorStatusResponse, decision);
+		}
+
+		return false;
 	};
 
 	sendTestNotification = async (notification: Partial<Notification>) => {

server/src/types/incident.ts

@@ -16,6 +16,7 @@ export interface Incident {
 	resolvedBy?: string | null;
 	resolvedByEmail?: string | null;
 	comment?: string | null;
+	escalationTime?: string | null;
 	createdAt: string;
 	updatedAt: string;
 }

server/src/types/monitor.ts

@@ -52,6 +52,10 @@ export interface Monitor {
 	group: string | null;
 	geoCheckEnabled?: boolean;
 	geoCheckLocations?: GeoContinent[];
+	escalation?:{
+		escalationDelay: number;
+		channelID: string;
+	}
 	geoCheckInterval?: number;
 	recentChecks: CheckSnapshot[];
 	createdAt: string;