diff --git a/client/package-lock.json b/client/package-lock.json index 7462d95d71..b0c1acfb46 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -83,6 +83,7 @@ "node_modules/@babel/core": { "version": "7.29.0", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -353,6 +354,7 @@ "node_modules/@emotion/react": { "version": "11.14.0", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -390,6 +392,7 @@ "node_modules/@emotion/styled": { "version": "11.14.1", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -1159,6 +1162,7 @@ "node_modules/@mui/material": { "version": "7.3.7", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.28.4", "@mui/core-downloads-tracker": "^7.3.7", @@ -1263,6 +1267,7 @@ "node_modules/@mui/system": { "version": "7.3.7", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.28.4", "@mui/private-theming": "^7.3.7", @@ -2303,6 +2308,7 @@ "version": "24.5.2", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.12.0" } @@ -2318,6 +2324,7 @@ "node_modules/@types/react": { "version": "18.3.27", "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -2439,6 +2446,7 @@ "version": "8.15.0", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2763,6 +2771,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3194,7 +3203,8 @@ }, "node_modules/dayjs": { "version": "1.11.13", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/debug": { "version": "4.4.3", @@ -3576,6 +3586,7 @@ "version": "8.57.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -4320,6 +4331,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.27.6" }, @@ -5103,6 +5115,7 @@ "resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-5.19.0.tgz", "integrity": "sha512-REhYUN8gNP3HlcIZS6QU2uy8iovl31cXsrNDkCcqWSQbCkcpdYLczqDz5PVIwNH42UQNyvukjes/RoHPDrOUmQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@mapbox/geojson-rewind": "^0.5.2", "@mapbox/jsonlint-lines-primitives": "^2.0.2", @@ -5750,6 +5763,7 @@ "node_modules/react-dom": { "version": "18.3.1", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -5761,6 +5775,7 @@ "node_modules/react-hook-form": { "version": "7.71.1", "license": "MIT", + "peer": true, "engines": { "node": ">=18.0.0" }, @@ -5960,7 +5975,8 @@ }, "node_modules/redux": { "version": "5.0.1", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/redux-persist": { "version": "6.0.0", @@ -6110,6 +6126,7 @@ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -6898,6 +6915,7 @@ "version": "5.9.3", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7033,6 +7051,7 @@ "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 963409fc8a..e5627d0c79 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -12,11 +12,13 @@ const getBaseDefaults = (data?: Monitor | null) => ({ description: data?.description || "", interval: data?.interval || 60000, notifications: data?.notifications || [], + escalationNotifications: data?.escalationNotifications || [], statusWindowSize: data?.statusWindowSize || 5, statusWindowThreshold: data?.statusWindowThreshold || 60, geoCheckEnabled: data?.geoCheckEnabled ?? false, geoCheckLocations: data?.geoCheckLocations || [], geoCheckInterval: data?.geoCheckInterval || 300000, + escalationDelay: data?.escalationDelay ?? 0, }); export const useMonitorForm = ({ diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 15b76eab36..1d46935677 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -765,6 +765,105 @@ const CreateMonitorPage = () => { } /> + + ( + { + const raw = e.target.value; + if (raw === "") { + field.onChange(0); + return; + } + const n = Number(raw); + field.onChange(Number.isFinite(n) ? n : 0); + }} + type="number" + inputProps={{ min: 0, step: 1 }} + fieldLabel={t( + "pages.createMonitor.form.escalation.option.delay.label" + )} + fullWidth + error={!!fieldState.error} + helperText={fieldState.error?.message ?? ""} + /> + )} + /> + { + const notificationOptions = (notifications ?? []).map((n) => ({ + ...n, + name: n.notificationName, + })); + const selectedEscalation = notificationOptions.filter((n) => + (field.value ?? []).includes(n.id) + ); + return ( + + option.name} + onChange={(_: unknown, newValue: typeof notificationOptions) => { + field.onChange(newValue.map((n) => n.id)); + }} + isOptionEqualToValue={(option, value) => option.id === value.id} + /> + {selectedEscalation.length > 0 && ( + + {selectedEscalation.map((notification, index) => ( + + + {notification.notificationName} + + { + field.onChange( + (field.value ?? []).filter( + (id: string) => id !== notification.id + ) + ); + }} + aria-label="Remove escalation notification" + > + + + {index < selectedEscalation.length - 1 && } + + ))} + + )} + + ); + }} + /> + + } + /> + {(watchedType === "http" || watchedType === "grpc" || watchedType === "websocket") && ( diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index 053b517d1d..dde959c48a 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -60,6 +60,7 @@ export interface Monitor { interval: number; uptimePercentage?: number; notifications: string[]; + escalationNotifications: string[]; secret?: string; cpuAlertThreshold: number; cpuAlertCounter: number; @@ -76,6 +77,7 @@ export interface Monitor { geoCheckEnabled?: boolean; geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; + escalationDelay?: number; recentChecks: CheckSnapshot[]; createdAt: string; updatedAt: string; diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 9acffe6fed..ef5abcac5e 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -13,6 +13,7 @@ const baseSchema = z.object({ description: z.string().optional(), interval: z.number().min(15000, "Interval must be at least 15 seconds"), notifications: z.array(z.string()), + escalationNotifications: z.array(z.string()), statusWindowSize: z .number({ message: "Status window size is required" }) .min(1, "Status window size must be at least 1") @@ -27,6 +28,10 @@ const baseSchema = z.object({ .number() .min(300000, "Interval must be at least 5 minutes") .optional(), + escalationDelay: z + .number({ message: "Escalation delay is required" }) + .int("Escalation delay must be a whole number of minutes") + .min(0, "Escalation delay cannot be negative"), }); // HTTP monitor schema diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 92a21939f3..6f2afc491d 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -543,6 +543,18 @@ "description": "Select the notification channels you want to use", "title": "Notifications" }, + "escalation": { + "description": "If the monitor stays down for the specified time, notify additional channels.", + "title": "Escalation rules", + "option": { + "delay": { + "label": "Escalate After (minutes)" + }, + "escalationNotifications": { + "label": "Escalation Notification Channels" + } + } + }, "type": { "description": "Select the type of check to perform", "optionDockerDescription": "Use Docker to monitor if a container is running.", diff --git a/server/openapi.json b/server/openapi.json index 35d109693d..688b1b4230 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -3599,6 +3599,13 @@ }, "description": "Array of notification IDs to associate with this monitor" }, + "escalationNotifications": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Array of notification IDs for escalation alerts" + }, "secret": { "type": "string" }, @@ -3655,6 +3662,11 @@ "geoCheckInterval": { "type": "integer", "minimum": 300000 + }, + "escalationDelay": { + "type": "integer", + "minimum": 0, + "description": "Escalation notification delay in minutes" } } }, @@ -3695,6 +3707,13 @@ "type": "string" } }, + "escalationNotifications": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Array of notification IDs for escalation alerts" + }, "secret": { "type": "string" }, @@ -3761,6 +3780,11 @@ "geoCheckInterval": { "type": "integer", "minimum": 300000 + }, + "escalationDelay": { + "type": "integer", + "minimum": 0, + "description": "Escalation notification delay in minutes" } } }, @@ -3817,6 +3841,18 @@ "type": "string" } }, + "escalationNotifications": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Array of notification IDs for escalation alerts" + }, + "escalationDelay": { + "type": "integer", + "minimum": 0, + "description": "Escalation notification delay in minutes" + }, "httpOptions": { "type": "object", "properties": { diff --git a/server/package-lock.json b/server/package-lock.json index 5d9d920ee5..21e72b6139 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -873,6 +873,7 @@ "integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/generator": "^7.28.6", @@ -1502,6 +1503,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -1546,6 +1548,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -4337,6 +4340,7 @@ "integrity": "sha512-wGA0NX93b19/dZC1J18tKWVIYWyyF2ZjT9vin/NRu0qzzvfVzWjs04iq2rQ3H65vCTQYlRqs3YHfY7zjdV+9Kw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -4497,6 +4501,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.1.tgz", "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -4734,6 +4739,7 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -5347,6 +5353,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5925,6 +5932,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.19", "caniuse-lite": "^1.0.30001751", @@ -6867,6 +6875,7 @@ "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-7.1.1.tgz", "integrity": "sha512-fm4D8ti0dQmFPeF8DXSAA//btEmqCOgAc/9Oa3C1LW94h5usNrJEfrON7b4FkPZgnDEn6OUs5NdxiJZmAtGOpQ==", "license": "MIT", + "peer": true, "dependencies": { "cssnano-preset-default": "^7.0.9", "lilconfig": "^3.1.3" @@ -7634,6 +7643,7 @@ "integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -7995,6 +8005,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -9639,6 +9650,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -12509,6 +12521,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -14604,6 +14617,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -14772,6 +14786,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -14934,6 +14949,7 @@ "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/server/src/config/services.ts b/server/src/config/services.ts index b31c8a5e91..f6f620b20f 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -220,9 +220,7 @@ export const initializeServices = async ({ const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService); - const statusService = new StatusService(logger, bufferService, monitorsRepository, monitorStatsRepository, checksRepository); - - // Notification providers + // Notification providers (before StatusService — status updates may schedule escalation) const webhookProvider = new WebhookProvider(logger); const slackProvider = new SlackProvider(logger); const emailProvider = new EmailProvider(emailService, logger); @@ -246,6 +244,15 @@ export const initializeServices = async ({ notificationMessageBuilder ); + const statusService = new StatusService( + logger, + bufferService, + monitorsRepository, + monitorStatsRepository, + checksRepository, + notificationsService + ); + const superSimpleQueueHelper = new SuperSimpleQueueHelper( logger, networkService, diff --git a/server/src/db/models/Monitor.ts b/server/src/db/models/Monitor.ts index 036aeadad6..fde412ab64 100644 --- a/server/src/db/models/Monitor.ts +++ b/server/src/db/models/Monitor.ts @@ -18,11 +18,12 @@ type CheckSnapshotDocument = Omit & { createdAt: Dat type MonitorDocumentBase = Omit< Monitor, - "id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" + "id" | "userId" | "teamId" | "notifications" | "escalationNotifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" > & { statusWindow: boolean[]; recentChecks: CheckSnapshotDocument[]; notifications: Types.ObjectId[]; + escalationNotifications: Types.ObjectId[]; selectedDisks: string[]; matchMethod?: MonitorMatchMethod; }; @@ -284,6 +285,12 @@ const MonitorSchema = new Schema( ref: "Notification", }, ], + escalationNotifications: [ + { + type: Schema.Types.ObjectId, + ref: "Notification", + }, + ], secret: { type: String, }, @@ -351,6 +358,10 @@ const MonitorSchema = new Schema( type: Number, default: 300000, }, + escalationDelay: { + type: Number, + default: 0, + }, recentChecks: { type: [checkSnapshotSchema], default: [], diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d7594483..5e8f6019b0 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -7,8 +7,20 @@ import { MongoBulkWriteError } from "mongodb"; import { AppError } from "@/utils/AppError.js"; class MongoMonitorsRepository implements IMonitorsRepository { + private readonly entityMetaKeysToOmit = ["id", "createdAt", "updatedAt"] as const; + + /** Strip API-layer fields that must not be written with Mongoose (avoids cast / strict issues on $set and create). */ + private omitEntityMetaForWrite = (payload: Record): Record => { + const copy = { ...payload }; + for (const key of this.entityMetaKeysToOmit) { + delete copy[key]; + } + return copy; + }; + create = async (monitor: Monitor, teamId: string, userId: string) => { - const monitorModel = new MonitorModel({ ...monitor, teamId, userId }); + const payload = this.omitEntityMetaForWrite({ ...monitor, teamId, userId } as Record); + const monitorModel = new MonitorModel(payload); const saved = await monitorModel.save(); return this.toEntity(saved); }; @@ -167,12 +179,11 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; updateById = async (monitorId: string, teamId: string, patch: Partial) => { + const setPayload = this.omitEntityMetaForWrite({ ...patch } as Record); const updatedMonitor = await MonitorModel.findOneAndUpdate( { _id: monitorId, teamId }, { - $set: { - ...patch, - }, + $set: setPayload, }, { new: true, runValidators: true } ); @@ -294,6 +305,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { removeNotificationFromMonitors = async (notificationId: string): Promise => { await MonitorModel.updateMany({ notifications: notificationId }, { $pull: { notifications: notificationId } }); + await MonitorModel.updateMany({ escalationNotifications: notificationId }, { $pull: { escalationNotifications: notificationId } }); }; updateNotifications = async ( @@ -351,6 +363,9 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification)); + const escalationNotificationIds = (doc.escalationNotifications ?? []).map((notification) => + toStringId(notification) + ); return { id: toStringId(doc._id), @@ -374,6 +389,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { interval: doc.interval, uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, + escalationNotifications: escalationNotificationIds, secret: doc.secret ?? undefined, cpuAlertThreshold: doc.cpuAlertThreshold, cpuAlertCounter: doc.cpuAlertCounter, @@ -391,6 +407,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, + escalationDelay: doc.escalationDelay ?? 0, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; @@ -410,6 +427,9 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification)); + const escalationNotificationIds = (doc.escalationNotifications ?? []).map((notification: unknown) => + toStringId(notification) + ); return { id: toStringId(doc._id), @@ -433,6 +453,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { interval: doc.interval, uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, + escalationNotifications: escalationNotificationIds, secret: doc.secret ?? undefined, cpuAlertThreshold: doc.cpuAlertThreshold, cpuAlertCounter: doc.cpuAlertCounter, @@ -450,6 +471,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, + escalationDelay: doc.escalationDelay ?? 0, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; diff --git a/server/src/service/business/monitorService.ts b/server/src/service/business/monitorService.ts index 71c9d9d906..dc8732da2f 100644 --- a/server/src/service/business/monitorService.ts +++ b/server/src/service/business/monitorService.ts @@ -33,7 +33,7 @@ export interface IMonitorService { readonly serviceName: string; // create - createMonitor(teamId: string, userId: string, body: Partial): Promise; + createMonitor(teamId: string, userId: string, body: Partial): Promise; createMonitors(monitors: Array): Promise; addDemoMonitors(args: { userId: string; teamId: string }): Promise; @@ -165,13 +165,25 @@ export class MonitorService implements IMonitorService { return formatLookup[dateRange]; }; - createMonitor = async (teamId: string, userId: string, body: Monitor): Promise => { + createMonitor = async (teamId: string, userId: string, body: Monitor): Promise => { const monitor = await this.monitorsRepository.create(body, teamId, userId); if (!monitor) { throw new AppError({ message: "Failed to create monitor", status: 500, service: SERVICE_NAME, method: "createMonitor" }); } + this.logger.info({ + message: "Monitor created", + service: SERVICE_NAME, + method: "createMonitor", + details: { + monitorId: monitor.id, + escalationDelay: monitor.escalationDelay, + escalationNotifications: monitor.escalationNotifications, + }, + }); + this.jobQueue.addJob(monitor.id, monitor); + return monitor; }; createMonitors = async (monitors: Array): Promise => { @@ -438,6 +450,16 @@ export class MonitorService implements IMonitorService { editMonitor = async ({ teamId, monitorId, body }: { teamId: string; monitorId: string; body: Partial }) => { const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body); + this.logger.info({ + message: "Monitor updated", + service: SERVICE_NAME, + method: "editMonitor", + details: { + monitorId: editedMonitor.id, + escalationDelay: editedMonitor.escalationDelay, + escalationNotifications: editedMonitor.escalationNotifications, + }, + }); await this.jobQueue.updateJob(editedMonitor); return editedMonitor; }; diff --git a/server/src/service/infrastructure/notificationMessageBuilder.ts b/server/src/service/infrastructure/notificationMessageBuilder.ts index 934163b2a9..8f5df48727 100644 --- a/server/src/service/infrastructure/notificationMessageBuilder.ts +++ b/server/src/service/infrastructure/notificationMessageBuilder.ts @@ -10,12 +10,12 @@ import type { export interface INotificationMessageBuilder { buildMessage( - monitor: Monitor, + monitor: Omit, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision, clientHost: string ): NotificationMessage; - extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[]; + extractThresholdBreaches(monitor: Omit, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[]; } const SERVICE_NAME = "NotificationMessageBuilder"; @@ -24,7 +24,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { static SERVICE_NAME = SERVICE_NAME; buildMessage( - monitor: Monitor, + monitor: Omit, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision, clientHost: string @@ -52,7 +52,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private determineNotificationType(decision: MonitorActionDecision, monitor: Monitor): NotificationType { + private determineNotificationType(decision: MonitorActionDecision, monitor: Omit): NotificationType { // Down status has highest priority (critical) if (monitor.status === "down") { return "monitor_down"; @@ -93,7 +93,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildContent(type: NotificationType, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + private buildContent(type: NotificationType, monitor: Omit, monitorStatusResponse: MonitorStatusResponse): NotificationContent { switch (type) { case "monitor_down": return this.buildMonitorDownContent(monitor, monitorStatusResponse); @@ -108,7 +108,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildMonitorDownContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + private buildMonitorDownContent(monitor: Omit, monitorStatusResponse: MonitorStatusResponse): NotificationContent { const title = `Monitor Down: ${monitor.name}`; const summary = `Monitor "${monitor.name}" is currently down and unreachable.`; const details = [`URL: ${monitor.url}`, `Status: Down`, `Type: ${monitor.type}`]; @@ -131,7 +131,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildMonitorUpContent(monitor: Monitor): NotificationContent { + private buildMonitorUpContent(monitor: Omit): NotificationContent { const title = `Monitor Recovered: ${monitor.name}`; const summary = `Monitor "${monitor.name}" is back up and operational.`; const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`]; @@ -144,7 +144,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildThresholdBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + private buildThresholdBreachContent(monitor: Omit, monitorStatusResponse: MonitorStatusResponse): NotificationContent { const title = `Threshold Exceeded: ${monitor.name}`; const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`; const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`]; @@ -160,7 +160,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildThresholdResolvedContent(monitor: Monitor): NotificationContent { + private buildThresholdResolvedContent(monitor: Omit): NotificationContent { const title = `Thresholds Resolved: ${monitor.name}`; const summary = `Monitor "${monitor.name}" thresholds have returned to normal.`; const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`]; @@ -173,7 +173,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildDefaultContent(monitor: Monitor): NotificationContent { + private buildDefaultContent(monitor: Omit): NotificationContent { return { title: `Monitor: ${monitor.name}`, summary: `Status update for monitor "${monitor.name}".`, @@ -182,7 +182,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[] { + public extractThresholdBreaches(monitor: Omit, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[] { const breaches: ThresholdBreach[] = []; // Check if this is a hardware monitor with threshold data diff --git a/server/src/service/infrastructure/notificationProviders/email.ts b/server/src/service/infrastructure/notificationProviders/email.ts index b3686651cc..76c12ff141 100644 --- a/server/src/service/infrastructure/notificationProviders/email.ts +++ b/server/src/service/infrastructure/notificationProviders/email.ts @@ -78,6 +78,10 @@ export class EmailProvider implements INotificationProvider { } private buildSubject(message: NotificationMessage): string { + const override = message.metadata.emailSubjectOverride; + if (override !== undefined && override !== "") { + return override; + } switch (message.type) { case "monitor_down": return `Monitor ${message.monitor.name} is down`; diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index c75477c88c..db732f2ab6 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -14,6 +14,13 @@ export interface INotificationsService { updateById(id: string, teamId: string, updateData: Partial): Promise; deleteById: (id: string, teamId: string) => Promise; handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; + /** Called when a monitor has just transitioned to `down` (from the job queue, after status is persisted). */ + scheduleEscalationOnDownTransition: ( + monitorId: string, + teamId: string, + monitorStatusResponse: MonitorStatusResponse, + downDecision: MonitorActionDecision + ) => Promise; sendTestNotification: (notification: Partial) => Promise; testAllNotifications: (notificationIds: string[]) => Promise; @@ -21,9 +28,23 @@ export interface INotificationsService { const SERVICE_NAME = "NotificationsService"; +const OBJECT_ID_HEX = /^[0-9a-fA-F]{24}$/; + +function sanitizeNotificationObjectIds(ids: string[] | undefined | null): string[] { + if (!ids?.length) { + return []; + } + return ids.map((id) => (typeof id === "string" ? id.trim() : "")).filter((id) => OBJECT_ID_HEX.test(id)); +} + +/** Resolved notification channels for send path (`Monitor.notifications` in DB/API remains `string[]`). */ +type MonitorWithResolvedNotifications = Omit & { notifications: Notification[] }; + export class NotificationsService implements INotificationsService { static SERVICE_NAME = SERVICE_NAME; + private escalationTimeouts = new Map>(); + private notificationsRepository: INotificationsRepository; private monitorsRepository: IMonitorsRepository; private webhookProvider: INotificationProvider; @@ -67,7 +88,7 @@ export class NotificationsService implements INotificationsService { private send = async ( notification: Notification, - monitor: Monitor, + monitor: Omit, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision, notificationMessage: NotificationMessage | undefined @@ -107,16 +128,29 @@ export class NotificationsService implements INotificationsService { } }; - private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { - const notificationIds = monitor.notifications ?? []; - const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds); + private clearEscalationSchedule = (monitorId: string): void => { + const existing = this.escalationTimeouts.get(monitorId); + if (existing !== undefined) { + clearTimeout(existing); + this.escalationTimeouts.delete(monitorId); + } + }; - // Build notification message once for all notifications + private sendNotifications = async ( + monitor: MonitorWithResolvedNotifications, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + notificationMessage?: NotificationMessage + ) => { + const notifications = monitor.notifications; + + // Build notification message once for all notifications (unless caller supplies one, e.g. escalation) const settings = this.settingsService.getSettings(); const clientHost = settings.clientHost || "Host not defined"; - const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost); + const builtMessage = + notificationMessage ?? this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost); - const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage)); + const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, builtMessage)); const outcomes = await Promise.all(tasks); const succeeded = outcomes.filter(Boolean).length; @@ -132,13 +166,127 @@ export class NotificationsService implements INotificationsService { return succeeded === notifications.length; }; + private runEscalation = async ( + monitorId: string, + teamId: string, + capturedStatusResponse: MonitorStatusResponse, + downDecision: MonitorActionDecision + ): Promise => { + try { + const current = await this.monitorsRepository.findById(monitorId, teamId); + if (current.status !== "down") { + return; + } + if (!current.escalationNotifications || current.escalationNotifications.length === 0) { + return; + } + const escalationIds = sanitizeNotificationObjectIds(current.escalationNotifications); + if (escalationIds.length === 0) { + return; + } + const allNotifications = await this.notificationsRepository.findByTeamId(teamId); + const escalationNotificationsFull = allNotifications.filter((n) => escalationIds.includes(n.id)); + console.log("Escalation IDs:", escalationIds); + console.log("Resolved escalation notifications:", escalationNotificationsFull); + console.log("Escalation triggered:", current.name); + const monitorForEscalationSend: MonitorWithResolvedNotifications = { + ...current, + notifications: escalationNotificationsFull, + }; + try { + const settings = this.settingsService.getSettings(); + const clientHost = settings.clientHost || "Host not defined"; + const originalMessage = this.notificationMessageBuilder.buildMessage( + monitorForEscalationSend, + capturedStatusResponse, + downDecision, + clientHost + ); + const delayMinutes = Math.max(0, Math.floor(Number(current.escalationDelay) || 0)); + const escalationTitle = `Escalation: Monitor ${current.name} still down`; + const escalationSummary = `Monitor "${current.name}" is still down after ${delayMinutes} minutes.`; + const escalationMessage: NotificationMessage = { + ...originalMessage, + metadata: { + ...originalMessage.metadata, + emailSubjectOverride: escalationTitle, + }, + content: { + ...originalMessage.content, + title: escalationTitle, + summary: escalationSummary, + }, + }; + await this.sendNotifications(monitorForEscalationSend, capturedStatusResponse, downDecision, escalationMessage); + } catch (error: unknown) { + console.error("Escalation email error:", error); + this.logger.error({ + message: `Escalation send failed for monitor ${monitorId}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "runEscalation", + stack: error instanceof Error ? error.stack : undefined, + }); + } + } catch (error: unknown) { + console.error("Escalation email error:", error); + this.logger.error({ + message: `Escalation run failed for monitor ${monitorId}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "runEscalation", + stack: error instanceof Error ? error.stack : undefined, + }); + } + }; + + scheduleEscalationOnDownTransition = async ( + monitorId: string, + teamId: string, + monitorStatusResponse: MonitorStatusResponse, + downDecision: MonitorActionDecision + ): Promise => { + try { + const monitor = await this.monitorsRepository.findById(monitorId, teamId); + if (monitor.status !== "down") { + return; + } + const delayMinutes = Math.max(0, Math.floor(Number(monitor.escalationDelay) || 0)); + if (!delayMinutes || !monitor.escalationNotifications || monitor.escalationNotifications.length === 0) { + return; + } + if (sanitizeNotificationObjectIds(monitor.escalationNotifications).length === 0) { + return; + } + + console.log("Escalation scheduled:", monitor.name); + this.clearEscalationSchedule(monitor.id); + const ms = delayMinutes * 60 * 1000; + const timeout = setTimeout(() => { + this.escalationTimeouts.delete(monitor.id); + void this.runEscalation(monitor.id, teamId, monitorStatusResponse, downDecision); + }, ms); + this.escalationTimeouts.set(monitor.id, timeout); + } catch (error: unknown) { + this.logger.error({ + message: `Failed to schedule escalation for monitor ${monitorId}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "scheduleEscalationOnDownTransition", + stack: error instanceof Error ? error.stack : undefined, + }); + } + }; + handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { if (!decision.shouldSendNotification) { return false; } - // Send notifications based on decision - return await this.sendNotifications(monitor, monitorStatusResponse, decision); + if (monitor.status !== "down") { + this.clearEscalationSchedule(monitor.id); + } + + const resolvedNotifications = await this.notificationsRepository.findNotificationsByIds(monitor.notifications ?? []); + const monitorForSend: MonitorWithResolvedNotifications = { ...monitor, notifications: resolvedNotifications }; + return await this.sendNotifications(monitorForSend, monitorStatusResponse, decision); }; sendTestNotification = async (notification: Partial) => { diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index ef725b80af..7cede4c2be 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -1,4 +1,6 @@ import { IChecksRepository, IMonitorsRepository, IMonitorStatsRepository } from "@/repositories/index.js"; +import type { INotificationsService } from "@/service/infrastructure/notificationsService.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { Monitor, MonitorStatus, @@ -22,6 +24,15 @@ import { ILogger } from "@/utils/logger.js"; import { IBufferService } from "./bufferService.js"; const SERVICE_NAME = "StatusService"; +/** Matches `evaluateMonitorAction` when an uptime-style monitor transitions to `down` (for escalation / message builder). */ +const STATUS_DOWN_NOTIFICATION_DECISION: MonitorActionDecision = { + shouldCreateIncident: true, + shouldResolveIncident: false, + shouldSendNotification: true, + incidentReason: "status_down", + notificationReason: "status_change", +}; + export interface IStatusService { updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise; updateMonitorStatus( @@ -47,19 +58,22 @@ export class StatusService implements IStatusService { private monitorsRepository: IMonitorsRepository; private monitorStatsRepository: IMonitorStatsRepository; private checksRepository: IChecksRepository; + private notificationsService: INotificationsService; constructor( logger: ILogger, buffer: IBufferService, monitorsRepository: IMonitorsRepository, monitorStatsRepository: IMonitorStatsRepository, - checksRepository: IChecksRepository + checksRepository: IChecksRepository, + notificationsService: INotificationsService ) { this.logger = logger; this.buffer = buffer; this.monitorsRepository = monitorsRepository; this.monitorStatsRepository = monitorStatsRepository; this.checksRepository = checksRepository; + this.notificationsService = notificationsService; } get serviceName() { @@ -350,6 +364,20 @@ export class StatusService implements IStatusService { const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor); + // Escalation: right after a check persists a threshold-based transition to DOWN (same moment incidents/alerts use). + if (statusChanged && updated.status === "down") { + void this.notificationsService + .scheduleEscalationOnDownTransition(updated.id, updated.teamId, statusResponse, STATUS_DOWN_NOTIFICATION_DECISION) + .catch((error: unknown) => { + this.logger.error({ + message: `Failed to schedule escalation for monitor ${updated.id}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "updateMonitorStatus", + stack: error instanceof Error ? error.stack : undefined, + }); + }); + } + return { monitor: updated, statusChanged, diff --git a/server/src/types/monitor.ts b/server/src/types/monitor.ts index f29ce75d78..49305cdac5 100644 --- a/server/src/types/monitor.ts +++ b/server/src/types/monitor.ts @@ -37,6 +37,7 @@ export interface Monitor { interval: number; uptimePercentage?: number; notifications: string[]; + escalationNotifications: string[]; secret?: string; cpuAlertThreshold: number; cpuAlertCounter: number; @@ -53,6 +54,7 @@ export interface Monitor { geoCheckEnabled?: boolean; geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; + escalationDelay?: number; recentChecks: CheckSnapshot[]; createdAt: string; updatedAt: string; diff --git a/server/src/types/notificationMessage.ts b/server/src/types/notificationMessage.ts index f06ff1bd9a..cd6a17b537 100644 --- a/server/src/types/notificationMessage.ts +++ b/server/src/types/notificationMessage.ts @@ -49,5 +49,7 @@ export interface NotificationMessage { metadata: { teamId: string; notificationReason: string; + /** When set (escalation only), email subject uses this instead of the default per-type subject. */ + emailSubjectOverride?: string; }; } diff --git a/server/src/validation/monitorValidation.ts b/server/src/validation/monitorValidation.ts index df000ecef2..142bda922a 100644 --- a/server/src/validation/monitorValidation.ts +++ b/server/src/validation/monitorValidation.ts @@ -67,6 +67,7 @@ export const createMonitorBodyValidation = z.object({ diskAlertThreshold: z.number().optional(), tempAlertThreshold: z.number().optional(), notifications: z.array(z.string()).optional(), + escalationNotifications: z.array(z.string()).optional(), secret: z.string().optional(), jsonPath: z.union([z.string(), z.literal("")]).optional(), expectedValue: z.union([z.string(), z.literal("")]).optional(), @@ -78,6 +79,7 @@ export const createMonitorBodyValidation = z.object({ geoCheckEnabled: z.boolean().optional(), geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), + escalationDelay: z.coerce.number().int().min(0).optional(), }); export const editMonitorBodyValidation = z.object({ @@ -89,6 +91,7 @@ export const editMonitorBodyValidation = z.object({ description: z.union([z.string(), z.literal("")]).optional(), interval: z.number().optional(), notifications: z.array(z.string()).optional(), + escalationNotifications: z.array(z.string()).optional(), secret: z.string().optional(), ignoreTlsErrors: z.boolean().optional(), useAdvancedMatching: z.boolean().optional(), @@ -107,6 +110,7 @@ export const editMonitorBodyValidation = z.object({ geoCheckEnabled: z.boolean().optional(), geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), + escalationDelay: z.coerce.number().int().min(0).optional(), }); export const pauseMonitorParamValidation = z.object({ @@ -144,6 +148,7 @@ const importedMonitorSchema = z.object({ interval: z.number().default(60000), uptimePercentage: z.number().optional(), notifications: z.array(z.string()).default([]), + escalationNotifications: z.array(z.string()).default([]), secret: z.string().optional(), cpuAlertThreshold: z.number().default(100), cpuAlertCounter: z.number().default(5), @@ -160,6 +165,7 @@ const importedMonitorSchema = z.object({ geoCheckEnabled: z.boolean().default(false), geoCheckLocations: z.array(z.enum(GeoContinents)).default([]), geoCheckInterval: z.number().min(300000).default(300000), + escalationDelay: z.coerce.number().int().min(0).default(0), createdAt: z.string().optional(), updatedAt: z.string().optional(), }); diff --git a/server/test/monitorService.test.ts b/server/test/monitorService.test.ts index 32708a42e2..a306f1ed54 100644 --- a/server/test/monitorService.test.ts +++ b/server/test/monitorService.test.ts @@ -171,6 +171,7 @@ describe("MonitorService", () => { tempAlertThreshold: 5, selectedDisks: [], notifications: [], + escalationNotifications: [], group: null, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),