Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions client/src/Hooks/useMonitorForm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const getBaseDefaults = (data?: Monitor | null) => ({
description: data?.description || "",
interval: data?.interval || 60000,
notifications: data?.notifications || [],
escalationNotifications: data?.escalationNotifications || [],
statusWindowSize: data?.statusWindowSize || 5,
statusWindowThreshold: data?.statusWindowThreshold || 60,
geoCheckEnabled: data?.geoCheckEnabled ?? false,
Expand Down
134 changes: 134 additions & 0 deletions client/src/Pages/CreateMonitor/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -765,6 +765,140 @@ const CreateMonitorPage = () => {
}
/>

<ConfigBox
title={t("pages.createMonitor.form.escalationNotifications.title")}
subtitle={t("pages.createMonitor.form.escalationNotifications.description")}
rightContent={
<Controller
name="escalationNotifications"
control={control}
render={({ field, fieldState }) => {
const notificationOptions = (notifications ?? []).map((n) => ({
...n,
name: n.notificationName,
}));

const selectedNotifications = notificationOptions.filter((n) =>
(field.value ?? []).some((entry) => entry.notificationId === n.id)
);

const selectedEscalations = (field.value ?? [])
.map((entry) => ({
entry,
notification: notificationOptions.find(
(option) => option.id === entry.notificationId
),
}))
.filter((item) => Boolean(item.notification));

return (
<Stack spacing={theme.spacing(LAYOUT.MD)}>
<Autocomplete
multiple
options={notificationOptions}
value={selectedNotifications}
fieldLabel={t(
"pages.createMonitor.form.escalationNotifications.option.channel.label"
)}
getOptionLabel={(option) => option.name}
onChange={(_: unknown, newValue: typeof notificationOptions) => {
const existingDelayById = new Map(
(field.value ?? []).map((entry) => [entry.notificationId, entry.delayMinutes])
);

field.onChange(
newValue.map((item) => ({
notificationId: item.id,
delayMinutes: existingDelayById.get(item.id) ?? 15,
}))
);
}}
isOptionEqualToValue={(option, value) => option.id === value.id}
/>
{selectedEscalations.length > 0 && (
<Stack
flex={1}
width="100%"
>
{selectedEscalations.map(({ entry, notification }, index) => {
if (!notification) {
return null;
}

return (
<Stack
direction={{ xs: "column", md: "row" }}
alignItems={{ xs: "stretch", md: "center" }}
spacing={theme.spacing(SPACING.SM)}
key={notification.id}
width="100%"
>
<Stack flexGrow={1}>
<Typography variant="caption" color="text.secondary">
{t(
"pages.createMonitor.form.escalationNotifications.option.channel.label"
)}
</Typography>
<Typography>{notification.notificationName}</Typography>
</Stack>
<Stack
direction="row"
alignItems="center"
spacing={theme.spacing(SPACING.SM)}
>
<TextField
type="number"
fieldLabel={t(
"pages.createMonitor.form.escalationNotifications.option.delay.label"
)}
value={entry.delayMinutes}
onChange={(e) => {
const nextValue = Math.max(1, Number(e.target.value) || 1);
field.onChange(
(field.value ?? []).map((item) =>
item.notificationId === notification.id
? { ...item, delayMinutes: nextValue }
: item
)
);
}}
sx={{ minWidth: { xs: "100%", md: 180 } }}
/>
<IconButton
size="small"
onClick={() => {
field.onChange(
(field.value ?? []).filter(
(item) => item.notificationId !== notification.id
)
);
}}
aria-label="Remove escalation notification"
>
<Trash2 size={16} />
</IconButton>
</Stack>
{index < selectedEscalations.length - 1 && <Divider />}
</Stack>
);
})}
</Stack>
)}
{fieldState.error?.message && (
<Typography
color="error"
variant="caption"
>
{fieldState.error.message}
</Typography>
)}
</Stack>
);
}}
/>
}
/>

{(watchedType === "http" ||
watchedType === "grpc" ||
watchedType === "websocket") && (
Expand Down
6 changes: 6 additions & 0 deletions client/src/Types/Monitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ export type MonitorStatus = (typeof MonitorStatuses)[number];

export type MonitorMatchMethod = "equal" | "include" | "regex" | "";

export interface MonitorEscalationNotification {
notificationId: string;
delayMinutes: number;
}

export interface Monitor {
id: string;
userId: string;
Expand All @@ -60,6 +65,7 @@ export interface Monitor {
interval: number;
uptimePercentage?: number;
notifications: string[];
escalationNotifications?: MonitorEscalationNotification[];
secret?: string;
cpuAlertThreshold: number;
cpuAlertCounter: number;
Expand Down
9 changes: 9 additions & 0 deletions client/src/Validation/monitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ const baseSchema = z.object({
description: z.string().optional(),
interval: z.number().min(15000, "Interval must be at least 15 seconds"),
notifications: z.array(z.string()),
escalationNotifications: z.array(
z.object({
notificationId: z.string().min(1, "Escalation notification channel is required"),
delayMinutes: z
.number({ message: "Escalation delay is required" })
.int("Escalation delay must be a whole number")
.min(1, "Escalation delay must be at least 1 minute"),
})
),
statusWindowSize: z
.number({ message: "Status window size is required" })
.min(1, "Status window size must be at least 1")
Expand Down
12 changes: 12 additions & 0 deletions client/src/locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -543,6 +543,18 @@
"description": "Select the notification channels you want to use",
"title": "Notifications"
},
"escalationNotifications": {
"title": "Escalation Notifications",
"description": "Choose channels for follow-up alerts if the monitor is still down after a delay.",
"option": {
"channel": {
"label": "Notification channel"
},
"delay": {
"label": "Escalation delay (minutes)"
}
}
},
"type": {
"description": "Select the type of check to perform",
"optionDockerDescription": "Use Docker to monitor if a container is running.",
Expand Down
1 change: 1 addition & 0 deletions server/src/config/services.ts
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ export const initializeServices = async ({
const notificationsService = new NotificationsService(
notificationsRepository,
monitorsRepository,
incidentsRepository,
webhookProvider,
emailProvider,
slackProvider,
Expand Down
23 changes: 22 additions & 1 deletion server/src/db/models/Monitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ type CheckSnapshotDocument = Omit<CheckSnapshot, "createdAt"> & { createdAt: Dat

type MonitorDocumentBase = Omit<
Monitor,
"id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt"
"id" | "userId" | "teamId" | "notifications" | "escalationNotifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt"
> & {
statusWindow: boolean[];
recentChecks: CheckSnapshotDocument[];
notifications: Types.ObjectId[];
escalationNotifications: { notificationId: Types.ObjectId; delayMinutes: number }[];
selectedDisks: string[];
matchMethod?: MonitorMatchMethod;
};
Expand Down Expand Up @@ -198,6 +199,22 @@ const checkSnapshotSchema = new Schema<CheckSnapshotDocument>(
{ _id: false }
);

const escalationNotificationSchema = new Schema<{ notificationId: Types.ObjectId; delayMinutes: number }>(
{
notificationId: {
type: Schema.Types.ObjectId,
ref: "Notification",
required: true,
},
delayMinutes: {
type: Number,
required: true,
min: 1,
},
},
{ _id: false }
);

const MonitorSchema = new Schema<MonitorDocument>(
{
userId: {
Expand Down Expand Up @@ -284,6 +301,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
ref: "Notification",
},
],
escalationNotifications: {
type: [escalationNotificationSchema],
default: [],
},
secret: {
type: String,
},
Expand Down
12 changes: 11 additions & 1 deletion server/src/repositories/monitors/MongoMonitorsRepository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
if (!monitors.length) {
return [];
}
const payload = monitors.map((monitor) => ({ ...monitor, notifications: undefined }));
const payload = monitors.map((monitor) => ({ ...monitor, notifications: undefined, escalationNotifications: undefined }));
try {
const inserted = await MonitorModel.insertMany(payload, { ordered: false });
return this.mapDocuments(inserted);
Expand Down Expand Up @@ -351,6 +351,10 @@ class MongoMonitorsRepository implements IMonitorsRepository {
};

const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification));
const escalationNotifications = (doc.escalationNotifications ?? []).map((item) => ({
notificationId: toStringId(item.notificationId),
delayMinutes: item.delayMinutes,
}));

return {
id: toStringId(doc._id),
Expand All @@ -374,6 +378,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
interval: doc.interval,
uptimePercentage: doc.uptimePercentage ?? undefined,
notifications: notificationIds,
escalationNotifications,
secret: doc.secret ?? undefined,
cpuAlertThreshold: doc.cpuAlertThreshold,
cpuAlertCounter: doc.cpuAlertCounter,
Expand Down Expand Up @@ -410,6 +415,10 @@ class MongoMonitorsRepository implements IMonitorsRepository {
};

const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification));
const escalationNotifications = (doc.escalationNotifications ?? []).map((item) => ({
notificationId: toStringId(item.notificationId),
delayMinutes: item.delayMinutes,
}));

return {
id: toStringId(doc._id),
Expand All @@ -433,6 +442,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
interval: doc.interval,
uptimePercentage: doc.uptimePercentage ?? undefined,
notifications: notificationIds,
escalationNotifications,
secret: doc.secret ?? undefined,
cpuAlertThreshold: doc.cpuAlertThreshold,
cpuAlertCounter: doc.cpuAlertCounter,
Expand Down
50 changes: 50 additions & 0 deletions server/src/service/infrastructure/notificationMessageBuilder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export interface INotificationMessageBuilder {
decision: MonitorActionDecision,
clientHost: string
): NotificationMessage;
buildEscalationMessage(monitor: Monitor, downForMinutes: number, clientHost: string): NotificationMessage;
extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[];
}

Expand Down Expand Up @@ -52,6 +53,38 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}

buildEscalationMessage(monitor: Monitor, downForMinutes: number, clientHost: string): NotificationMessage {
const downForLabel = this.formatDownDuration(downForMinutes);

return {
type: "monitor_down_escalation",
severity: "critical",
monitor: {
id: monitor.id,
name: monitor.name,
url: monitor.url,
type: monitor.type,
status: monitor.status,
},
content: {
title: `Escalation: ${monitor.name} is still down`,
summary: `Monitor "${monitor.name}" is still down after ${downForLabel}.`,
details: [
`URL: ${monitor.url}`,
`Status: Down`,
`Type: ${monitor.type}`,
`Down for: ${downForLabel}`,
],
timestamp: new Date(),
},
clientHost,
metadata: {
teamId: monitor.teamId,
notificationReason: "escalation",
},
};
}

private determineNotificationType(decision: MonitorActionDecision, monitor: Monitor): NotificationType {
// Down status has highest priority (critical)
if (monitor.status === "down") {
Expand Down Expand Up @@ -80,6 +113,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
private determineSeverity(type: NotificationType): NotificationSeverity {
switch (type) {
case "monitor_down":
case "monitor_down_escalation":
return "critical";
case "threshold_breach":
return "warning";
Expand All @@ -97,6 +131,8 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
switch (type) {
case "monitor_down":
return this.buildMonitorDownContent(monitor, monitorStatusResponse);
case "monitor_down_escalation":
return this.buildMonitorDownContent(monitor, monitorStatusResponse);
case "monitor_up":
return this.buildMonitorUpContent(monitor);
case "threshold_breach":
Expand Down Expand Up @@ -182,6 +218,20 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}

private formatDownDuration(downForMinutes: number): string {
if (downForMinutes < 60) {
return `${downForMinutes} minute${downForMinutes === 1 ? "" : "s"}`;
}

const hours = Math.floor(downForMinutes / 60);
const minutes = downForMinutes % 60;
if (minutes === 0) {
return `${hours} hour${hours === 1 ? "" : "s"}`;
}

return `${hours} hour${hours === 1 ? "" : "s"} ${minutes} minute${minutes === 1 ? "" : "s"}`;
}

public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<HardwareStatusPayload>): ThresholdBreach[] {
const breaches: ThresholdBreach[] = [];

Expand Down
Loading