Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions client/src/Hooks/useMonitorForm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ const getBaseDefaults = (data?: Monitor | null) => ({
notifications: data?.notifications || [],
statusWindowSize: data?.statusWindowSize || 5,
statusWindowThreshold: data?.statusWindowThreshold || 60,
escalationDelay: data?.escalationDelay || 0,
escalationNotifications: data?.escalationNotifications || [],
geoCheckEnabled: data?.geoCheckEnabled ?? false,
geoCheckLocations: data?.geoCheckLocations || [],
geoCheckInterval: data?.geoCheckInterval || 300000,
Expand Down
12 changes: 10 additions & 2 deletions client/src/Pages/Auth/Register/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ interface InviteVerifyResponse {
const RegisterPage = () => {
const { t } = useTranslation();
const { schema, defaults } = useRegisterForm();
const { post, loading } = usePost<RegisterPayload, AuthResponse>();
const { loading } = usePost<RegisterPayload, AuthResponse>();
const dispatch = useDispatch();
const navigate = useNavigate();
const { token } = useParams<{ token?: string }>();
Expand Down Expand Up @@ -73,7 +73,15 @@ const RegisterPage = () => {
if (token) {
payload.token = token;
}
const result = await post("/auth/register", payload);
const response = await fetch("http://localhost:52345/api/v1/auth/register", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(payload),
});

const result = await response.json();

if (result?.success) {
dispatch(setAuthState(result));
Expand Down
90 changes: 88 additions & 2 deletions client/src/Pages/CreateMonitor/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { useMemo, useState } from "react";
import { useEffect } from "react";
import { logger } from "@/Utils/logger";
import { useParams, useLocation, useNavigate } from "react-router";
import { useForm, Controller } from "react-hook-form";
import { useForm, Controller, type Resolver } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import { useTheme } from "@mui/material";
import Stack from "@mui/material/Stack";
Expand Down Expand Up @@ -199,7 +199,7 @@ const CreateMonitorPage = () => {
});

const form = useForm<MonitorFormData>({
resolver: zodResolver(schema),
resolver: zodResolver(schema) as unknown as Resolver<MonitorFormData>,
defaultValues: defaults,
});
const { control, watch, handleSubmit, clearErrors } = form;
Expand Down Expand Up @@ -765,6 +765,92 @@ const CreateMonitorPage = () => {
}
/>

<ConfigBox
title="Escalated Notifications"
subtitle="If the monitor stays down for the specified time, notify additional channels."
rightContent={
<Stack spacing={theme.spacing(LAYOUT.MD)}>
<Controller
name="escalationDelay"
control={control}
render={({ field, fieldState }) => (
<TextField
{...field}
value={field.value ?? 0}
onChange={(e) => field.onChange(Number(e.target.value))}
type="number"
fieldLabel="Escalate after (minutes)"
fullWidth
error={!!fieldState.error}
helperText={fieldState.error?.message ?? ""}
/>
)}
/>
<Typography>Escalation notification channels</Typography>
<Controller
name="escalationNotifications"
control={control}
render={({ field }) => {
const notificationOptions = (notifications ?? []).map((n) => ({
...n,
name: n.notificationName,
}));
const selectedNotifications = notificationOptions.filter((n) =>
(field.value ?? []).includes(n.id)
);
return (
<Stack spacing={theme.spacing(LAYOUT.MD)}>
<Autocomplete
multiple
options={notificationOptions}
value={selectedNotifications}
getOptionLabel={(option) => option.name}
onChange={(_: unknown, newValue: typeof notificationOptions) => {
field.onChange(newValue.map((n) => n.id));
}}
isOptionEqualToValue={(option, value) => option.id === value.id}
/>
{selectedNotifications.length > 0 && (
<Stack
flex={1}
width="100%"
>
{selectedNotifications.map((notification, index) => (
<Stack
direction="row"
alignItems="center"
key={notification.id}
width="100%"
>
<Typography flexGrow={1}>
{notification.notificationName}
</Typography>
<IconButton
size="small"
onClick={() => {
field.onChange(
(field.value ?? []).filter(
(id: string) => id !== notification.id
)
);
}}
aria-label="Remove notification"
>
<Trash2 size={16} />
</IconButton>
{index < selectedNotifications.length - 1 && <Divider />}
</Stack>
))}
</Stack>
)}
</Stack>
);
}}
/>
</Stack>
}
/>

{(watchedType === "http" ||
watchedType === "grpc" ||
watchedType === "websocket") && (
Expand Down
2 changes: 2 additions & 0 deletions client/src/Types/Monitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export interface Monitor {
statusWindow: boolean[];
statusWindowSize: number;
statusWindowThreshold: number;
escalationDelay?: number;
type: MonitorType;
ignoreTlsErrors: boolean;
useAdvancedMatching: boolean;
Expand All @@ -60,6 +61,7 @@ export interface Monitor {
interval: number;
uptimePercentage?: number;
notifications: string[];
escalationNotifications?: string[];
secret?: string;
cpuAlertThreshold: number;
cpuAlertCounter: number;
Expand Down
6 changes: 6 additions & 0 deletions client/src/Validation/monitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ const baseSchema = z.object({
.number({ message: "Threshold percentage is required" })
.min(1, "Incident percentage must be at least 1")
.max(100, "Incident percentage must be at most 100"),
escalationDelay: z
.number()
.min(0, "Escalation delay must be at least 0")
.max(60, "Escalation delay must be at most 60 minutes")
.optional(),
escalationNotifications: z.array(z.string()).default([]),
geoCheckEnabled: z.boolean().optional(),
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
geoCheckInterval: z
Expand Down
3 changes: 3 additions & 0 deletions client/src/locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,9 @@
},
"percentage": {
"label": "What percentage of checks in the sliding window fail/succeed before monitor status changes?"
},
"escalationDelay": {
"label": "Escalation delay (minutes)"
}
},
"title": "Incidents"
Expand Down
27 changes: 27 additions & 0 deletions server/.enves
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# ==============================================
# Checkmate Server Configuration
# ==============================================
# Copy this file to .env and fill in your values

# Server Configuration
# --------------------
NODE_ENV=development
LOG_LEVEL=debug

# Database
# --------
DB_CONNECTION_STRING=mongodb://localhost:27017/uptime_db

# JWT Authentication
# ------------------
JWT_SECRET=my_secret
TOKEN_TTL=99d
CLIENT_HOST=http://localhost:5173

# Client Configuration
# --------------------
CLIENT_HOST=http://localhost:5173

# Optional Configuration
# ----------------------
ORIGIN=localhost
17 changes: 17 additions & 0 deletions server/src/db/models/Monitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@ type MonitorDocumentBase = Omit<
statusWindow: boolean[];
recentChecks: CheckSnapshotDocument[];
notifications: Types.ObjectId[];
escalationNotifications: Types.ObjectId[];
selectedDisks: string[];
matchMethod?: MonitorMatchMethod;
escalationDelay?: number;
_escalationStart?: number;
};

interface MonitorDocument extends MonitorDocumentBase {
Expand Down Expand Up @@ -274,6 +277,20 @@ const MonitorSchema = new Schema<MonitorDocument>(
type: Number,
default: 60000,
},
escalationDelay: {
type: Number,
default: 0,
},
escalationNotifications: [
{
type: Schema.Types.ObjectId,
ref: "Notification",
},
],
_escalationStart: {
type: Number,
default: undefined,
},
uptimePercentage: {
type: Number,
default: undefined,
Expand Down
4 changes: 4 additions & 0 deletions server/src/repositories/monitors/MongoMonitorsRepository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
};

const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification));
const escalationNotificationIds = (doc.escalationNotifications ?? []).map((notification) => toStringId(notification));

return {
id: toStringId(doc._id),
Expand All @@ -374,6 +375,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
interval: doc.interval,
uptimePercentage: doc.uptimePercentage ?? undefined,
notifications: notificationIds,
escalationNotifications: escalationNotificationIds,
secret: doc.secret ?? undefined,
cpuAlertThreshold: doc.cpuAlertThreshold,
cpuAlertCounter: doc.cpuAlertCounter,
Expand Down Expand Up @@ -410,6 +412,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
};

const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification));
const escalationNotificationIds = (doc.escalationNotifications ?? []).map((notification: unknown) => toStringId(notification));

return {
id: toStringId(doc._id),
Expand All @@ -433,6 +436,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
interval: doc.interval,
uptimePercentage: doc.uptimePercentage ?? undefined,
notifications: notificationIds,
escalationNotifications: escalationNotificationIds,
secret: doc.secret ?? undefined,
cpuAlertThreshold: doc.cpuAlertThreshold,
cpuAlertCounter: doc.cpuAlertCounter,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export interface MonitorActionDecision {
shouldSendNotification: boolean;
incidentReason: "status_down" | "threshold_breach" | null;
notificationReason: "status_change" | "threshold_breach" | null;
isEscalation?: boolean;
thresholdBreaches?: {
cpu?: boolean;
memory?: boolean;
Expand Down Expand Up @@ -428,9 +429,19 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
isEscalation: false,
};

if (!statusChanged) {
// Check if monitor is still down and escalation delay has been met
if (monitor.status === "down" && monitor._escalationStart && monitor.escalationDelay) {
const elapsed = Date.now() - monitor._escalationStart;
if (elapsed >= monitor.escalationDelay) {
decision.shouldSendNotification = true;
decision.notificationReason = "status_change";
decision.isEscalation = true;
}
}
return decision;
}

Expand All @@ -451,6 +462,8 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
decision.shouldResolveIncident = true;
decision.shouldSendNotification = true;
decision.notificationReason = "status_change";
// Reset escalation when monitor recovers
monitor._escalationStart = undefined;
}

return decision;
Expand Down
36 changes: 36 additions & 0 deletions server/src/service/infrastructure/notificationMessageBuilder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
}

private determineNotificationType(decision: MonitorActionDecision, monitor: Monitor): NotificationType {
// Check for escalation first
if (decision.isEscalation && monitor.status === "down") {
return "escalation";
}

// Down status has highest priority (critical)
if (monitor.status === "down") {
return "monitor_down";
Expand Down Expand Up @@ -80,6 +85,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
private determineSeverity(type: NotificationType): NotificationSeverity {
switch (type) {
case "monitor_down":
case "escalation":
return "critical";
case "threshold_breach":
return "warning";
Expand All @@ -97,6 +103,8 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
switch (type) {
case "monitor_down":
return this.buildMonitorDownContent(monitor, monitorStatusResponse);
case "escalation":
return this.buildEscalationContent(monitor, monitorStatusResponse);
case "monitor_up":
return this.buildMonitorUpContent(monitor);
case "threshold_breach":
Expand Down Expand Up @@ -131,6 +139,34 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}

private buildEscalationContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent {
const title = `ESCALATION: Monitor Still Down: ${monitor.name}`;
const summary = `Monitor "${monitor.name}" continues to be down. Escalation alert triggered.`;
const details = [
`URL: ${monitor.url}`,
`Status: Down (Escalation Alert)`,
`Type: ${monitor.type}`,
`Escalation Delay: ${monitor.escalationDelay}ms`,
];

// Add response code if available
if (monitorStatusResponse.code) {
details.push(`Response Code: ${monitorStatusResponse.code}`);
}

// Add error message if available
if (monitorStatusResponse.message) {
details.push(`Error: ${monitorStatusResponse.message}`);
}

return {
title,
summary,
details,
timestamp: new Date(),
};
}

private buildMonitorUpContent(monitor: Monitor): NotificationContent {
const title = `Monitor Recovered: ${monitor.name}`;
const summary = `Monitor "${monitor.name}" is back up and operational.`;
Expand Down
4 changes: 3 additions & 1 deletion server/src/service/infrastructure/notificationsService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ export class NotificationsService implements INotificationsService {
};

private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
const notificationIds = monitor.notifications ?? [];
const notificationIds = decision.isEscalation
? (monitor.escalationNotifications?.length ? monitor.escalationNotifications : monitor.notifications ?? [])
: monitor.notifications ?? [];
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);

// Build notification message once for all notifications
Expand Down
Loading