From f350faddfbcf0d715cf8d35c0fbbb04395ff837f Mon Sep 17 00:00:00 2001 From: SebastienLaguerre Date: Fri, 10 Apr 2026 16:19:01 -0400 Subject: [PATCH] Escalation and Notification feature --- client/package-lock.json | 279 +++++++------ client/src/Hooks/useNotificationForm.ts | 2 + client/src/Pages/CreateMonitor/index.tsx | 358 +++++++++++++++- .../src/Pages/Notifications/create/index.tsx | 9 +- client/src/Types/Notification.ts | 10 + client/src/Validation/notifications.ts | 85 +++- client/src/locales/en.json | 10 + server/package-lock.json | 174 ++++---- server/src/business/escalationService.ts | 385 ++++++++++++++++++ server/src/config/services.ts | 7 + .../src/controllers/notificationController.ts | 2 + .../db/migration/0006_fixGoogleDemoMonitor.ts | 31 ++ .../0007_resetGoogleDnsFailureState.ts | 24 ++ server/src/db/migration/index.ts | 4 + server/src/db/models/Incident.ts | 33 +- server/src/db/models/Notification.ts | 27 +- .../incidents/MongoIncidentRepository.ts | 5 + .../monitors/MongoMonitorsRepository.ts | 54 ++- .../MongoNotificationsRepository.ts | 5 + server/src/service/business/monitorService.ts | 7 + .../SuperSimpleQueue/SuperSimpleQueue.ts | 2 + .../SuperSimpleQueueHelper.ts | 42 +- .../infrastructure/network/HttpProvider.ts | 126 ++++-- .../infrastructure/notificationsService.ts | 32 +- server/src/types/incident.ts | 7 + server/src/types/notification.ts | 10 + server/src/utils/demoMonitors.json | 2 +- .../src/validation/notificationValidation.ts | 82 +++- 28 files changed, 1495 insertions(+), 319 deletions(-) create mode 100644 server/src/business/escalationService.ts create mode 100644 server/src/db/migration/0006_fixGoogleDemoMonitor.ts create mode 100644 server/src/db/migration/0007_resetGoogleDnsFailureState.ts diff --git a/client/package-lock.json b/client/package-lock.json index 7462d95d71..f78bcfea3c 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1634,9 +1634,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", - "integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.1.tgz", + "integrity": "sha512-d6FinEBLdIiK+1uACUttJKfgZREXrF0Qc2SmLII7W2AD8FfiZ9Wjd+rD/iRuf5s5dWrr1GgwXCvPqOuDquOowA==", "cpu": [ "arm" ], @@ -1647,9 +1647,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz", - "integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.1.tgz", + "integrity": "sha512-YjG/EwIDvvYI1YvYbHvDz/BYHtkY4ygUIXHnTdLhG+hKIQFBiosfWiACWortsKPKU/+dUwQQCKQM3qrDe8c9BA==", "cpu": [ "arm64" ], @@ -1660,9 +1660,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz", - "integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.1.tgz", + "integrity": "sha512-mjCpF7GmkRtSJwon+Rq1N8+pI+8l7w5g9Z3vWj4T7abguC4Czwi3Yu/pFaLvA3TTeMVjnu3ctigusqWUfjZzvw==", "cpu": [ "arm64" ], @@ -1673,9 +1673,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz", - "integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.1.tgz", + "integrity": "sha512-haZ7hJ1JT4e9hqkoT9R/19XW2QKqjfJVv+i5AGg57S+nLk9lQnJ1F/eZloRO3o9Scy9CM3wQ9l+dkXtcBgN5Ew==", "cpu": [ "x64" ], @@ -1686,9 +1686,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz", - "integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.1.tgz", + "integrity": "sha512-czw90wpQq3ZsAVBlinZjAYTKduOjTywlG7fEeWKUA7oCmpA8xdTkxZZlwNJKWqILlq0wehoZcJYfBvOyhPTQ6w==", "cpu": [ "arm64" ], @@ -1699,9 +1699,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz", - "integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.1.tgz", + "integrity": "sha512-KVB2rqsxTHuBtfOeySEyzEOB7ltlB/ux38iu2rBQzkjbwRVlkhAGIEDiiYnO2kFOkJp+Z7pUXKyrRRFuFUKt+g==", "cpu": [ "x64" ], @@ -1712,12 +1712,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz", - "integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.1.tgz", + "integrity": "sha512-L+34Qqil+v5uC0zEubW7uByo78WOCIrBvci69E7sFASRl0X7b/MB6Cqd1lky/CtcSVTydWa2WZwFuWexjS5o6g==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1725,12 +1728,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz", - "integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.1.tgz", + "integrity": "sha512-n83O8rt4v34hgFzlkb1ycniJh7IR5RCIqt6mz1VRJD6pmhRi0CXdmfnLu9dIUS6buzh60IvACM842Ffb3xd6Gg==", "cpu": [ "arm" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1738,12 +1744,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz", - "integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.1.tgz", + "integrity": "sha512-Nql7sTeAzhTAja3QXeAI48+/+GjBJ+QmAH13snn0AJSNL50JsDqotyudHyMbO2RbJkskbMbFJfIJKWA6R1LCJQ==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1751,12 +1760,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz", - "integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.1.tgz", + "integrity": "sha512-+pUymDhd0ys9GcKZPPWlFiZ67sTWV5UU6zOJat02M1+PiuSGDziyRuI/pPue3hoUwm2uGfxdL+trT6Z9rxnlMA==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1764,12 +1776,15 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz", - "integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.1.tgz", + "integrity": "sha512-VSvgvQeIcsEvY4bKDHEDWcpW4Yw7BtlKG1GUT4FzBUlEKQK0rWHYBqQt6Fm2taXS+1bXvJT6kICu5ZwqKCnvlQ==", "cpu": [ "loong64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1777,12 +1792,15 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz", - "integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.1.tgz", + "integrity": "sha512-4LqhUomJqwe641gsPp6xLfhqWMbQV04KtPp7/dIp0nzPxAkNY1AbwL5W0MQpcalLYk07vaW9Kp1PBhdpZYYcEw==", "cpu": [ "loong64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1790,12 +1808,15 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz", - "integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.1.tgz", + "integrity": "sha512-tLQQ9aPvkBxOc/EUT6j3pyeMD6Hb8QF2BTBnCQWP/uu1lhc9AIrIjKnLYMEroIz/JvtGYgI9dF3AxHZNaEH0rw==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1803,12 +1824,15 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz", - "integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.1.tgz", + "integrity": "sha512-RMxFhJwc9fSXP6PqmAz4cbv3kAyvD1etJFjTx4ONqFP9DkTkXsAMU4v3Vyc5BgzC+anz7nS/9tp4obsKfqkDHg==", "cpu": [ "ppc64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1816,12 +1840,15 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz", - "integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.1.tgz", + "integrity": "sha512-QKgFl+Yc1eEk6MmOBfRHYF6lTxiiiV3/z/BRrbSiW2I7AFTXoBFvdMEyglohPj//2mZS4hDOqeB0H1ACh3sBbg==", "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1829,12 +1856,15 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz", - "integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.1.tgz", + "integrity": "sha512-RAjXjP/8c6ZtzatZcA1RaQr6O1TRhzC+adn8YZDnChliZHviqIjmvFwHcxi4JKPSDAt6Uhf/7vqcBzQJy0PDJg==", "cpu": [ "riscv64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1842,12 +1872,15 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz", - "integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.1.tgz", + "integrity": "sha512-wcuocpaOlaL1COBYiA89O6yfjlp3RwKDeTIA0hM7OpmhR1Bjo9j31G1uQVpDlTvwxGn2nQs65fBFL5UFd76FcQ==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1855,12 +1888,15 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz", - "integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.1.tgz", + "integrity": "sha512-77PpsFQUCOiZR9+LQEFg9GClyfkNXj1MP6wRnzYs0EeWbPcHs02AXu4xuUbM1zhwn3wqaizle3AEYg5aeoohhg==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1868,12 +1904,15 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz", - "integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.1.tgz", + "integrity": "sha512-5cIATbk5vynAjqqmyBjlciMJl1+R/CwX9oLk/EyiFXDWd95KpHdrOJT//rnUl4cUcskrd0jCCw3wpZnhIHdD9w==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1881,9 +1920,9 @@ ] }, "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz", - "integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.1.tgz", + "integrity": "sha512-cl0w09WsCi17mcmWqqglez9Gk8isgeWvoUZ3WiJFYSR3zjBQc2J5/ihSjpl+VLjPqjQ/1hJRcqBfLjssREQILw==", "cpu": [ "x64" ], @@ -1894,9 +1933,9 @@ ] }, "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz", - "integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.1.tgz", + "integrity": "sha512-4Cv23ZrONRbNtbZa37mLSueXUCtN7MXccChtKpUnQNgF010rjrjfHx3QxkS2PI7LqGT5xXyYs1a7LbzAwT0iCA==", "cpu": [ "arm64" ], @@ -1907,9 +1946,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz", - "integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.1.tgz", + "integrity": "sha512-i1okWYkA4FJICtr7KpYzFpRTHgy5jdDbZiWfvny21iIKky5YExiDXP+zbXzm3dUcFpkEeYNHgQ5fuG236JPq0g==", "cpu": [ "arm64" ], @@ -1920,9 +1959,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz", - "integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.1.tgz", + "integrity": "sha512-u09m3CuwLzShA0EYKMNiFgcjjzwqtUMLmuCJLeZWjjOYA3IT2Di09KaxGBTP9xVztWyIWjVdsB2E9goMjZvTQg==", "cpu": [ "ia32" ], @@ -1933,9 +1972,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz", - "integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.1.tgz", + "integrity": "sha512-k+600V9Zl1CM7eZxJgMyTUzmrmhB/0XZnF4pRypKAlAgxmedUA+1v9R+XOFv56W4SlHEzfeMtzujLJD22Uz5zg==", "cpu": [ "x64" ], @@ -1946,9 +1985,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz", - "integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.1.tgz", + "integrity": "sha512-lWMnixq/QzxyhTV6NjQJ4SFo1J6PvOX8vUx5Wb4bBPsEb+8xZ89Bz6kOXpfXj9ak9AHTQVQzlgzBEc1SyM27xQ==", "cpu": [ "x64" ], @@ -2725,7 +2764,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.12", + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", + "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", "dev": true, "license": "MIT", "dependencies": { @@ -2996,9 +3037,9 @@ } }, "node_modules/cosmiconfig/node_modules/yaml": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", - "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", + "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", "license": "ISC", "engines": { "node": ">= 6" @@ -3909,7 +3950,9 @@ } }, "node_modules/flatted": { - "version": "3.3.3", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -5010,7 +5053,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/lodash.merge": { @@ -5157,9 +5202,9 @@ } }, "node_modules/micromatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -5568,7 +5613,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", "engines": { "node": ">=12" @@ -6106,9 +6153,9 @@ "license": "Unlicense" }, "node_modules/rollup": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", - "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", + "version": "4.60.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.1.tgz", + "integrity": "sha512-VmtB2rFU/GroZ4oL8+ZqXgSA38O6GR8KSIvWmEFv63pQ0G6KaBH9s07PO8XTXP4vI+3UJUEypOfjkGfmSBBR0w==", "license": "MIT", "dependencies": { "@types/estree": "1.0.8" @@ -6121,31 +6168,31 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.59.0", - "@rollup/rollup-android-arm64": "4.59.0", - "@rollup/rollup-darwin-arm64": "4.59.0", - "@rollup/rollup-darwin-x64": "4.59.0", - "@rollup/rollup-freebsd-arm64": "4.59.0", - "@rollup/rollup-freebsd-x64": "4.59.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", - "@rollup/rollup-linux-arm-musleabihf": "4.59.0", - "@rollup/rollup-linux-arm64-gnu": "4.59.0", - "@rollup/rollup-linux-arm64-musl": "4.59.0", - "@rollup/rollup-linux-loong64-gnu": "4.59.0", - "@rollup/rollup-linux-loong64-musl": "4.59.0", - "@rollup/rollup-linux-ppc64-gnu": "4.59.0", - "@rollup/rollup-linux-ppc64-musl": "4.59.0", - "@rollup/rollup-linux-riscv64-gnu": "4.59.0", - "@rollup/rollup-linux-riscv64-musl": "4.59.0", - "@rollup/rollup-linux-s390x-gnu": "4.59.0", - "@rollup/rollup-linux-x64-gnu": "4.59.0", - "@rollup/rollup-linux-x64-musl": "4.59.0", - "@rollup/rollup-openbsd-x64": "4.59.0", - "@rollup/rollup-openharmony-arm64": "4.59.0", - "@rollup/rollup-win32-arm64-msvc": "4.59.0", - "@rollup/rollup-win32-ia32-msvc": "4.59.0", - "@rollup/rollup-win32-x64-gnu": "4.59.0", - "@rollup/rollup-win32-x64-msvc": "4.59.0", + "@rollup/rollup-android-arm-eabi": "4.60.1", + "@rollup/rollup-android-arm64": "4.60.1", + "@rollup/rollup-darwin-arm64": "4.60.1", + "@rollup/rollup-darwin-x64": "4.60.1", + "@rollup/rollup-freebsd-arm64": "4.60.1", + "@rollup/rollup-freebsd-x64": "4.60.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.1", + "@rollup/rollup-linux-arm-musleabihf": "4.60.1", + "@rollup/rollup-linux-arm64-gnu": "4.60.1", + "@rollup/rollup-linux-arm64-musl": "4.60.1", + "@rollup/rollup-linux-loong64-gnu": "4.60.1", + "@rollup/rollup-linux-loong64-musl": "4.60.1", + "@rollup/rollup-linux-ppc64-gnu": "4.60.1", + "@rollup/rollup-linux-ppc64-musl": "4.60.1", + "@rollup/rollup-linux-riscv64-gnu": "4.60.1", + "@rollup/rollup-linux-riscv64-musl": "4.60.1", + "@rollup/rollup-linux-s390x-gnu": "4.60.1", + "@rollup/rollup-linux-x64-gnu": "4.60.1", + "@rollup/rollup-linux-x64-musl": "4.60.1", + "@rollup/rollup-openbsd-x64": "4.60.1", + "@rollup/rollup-openharmony-arm64": "4.60.1", + "@rollup/rollup-win32-arm64-msvc": "4.60.1", + "@rollup/rollup-win32-ia32-msvc": "4.60.1", + "@rollup/rollup-win32-x64-gnu": "4.60.1", + "@rollup/rollup-win32-x64-msvc": "4.60.1", "fsevents": "~2.3.2" } }, @@ -7312,9 +7359,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "devOptional": true, "license": "ISC", "bin": { diff --git a/client/src/Hooks/useNotificationForm.ts b/client/src/Hooks/useNotificationForm.ts index dd882db678..d21b7a1637 100644 --- a/client/src/Hooks/useNotificationForm.ts +++ b/client/src/Hooks/useNotificationForm.ts @@ -16,11 +16,13 @@ export const useNotificationForm = ({ data = null }: UseNotificationFormOptions homeserverUrl: data.homeserverUrl || "", roomId: data.roomId || "", accessToken: data.accessToken || "", + escalationRules: data.escalationRules || [], } : { type: (data?.type || "email") as Exclude, notificationName: data?.notificationName || "", address: data?.address || "", + escalationRules: data?.escalationRules || [], }; return { schema: notificationSchema, defaults }; diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 15b76eab36..39f78f279b 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -38,8 +38,110 @@ import { type GamesMap, supportsGeoCheck, } from "@/Types/Monitor"; -import type { Notification } from "@/Types/Notification"; +import { + EscalationRuleChannels, + type EscalationRule, + type EscalationRuleChannel, + type Notification, +} from "@/Types/Notification"; import type { MonitorFormData } from "@/Validation/monitor"; +import type { NotificationFormData } from "@/Validation/notifications"; + +type NotificationOption = Notification & { name: string }; +type EscalationRuleFormValue = EscalationRule; +type NotificationEscalationMap = Record; + +const normalizeEscalationRules = ( + rules: EscalationRule[] | undefined, + type: Notification["type"] +): EscalationRuleFormValue[] => { + if (!EscalationRuleChannels.includes(type as EscalationRuleChannel)) { + return []; + } + + return (rules ?? []).map((rule) => ({ + type: type as EscalationRuleChannel, + delayMinutes: rule.delayMinutes, + trigger: "escalation", + })); +}; + +const areEscalationRulesEqual = ( + left: EscalationRuleFormValue[], + right: EscalationRuleFormValue[] +): boolean => { + if (left.length !== right.length) { + return false; + } + + return left.every((rule, index) => { + const candidate = right[index]; + return ( + candidate?.type === rule.type && + candidate?.delayMinutes === rule.delayMinutes && + candidate?.trigger === rule.trigger + ); + }); +}; + +const buildNotificationPayload = ( + notification: Notification, + escalationRules: EscalationRuleFormValue[] +): NotificationFormData => { + switch (notification.type) { + case "email": + return { + type: "email", + notificationName: notification.notificationName, + address: notification.address ?? "", + escalationRules, + }; + case "slack": + return { + type: "slack", + notificationName: notification.notificationName, + address: notification.address ?? "", + escalationRules, + }; + case "discord": + return { + type: "discord", + notificationName: notification.notificationName, + address: notification.address ?? "", + escalationRules, + }; + case "webhook": + return { + type: "webhook", + notificationName: notification.notificationName, + address: notification.address ?? "", + escalationRules, + }; + case "pager_duty": + return { + type: "pager_duty", + notificationName: notification.notificationName, + address: notification.address ?? "", + escalationRules: [], + }; + case "matrix": + return { + type: "matrix", + notificationName: notification.notificationName, + homeserverUrl: notification.homeserverUrl ?? "", + roomId: notification.roomId ?? "", + accessToken: notification.accessToken ?? "", + escalationRules: [], + }; + case "teams": + return { + type: "teams", + notificationName: notification.notificationName, + address: notification.address ?? "", + escalationRules: [], + }; + } +}; interface GeneralSettingsConfig { urlLabel: string; @@ -224,11 +326,35 @@ const CreateMonitorPage = () => { const { post, loading: isCreating } = usePost(); const { patch, loading: isUpdating } = usePatch(); - const isSubmitting = isCreating || isUpdating; + const { patch: patchNotification, loading: isUpdatingNotifications } = + usePatch(); + const isSubmitting = isCreating || isUpdating || isUpdatingNotifications; + const [notificationEscalations, setNotificationEscalations] = + useState({}); // Delete functionality const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false); const { deleteFn, loading: isDeleting } = useDelete(); + useEffect(() => { + if (!notifications?.length) { + return; + } + + setNotificationEscalations((current) => { + const next = { ...current }; + for (const notification of notifications) { + if (next[notification.id]) { + continue; + } + next[notification.id] = normalizeEscalationRules( + notification.escalationRules, + notification.type + ); + } + return next; + }); + }, [notifications]); + const handleDeleteClick = () => { setIsDeleteDialogOpen(true); }; @@ -251,6 +377,99 @@ const CreateMonitorPage = () => { setIsDeleteDialogOpen(false); }; + const syncNotificationEscalations = async (notificationIds: string[]) => { + if (!notifications?.length || notificationIds.length === 0) { + return true; + } + + const notificationsById = new Map( + notifications.map((notification) => [notification.id, notification]) + ); + + for (const notificationId of notificationIds) { + const notification = notificationsById.get(notificationId); + if (!notification) { + continue; + } + + const nextRules = normalizeEscalationRules( + notificationEscalations[notificationId] ?? notification.escalationRules, + notification.type + ); + const currentRules = normalizeEscalationRules( + notification.escalationRules, + notification.type + ); + + if (areEscalationRulesEqual(currentRules, nextRules)) { + continue; + } + + const result = await patchNotification( + `/notifications/${notification.id}`, + buildNotificationPayload(notification, nextRules) + ); + if (!result?.success) { + return false; + } + } + + return true; + }; + + const updateEscalationRules = ( + notificationId: string, + updater: (rules: EscalationRuleFormValue[]) => EscalationRuleFormValue[] + ) => { + setNotificationEscalations((current) => ({ + ...current, + [notificationId]: updater(current[notificationId] ?? []), + })); + }; + + const handleAddEscalationStep = (notification: NotificationOption) => { + if (!EscalationRuleChannels.includes(notification.type as EscalationRuleChannel)) { + return; + } + + updateEscalationRules(notification.id, (rules) => { + const previousDelay = rules[rules.length - 1]?.delayMinutes ?? 0; + return [ + ...rules, + { + type: notification.type as EscalationRuleChannel, + delayMinutes: previousDelay + 5, + trigger: "escalation", + }, + ]; + }); + }; + + const handleRemoveEscalationStep = (notificationId: string, index: number) => { + updateEscalationRules(notificationId, (rules) => + rules.filter((_, ruleIndex) => ruleIndex !== index) + ); + }; + + const handleEscalationDelayChange = ( + notification: NotificationOption, + index: number, + delayMinutes: number + ) => { + updateEscalationRules(notification.id, (rules) => + rules.map((rule, ruleIndex) => + ruleIndex === index + ? { + ...rule, + type: notification.type as EscalationRuleChannel, + delayMinutes, + trigger: "escalation", + } + : rule + ) + ); + }; + const onSubmit = async (data: MonitorFormData) => { let result; if (isEditMode && monitorId) { @@ -260,6 +479,13 @@ const CreateMonitorPage = () => { } if (result?.success) { + const escalationsSaved = await syncNotificationEscalations( + data.notifications ?? [] + ); + if (!escalationsSaved) { + return; + } + if (pageType === "pagespeed") { navigate("/pagespeed"); } else if (pageType === "hardware") { @@ -706,7 +932,7 @@ const CreateMonitorPage = () => { control={control} render={({ field }) => { // Map notifications to have 'name' property for Autocomplete - const notificationOptions = (notifications ?? []).map((n) => ({ + const notificationOptions: NotificationOption[] = (notifications ?? []).map((n) => ({ ...n, name: n.notificationName, })); @@ -732,27 +958,119 @@ const CreateMonitorPage = () => { > {selectedNotifications.map((notification, index) => ( - - {notification.notificationName} - - { - field.onChange( - (field.value ?? []).filter( - (id: string) => id !== notification.id - ) - ); - }} - aria-label="Remove notification" + - - + + {notification.notificationName} + + {notification.type} + + + { + field.onChange( + (field.value ?? []).filter( + (id: string) => id !== notification.id + ) + ); + }} + aria-label={t("common.buttons.delete")} + > + + + + {EscalationRuleChannels.includes( + notification.type as EscalationRuleChannel + ) ? ( + + + + {t("pages.notifications.form.escalation.title")} + + + {t("pages.notifications.form.escalation.description")} + + + {(notificationEscalations[notification.id] ?? []).map( + (rule, ruleIndex) => ( + + + handleEscalationDelayChange( + notification, + ruleIndex, + Number(event.target.value) + ) + } + /> + + + + ) + )} + + + {t("pages.notifications.form.escalation.helper")} + + + + + ) : ( + + + {t("pages.notifications.form.escalation.title")} + + + {t("pages.notifications.form.escalation.unsupported")} + + + )} {index < selectedNotifications.length - 1 && } ))} diff --git a/client/src/Pages/Notifications/create/index.tsx b/client/src/Pages/Notifications/create/index.tsx index 7aa7bc831b..407274f839 100644 --- a/client/src/Pages/Notifications/create/index.tsx +++ b/client/src/Pages/Notifications/create/index.tsx @@ -12,7 +12,7 @@ import { Controller, useForm } from "react-hook-form"; import { zodResolver } from "@hookform/resolvers/zod"; import { useGet, usePost, usePatch } from "@/Hooks/UseApi"; import { useNotificationForm } from "@/Hooks/useNotificationForm"; -import type { NotificationFormData } from "@/Validation/notifications"; +import type { NotificationFormData, NotificationFormInput } from "@/Validation/notifications"; import type { Notification } from "@/Types/Notification"; import { useTranslation } from "react-i18next"; import { NotificationChannels } from "@/Types/Notification"; @@ -34,7 +34,7 @@ const NotificationsCreatePage = () => { const { schema, defaults } = useNotificationForm({ data: existingNotification }); - const form = useForm({ + const form = useForm({ resolver: zodResolver(schema), defaultValues: defaults, }); @@ -89,7 +89,10 @@ const NotificationsCreatePage = () => { const isValid = await trigger(); if (!isValid) return; const data = getValues(); - await testPost("/notifications/test", data); + await testPost("/notifications/test", { + ...data, + escalationRules: data.escalationRules ?? [], + } as NotificationFormData); }; return ( diff --git a/client/src/Types/Notification.ts b/client/src/Types/Notification.ts index 610f73547d..cb47a954e7 100644 --- a/client/src/Types/Notification.ts +++ b/client/src/Types/Notification.ts @@ -9,6 +9,15 @@ export const NotificationChannels = [ ] as const; export type NotificationChannel = (typeof NotificationChannels)[number]; +export const EscalationRuleChannels = ["email", "slack", "discord", "webhook"] as const; +export type EscalationRuleChannel = (typeof EscalationRuleChannels)[number]; + +export interface EscalationRule { + type: EscalationRuleChannel; + delayMinutes: number; + trigger: "escalation"; +} + export interface Notification { id: string; userId: string; @@ -20,6 +29,7 @@ export interface Notification { homeserverUrl?: string; roomId?: string; accessToken?: string; + escalationRules: EscalationRule[]; createdAt: string; updatedAt: string; } diff --git a/client/src/Validation/notifications.ts b/client/src/Validation/notifications.ts index 2bbb871c94..1231e34e6d 100644 --- a/client/src/Validation/notifications.ts +++ b/client/src/Validation/notifications.ts @@ -1,5 +1,59 @@ import { z } from "zod"; +const escalationRuleChannels = ["email", "slack", "discord", "webhook"] as const; + +const escalationRuleSchema = z.object({ + type: z.enum(escalationRuleChannels), + delayMinutes: z.number().int("Delay must be a whole number").positive("Delay must be a positive integer"), + trigger: z.literal("escalation"), +}); + +type EscalationRuleFormInput = z.infer; +type EscalationValidationInput = { + type: string; + escalationRules?: EscalationRuleFormInput[]; +}; + +const withEscalationRules = (schema: z.ZodObject) => + schema + .extend({ + escalationRules: z.array(escalationRuleSchema).default([]), + }) + .superRefine((data, ctx) => { + const normalizedData = data as EscalationValidationInput; + const rules = normalizedData.escalationRules ?? []; + if (rules.length === 0) { + return; + } + + if (!escalationRuleChannels.includes(normalizedData.type as (typeof escalationRuleChannels)[number])) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Escalation rules are only supported for email, slack, discord, and webhook notifications", + path: ["escalationRules"], + }); + } + + for (const [index, rule] of rules.entries()) { + if (rule.type !== normalizedData.type) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Escalation rule type must match the notification type ${normalizedData.type}`, + path: ["escalationRules", index, "type"], + }); + } + + const previousRule = rules[index - 1]; + if (previousRule && rule.delayMinutes <= previousRule.delayMinutes) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Escalation delays must be sorted in strictly increasing order with no duplicates", + path: ["escalationRules", index, "delayMinutes"], + }); + } + } + }); + const baseSchema = z.object({ notificationName: z .string() @@ -7,35 +61,35 @@ const baseSchema = z.object({ .max(100, "Notification name must be at most 100 characters"), }); -const emailSchema = baseSchema.extend({ +const emailSchema = withEscalationRules(baseSchema.extend({ type: z.literal("email"), address: z .string() .min(1, "Email is required") .email("Please enter a valid email address"), -}); +})); -const slackSchema = baseSchema.extend({ +const slackSchema = withEscalationRules(baseSchema.extend({ type: z.literal("slack"), address: z.string().min(1, "Webhook URL is required").url("Please enter a valid URL"), -}); +})); -const discordSchema = baseSchema.extend({ +const discordSchema = withEscalationRules(baseSchema.extend({ type: z.literal("discord"), address: z.string().min(1, "Webhook URL is required").url("Please enter a valid URL"), -}); +})); -const webhookSchema = baseSchema.extend({ +const webhookSchema = withEscalationRules(baseSchema.extend({ type: z.literal("webhook"), address: z.string().min(1, "Webhook URL is required").url("Please enter a valid URL"), -}); +})); -const pagerDutySchema = baseSchema.extend({ +const pagerDutySchema = withEscalationRules(baseSchema.extend({ type: z.literal("pager_duty"), address: z.string().min(1, "Integration key is required"), -}); +})); -const matrixSchema = baseSchema.extend({ +const matrixSchema = withEscalationRules(baseSchema.extend({ type: z.literal("matrix"), homeserverUrl: z .string() @@ -43,12 +97,12 @@ const matrixSchema = baseSchema.extend({ .url("Please enter a valid URL"), roomId: z.string().min(1, "Room ID is required"), accessToken: z.string().min(1, "Access token is required"), -}); +})); -const teamsSchema = baseSchema.extend({ +const teamsSchema = withEscalationRules(baseSchema.extend({ type: z.literal("teams"), address: z.string().min(1, "Webhook URL is required").url("Please enter a valid URL"), -}); +})); export const notificationSchema = z.discriminatedUnion("type", [ emailSchema, @@ -60,4 +114,5 @@ export const notificationSchema = z.discriminatedUnion("type", [ teamsSchema, ]); -export type NotificationFormData = z.infer; +export type NotificationFormInput = z.input; +export type NotificationFormData = z.output; diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 92a21939f3..9e3e243469 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -931,6 +931,16 @@ "description": "Configure your Matrix homeserver connection for notifications.", "title": "Matrix configuration" }, + "escalation": { + "addStep": "Add step", + "channelLabel": "Channel type", + "delayLabel": "Delay in minutes", + "deleteStep": "Delete escalation step", + "description": "Add delayed escalation steps for incidents that remain active.", + "helper": "Escalation steps are sent after the incident has remained open for the configured number of minutes.", + "title": "Escalation steps", + "unsupported": "Escalation steps are currently available only for email, slack, discord, and webhook channels." + }, "notificationName": { "description": "A descriptive name for the notification channel", "optionName": "Channel name", diff --git a/server/package-lock.json b/server/package-lock.json index 5d9d920ee5..106bf86cfa 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -799,40 +799,20 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.10.tgz", - "integrity": "sha512-OnejAIVD+CxzyAUrVic7lG+3QRltyja9LoNqCE/1YVs8ichoTbJlVSaZ9iSMcnHLyzrSNtvaOGjSDRP+d/ouFA==", + "version": "3.972.17", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.17.tgz", + "integrity": "sha512-Ra7hjqAZf1OXRRMueB13qex7mFJRDK/pgCvdSFemXBT8KCGnQDPoKzHY1SjN+TjJVmnpSF14W5tJ1vDamFu+Gg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", - "fast-xml-parser": "5.4.1", + "@smithy/types": "^4.14.0", + "fast-xml-parser": "5.5.8", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/xml-builder/node_modules/fast-xml-parser": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.4.1.tgz", - "integrity": "sha512-BQ30U1mKkvXQXXkAGcuyUA/GA26oEB7NzOtsxCDtyu62sjGw5QraKFhx2Em3WQNjPw9PG6MQ9yuIIgkSDfGu5A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "fast-xml-builder": "^1.0.0", - "strnum": "^2.1.2" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/@aws/lambda-invoke-store": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.3.tgz", @@ -3850,9 +3830,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.13.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.13.0.tgz", - "integrity": "sha512-COuLsZILbbQsdrwKQpkkpyep7lCsByxwj7m0Mg5v66/ZTyenlfBc40/QFQ5chO0YN/PNEH1Bi3fGtfXPnYNeDw==", + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.14.0.tgz", + "integrity": "sha512-OWgntFLW88kx2qvf/c/67Vno1yuXm/f9M7QFAtVkkO29IJXGBIg0ycEaBTH0kvCtwmvZxRujrgP5a86RvsXJAQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -4942,9 +4922,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { @@ -5884,9 +5864,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", + "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", "dev": true, "license": "MIT", "dependencies": { @@ -8127,9 +8107,9 @@ "license": "MIT" }, "node_modules/fast-xml-builder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.0.0.tgz", - "integrity": "sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", + "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", "dev": true, "funding": [ { @@ -8137,7 +8117,31 @@ "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT" + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.5.8", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.8.tgz", + "integrity": "sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "fast-xml-builder": "^1.1.4", + "path-expression-matcher": "^1.2.0", + "strnum": "^2.2.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } }, "node_modules/fastq": { "version": "1.20.1", @@ -8278,9 +8282,9 @@ } }, "node_modules/flatted": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", - "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -8727,9 +8731,9 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -8829,9 +8833,9 @@ "license": "ISC" }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "license": "MIT", "dependencies": { "minimist": "^1.2.5", @@ -10130,9 +10134,9 @@ } }, "node_modules/jest-util/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -10667,9 +10671,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/lodash.camelcase": { @@ -11211,9 +11215,9 @@ } }, "node_modules/mjml-cli/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -12307,6 +12311,22 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.4.0.tgz", + "integrity": "sha512-s4DQMxIdhj3jLFWd9LxHOplj4p9yQ4ffMGowFf3cpEgrrJjEhN0V5nxw4Ye1EViAGDoL4/1AeO6qHpqYPOzE4Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -12343,9 +12363,9 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", "license": "MIT" }, "node_modules/path-type": { @@ -12365,9 +12385,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "license": "MIT", "engines": { "node": ">=8.6" @@ -14344,9 +14364,9 @@ } }, "node_modules/strnum": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.0.tgz", - "integrity": "sha512-Y7Bj8XyJxnPAORMZj/xltsfo55uOiyHcU2tnAVzHUnSJR/KsEX+9RoDeXEnsXtl/CX4fAcrt64gZ13aGaWPeBg==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", + "integrity": "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==", "dev": true, "funding": [ { @@ -14532,9 +14552,9 @@ } }, "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "dev": true, "license": "MIT", "dependencies": { @@ -14599,9 +14619,9 @@ } }, "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -14987,9 +15007,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.23.0.tgz", - "integrity": "sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==", + "version": "6.24.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.24.1.tgz", + "integrity": "sha512-sC+b0tB1whOCzbtlx20fx3WgCXwkW627p4EA9uM+/tNNPkSS+eSEld6pAs9nDv7WbY1UUljBMYPtu9BCOrCWKA==", "license": "MIT", "engines": { "node": ">=18.17" @@ -15657,9 +15677,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "dev": true, "license": "ISC", "bin": { diff --git a/server/src/business/escalationService.ts b/server/src/business/escalationService.ts new file mode 100644 index 0000000000..56c72a7b7f --- /dev/null +++ b/server/src/business/escalationService.ts @@ -0,0 +1,385 @@ +import mongoose from "mongoose"; +import IncidentModel from "@/db/models/Incident.js"; +import MonitorModel from "@/db/models/Monitor.js"; +import NotificationModel from "@/db/models/Notification.js"; +import type { IncidentDocument } from "@/db/models/Incident.js"; +import type { NotificationDocument } from "@/db/models/Notification.js"; +import type { NotificationMessage } from "@/types/notificationMessage.js"; +import type { INotificationsService } from "@/service/infrastructure/notificationsService.js"; +import type { ISettingsService } from "@/service/system/settingsService.js"; +import type { ILogger } from "@/utils/logger.js"; +import type { Notification } from "@/types/notification.js"; + +const SERVICE_NAME = "escalationService"; + +type EscalationDependencies = { + notificationsService: Pick; + settingsService: ISettingsService; + logger: ILogger; +}; + +type ProcessEscalationsOptions = { + monitorIds?: string[]; + notificationIds?: string[]; + catchUpBaseNotification?: boolean; +}; + +let escalationDependencies: EscalationDependencies | null = null; + +export const initializeEscalationService = (dependencies: EscalationDependencies) => { + escalationDependencies = dependencies; +}; + +const toObjectIds = (values: string[] | undefined): mongoose.Types.ObjectId[] => { + if (!values?.length) { + return []; + } + + return values.flatMap((value) => { + try { + return [new mongoose.Types.ObjectId(value)]; + } catch { + return []; + } + }); +}; + +const toStringId = (value?: mongoose.Types.ObjectId | string | null): string => { + if (!value) { + return ""; + } + return value instanceof mongoose.Types.ObjectId ? value.toString() : String(value); +}; + +const mapNotification = (doc: NotificationDocument): Notification => { + const toDateString = (value?: Date | string | null): string => { + if (!value) { + return new Date(0).toISOString(); + } + return value instanceof Date ? value.toISOString() : String(value); + }; + + return { + id: toStringId(doc._id), + userId: toStringId(doc.userId), + teamId: toStringId(doc.teamId), + type: doc.type, + notificationName: doc.notificationName, + address: doc.address ?? undefined, + phone: doc.phone ?? undefined, + homeserverUrl: doc.homeserverUrl ?? undefined, + roomId: doc.roomId ?? undefined, + accessToken: doc.accessToken ?? undefined, + escalationRules: (doc.escalationRules ?? []).map((rule: NotificationDocument["escalationRules"][number]) => ({ + type: rule.type, + delayMinutes: rule.delayMinutes, + trigger: rule.trigger, + })), + createdAt: toDateString(doc.createdAt), + updatedAt: toDateString(doc.updatedAt), + }; +}; + +const hasNotificationLog = (incident: IncidentDocument, notificationId: string, delayMinutes: number): boolean => { + return (incident.escalationLogs ?? []).some( + (log) => toStringId(log.notificationId) === notificationId && log.delayMinutes === delayMinutes + ); +}; + +const buildActiveIncidentCatchUpMessage = ({ + monitor, + incident, + clientHost, +}: { + monitor: { id: string; name?: string | null; url?: string | null; type?: string | null; status?: string | null; teamId?: string | null }; + incident: IncidentDocument; + clientHost: string; +}): NotificationMessage => { + const isThresholdIncident = incident.statusCode === 9999; + const title = isThresholdIncident ? `Threshold Exceeded: ${monitor.name}` : `Monitor Down: ${monitor.name}`; + const summary = isThresholdIncident + ? `Monitor "${monitor.name}" still has an active threshold breach.` + : `Monitor "${monitor.name}" currently has an active downtime incident.`; + const details = [ + `Monitor URL: ${monitor.url ?? "Unknown"}`, + `Incident started at: ${incident.createdAt instanceof Date ? incident.createdAt.toISOString() : String(incident.createdAt)}`, + ]; + + if (incident.message) { + details.push(`Incident message: ${incident.message}`); + } + + if (incident.statusCode) { + details.push(`Status code: ${incident.statusCode}`); + } + + return { + type: isThresholdIncident ? "threshold_breach" : "monitor_down", + severity: isThresholdIncident ? "warning" : "critical", + monitor: { + id: monitor.id, + name: monitor.name ?? "Unknown monitor", + url: monitor.url ?? "", + type: monitor.type ?? "unknown", + status: monitor.status ?? (isThresholdIncident ? "breached" : "down"), + }, + content: { + title, + summary, + details, + incident: { + id: toStringId(incident._id), + url: `${clientHost}/incidents`, + createdAt: incident.createdAt, + }, + timestamp: new Date(), + }, + clientHost, + metadata: { + teamId: monitor.teamId ?? "", + notificationReason: "active_incident_catchup", + }, + }; +}; + +const buildEscalationMessage = ({ + monitor, + incident, + clientHost, + delayMinutes, + elapsedMinutes, +}: { + monitor: { id: string; name?: string | null; url?: string | null; type?: string | null; status?: string | null; teamId?: string | null }; + incident: IncidentDocument; + clientHost: string; + delayMinutes: number; + elapsedMinutes: number; +}): NotificationMessage => { + const isThresholdIncident = incident.statusCode === 9999; + const title = isThresholdIncident ? `Escalation: Threshold Exceeded for ${monitor.name}` : `Escalation: Monitor Down for ${monitor.name}`; + const summary = isThresholdIncident + ? `Incident for \"${monitor.name}\" is still active ${elapsedMinutes} minute(s) after it started.` + : `Monitor \"${monitor.name}\" is still down ${elapsedMinutes} minute(s) after it went offline.`; + const details = [ + `Escalation delay reached: ${delayMinutes} minute(s)`, + `Elapsed since incident creation: ${elapsedMinutes} minute(s)`, + `Monitor URL: ${monitor.url ?? "Unknown"}`, + ]; + + if (incident.message) { + details.push(`Incident message: ${incident.message}`); + } + + if (incident.statusCode) { + details.push(`Status code: ${incident.statusCode}`); + } + + return { + type: isThresholdIncident ? "threshold_breach" : "monitor_down", + severity: isThresholdIncident ? "warning" : "critical", + monitor: { + id: monitor.id, + name: monitor.name ?? "Unknown monitor", + url: monitor.url ?? "", + type: monitor.type ?? "unknown", + status: monitor.status ?? "down", + }, + content: { + title, + summary, + details, + incident: { + id: toStringId(incident._id), + url: `${clientHost}/incidents`, + createdAt: incident.createdAt, + duration: `${elapsedMinutes} minute(s)`, + }, + timestamp: new Date(), + }, + clientHost, + metadata: { + teamId: monitor.teamId ?? "", + notificationReason: "escalation", + }, + }; +}; + +export async function processEscalations(options: ProcessEscalationsOptions = {}): Promise { + if (!escalationDependencies) { + throw new Error("Escalation service has not been initialized"); + } + + const { notificationsService, settingsService, logger } = escalationDependencies; + const clientHost = settingsService.getSettings().clientHost || "Host not defined"; + const now = Date.now(); + const monitorObjectIds = toObjectIds(options.monitorIds); + const notificationObjectIds = toObjectIds(options.notificationIds); + const incidentQuery: Record = { + status: true, + $or: [{ endTime: null }, { endTime: { $exists: false } }], + }; + + if (monitorObjectIds.length > 0) { + incidentQuery.monitorId = { $in: monitorObjectIds }; + } + + const incidents = await IncidentModel.find(incidentQuery).lean(); + + for (const incident of incidents) { + try { + const createdAt = incident.createdAt instanceof Date ? incident.createdAt : new Date(incident.createdAt); + const elapsedMs = now - createdAt.getTime(); + if (elapsedMs < 0) { + continue; + } + + const elapsedMinutes = Math.floor(elapsedMs / (1000 * 60)); + const monitor = await MonitorModel.findById(incident.monitorId).lean(); + if (!monitor || !Array.isArray(monitor.notifications) || monitor.notifications.length === 0) { + continue; + } + + const applicableNotificationIds = + notificationObjectIds.length > 0 + ? monitor.notifications.filter((notificationId) => + notificationObjectIds.some((candidate) => candidate.equals(notificationId)) + ) + : monitor.notifications; + + if (applicableNotificationIds.length === 0) { + continue; + } + + const notifications = await NotificationModel.find({ _id: { $in: applicableNotificationIds } }).lean(); + if (notifications.length === 0) { + continue; + } + + for (const notificationDoc of notifications) { + const sentLogs = incident.escalationLogs ?? []; + const notificationId = toStringId(notificationDoc._id); + const shouldCatchUpBaseNotification = + options.catchUpBaseNotification === true && !hasNotificationLog(incident, notificationId, 0); + const pendingRules = (notificationDoc.escalationRules ?? []).filter((rule) => { + if (rule.trigger !== "escalation") { + return false; + } + if (rule.delayMinutes > elapsedMinutes) { + return false; + } + if (rule.type !== notificationDoc.type) { + logger.warn({ + message: `Skipping escalation rule ${rule.delayMinutes} for notification ${notificationId} because rule type ${rule.type} does not match notification type ${notificationDoc.type}`, + service: SERVICE_NAME, + method: "processEscalations", + }); + return false; + } + return !sentLogs.some((log) => toStringId(log.notificationId) === notificationId && log.delayMinutes === rule.delayMinutes); + }); + + if (pendingRules.length === 0) { + if (!shouldCatchUpBaseNotification) { + continue; + } + } + + const notification = mapNotification(notificationDoc); + + if (shouldCatchUpBaseNotification) { + const catchUpMessage = buildActiveIncidentCatchUpMessage({ + monitor: { + id: toStringId(monitor._id), + name: monitor.name, + url: monitor.url, + type: monitor.type, + status: monitor.status, + teamId: toStringId(monitor.teamId), + }, + incident, + clientHost, + }); + + const sentCatchUp = await notificationsService.sendNotificationMessage(notification, catchUpMessage); + if (sentCatchUp) { + await IncidentModel.updateOne( + { + _id: incident._id, + escalationLogs: { + $not: { + $elemMatch: { + notificationId: notificationDoc._id, + delayMinutes: 0, + }, + }, + }, + }, + { + $push: { + escalationLogs: { + notificationId: notificationDoc._id, + delayMinutes: 0, + sentAt: new Date(), + }, + }, + } + ); + } + } + + for (const rule of pendingRules) { + const message = buildEscalationMessage({ + monitor: { + id: toStringId(monitor._id), + name: monitor.name, + url: monitor.url, + type: monitor.type, + status: monitor.status, + teamId: toStringId(monitor.teamId), + }, + incident, + clientHost, + delayMinutes: rule.delayMinutes, + elapsedMinutes, + }); + + const sent = await notificationsService.sendNotificationMessage(notification, message); + if (!sent) { + continue; + } + + await IncidentModel.updateOne( + { + _id: incident._id, + escalationLogs: { + $not: { + $elemMatch: { + notificationId: notificationDoc._id, + delayMinutes: rule.delayMinutes, + }, + }, + }, + }, + { + $push: { + escalationLogs: { + notificationId: notificationDoc._id, + delayMinutes: rule.delayMinutes, + sentAt: new Date(), + }, + }, + } + ); + } + } + } catch (error: unknown) { + logger.error({ + message: error instanceof Error ? error.message : "Unknown escalation processing error", + service: SERVICE_NAME, + method: "processEscalations", + details: { incidentId: toStringId(incident._id) }, + stack: error instanceof Error ? error.stack : undefined, + }); + } + } +} \ No newline at end of file diff --git a/server/src/config/services.ts b/server/src/config/services.ts index b31c8a5e91..e0f1829a9a 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -110,6 +110,7 @@ import { IMaintenanceWindowsRepository, } from "@/repositories/index.js"; import { ILogger } from "@/utils/logger.js"; +import { initializeEscalationService } from "@/business/escalationService.js"; export type InitializedServices = { settingsService: ISettingsService; @@ -246,6 +247,12 @@ export const initializeServices = async ({ notificationMessageBuilder ); + initializeEscalationService({ + notificationsService, + settingsService, + logger, + }); + const superSimpleQueueHelper = new SuperSimpleQueueHelper( logger, networkService, diff --git a/server/src/controllers/notificationController.ts b/server/src/controllers/notificationController.ts index 0a49cbdb12..19e3268aa4 100644 --- a/server/src/controllers/notificationController.ts +++ b/server/src/controllers/notificationController.ts @@ -12,6 +12,7 @@ import { AppError } from "@/utils/AppError.js"; import { INotificationsService } from "@/service/index.js"; import { requireTeamId, requireUserId } from "./controllerUtils.js"; import { IMonitorsRepository } from "@/repositories/index.js"; +import { processEscalations } from "@/business/escalationService.js"; const SERVICE_NAME = "NotificationController"; @@ -125,6 +126,7 @@ class NotificationController implements INotificationController { const notificationId = validatedParams.id; const editedNotification = await this.notificationsService.updateById(notificationId, teamId, validatedBody); + await processEscalations({ notificationIds: [notificationId], catchUpBaseNotification: true }); return res.status(200).json({ success: true, msg: "Notification updated successfully", diff --git a/server/src/db/migration/0006_fixGoogleDemoMonitor.ts b/server/src/db/migration/0006_fixGoogleDemoMonitor.ts new file mode 100644 index 0000000000..de573481dc --- /dev/null +++ b/server/src/db/migration/0006_fixGoogleDemoMonitor.ts @@ -0,0 +1,31 @@ +import { MonitorModel } from "@/db/models/index.js"; + +const GOOGLE_DEMO_MONITOR_NAME = "Google"; +const GOOGLE_DEMO_MONITOR_URL = "https://www.google.com/generate_204"; +const LEGACY_GOOGLE_DEMO_URLS = [ + "https://www.google.com", + "https://www.google.com/", + "https://google.com", + "https://google.com/", + GOOGLE_DEMO_MONITOR_URL, +] as const; + +export async function fixGoogleDemoMonitor(): Promise { + await MonitorModel.updateMany( + { + name: GOOGLE_DEMO_MONITOR_NAME, + description: GOOGLE_DEMO_MONITOR_NAME, + type: "http", + url: { $in: [...LEGACY_GOOGLE_DEMO_URLS] }, + }, + { + $set: { + url: GOOGLE_DEMO_MONITOR_URL, + status: "initializing", + statusWindow: [], + recentChecks: [], + uptimePercentage: undefined, + }, + } + ); +} \ No newline at end of file diff --git a/server/src/db/migration/0007_resetGoogleDnsFailureState.ts b/server/src/db/migration/0007_resetGoogleDnsFailureState.ts new file mode 100644 index 0000000000..de48473609 --- /dev/null +++ b/server/src/db/migration/0007_resetGoogleDnsFailureState.ts @@ -0,0 +1,24 @@ +import { MonitorModel } from "@/db/models/index.js"; + +export async function resetGoogleDnsFailureState(): Promise { + await MonitorModel.updateMany( + { + type: "http", + url: /google\.com/i, + status: "down", + recentChecks: { + $elemMatch: { + message: /EDESTRUCTION/i, + }, + }, + }, + { + $set: { + status: "initializing", + statusWindow: [], + recentChecks: [], + uptimePercentage: undefined, + }, + } + ); +} \ No newline at end of file diff --git a/server/src/db/migration/index.ts b/server/src/db/migration/index.ts index f1a672af9a..f9b49a99c9 100644 --- a/server/src/db/migration/index.ts +++ b/server/src/db/migration/index.ts @@ -4,6 +4,8 @@ import { cleanupDuplicateMonitorStats } from "./0003_cleanupDuplicateMonitorStat import { fixInfrastructureThresholds } from "./0004_fixInfrastructureThresholds.js"; import MigrationModel from "../models/Migration.js"; import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js"; +import { fixGoogleDemoMonitor } from "./0006_fixGoogleDemoMonitor.js"; +import { resetGoogleDnsFailureState } from "./0007_resetGoogleDnsFailureState.js"; import type { ILogger } from "@/utils/logger.js"; type MigrationEntry = { @@ -17,6 +19,8 @@ const migrations: MigrationEntry[] = [ { name: "0003_cleanupDuplicateMonitorStats", execute: cleanupDuplicateMonitorStats }, { name: "0004_fixInfrastructureThresholds", execute: fixInfrastructureThresholds }, { name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray }, + { name: "0006_fixGoogleDemoMonitor", execute: fixGoogleDemoMonitor }, + { name: "0007_resetGoogleDnsFailureState", execute: resetGoogleDnsFailureState }, ]; const runMigrations = async (logger?: ILogger) => { diff --git a/server/src/db/models/Incident.ts b/server/src/db/models/Incident.ts index 82e2b5eb2b..6b43aad30a 100644 --- a/server/src/db/models/Incident.ts +++ b/server/src/db/models/Incident.ts @@ -1,12 +1,19 @@ import { Schema, model, type Types } from "mongoose"; import { IncidentResolutionTypes, type Incident } from "@/types/incident.js"; -type IncidentDocumentBase = Omit & { +type EscalationLogDocument = { + notificationId: Types.ObjectId; + delayMinutes: number; + sentAt: Date; +}; + +type IncidentDocumentBase = Omit & { monitorId: Types.ObjectId; teamId: Types.ObjectId; resolvedBy?: Types.ObjectId | null; startTime: Date; endTime: Date | null; + escalationLogs: EscalationLogDocument[]; createdAt: Date; updatedAt: Date; }; @@ -15,6 +22,26 @@ export interface IncidentDocument extends IncidentDocumentBase { _id: Types.ObjectId; } +const EscalationLogSchema = new Schema( + { + notificationId: { + type: Schema.Types.ObjectId, + ref: "Notification", + required: true, + }, + delayMinutes: { + type: Number, + required: true, + min: 0, + }, + sentAt: { + type: Date, + required: true, + }, + }, + { _id: false } +); + const IncidentSchema = new Schema( { monitorId: { @@ -72,6 +99,10 @@ const IncidentSchema = new Schema( type: String, default: null, }, + escalationLogs: { + type: [EscalationLogSchema], + default: [], + }, }, { timestamps: true } ); diff --git a/server/src/db/models/Notification.ts b/server/src/db/models/Notification.ts index 9ea3f17bdd..4dd702accf 100755 --- a/server/src/db/models/Notification.ts +++ b/server/src/db/models/Notification.ts @@ -1,5 +1,26 @@ import { Schema, model, type Types } from "mongoose"; -import type { Notification, NotificationChannel } from "@/types/notification.js"; +import type { EscalationRule, Notification, NotificationChannel } from "@/types/notification.js"; + +const EscalationRuleSchema = new Schema( + { + type: { + type: String, + enum: ["email", "slack", "discord", "webhook"], + required: true, + }, + delayMinutes: { + type: Number, + required: true, + min: 1, + }, + trigger: { + type: String, + enum: ["escalation"], + required: true, + }, + }, + { _id: false } +); interface NotificationDocument extends Omit { _id: Types.ObjectId; @@ -37,6 +58,10 @@ const NotificationSchema = new Schema( homeserverUrl: { type: String }, roomId: { type: String }, accessToken: { type: String }, + escalationRules: { + type: [EscalationRuleSchema], + default: [], + }, }, { timestamps: true, diff --git a/server/src/repositories/incidents/MongoIncidentRepository.ts b/server/src/repositories/incidents/MongoIncidentRepository.ts index 096ba3d37b..40f3b799b3 100644 --- a/server/src/repositories/incidents/MongoIncidentRepository.ts +++ b/server/src/repositories/incidents/MongoIncidentRepository.ts @@ -60,6 +60,11 @@ class MongoIncidentRepository implements IIncidentsRepository { resolvedBy: doc.resolvedBy ? this.toStringId(doc.resolvedBy) : null, resolvedByEmail: doc.resolvedByEmail ?? null, comment: doc.comment ?? null, + escalationLogs: (doc.escalationLogs ?? []).map((log) => ({ + notificationId: this.toStringId(log.notificationId), + delayMinutes: log.delayMinutes, + sentAt: this.toDateString(log.sentAt), + })), createdAt: this.toDateString(doc.createdAt), updatedAt: this.toDateString(doc.updatedAt), }; diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d7594483..d2d809855c 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -6,6 +6,8 @@ import type { IMonitorsRepository, TeamQueryConfig, SummaryConfig } from "./IMon import { MongoBulkWriteError } from "mongodb"; import { AppError } from "@/utils/AppError.js"; +const NON_PENDING_MONITOR_STATUSES = ["paused", "maintenance"] as const; + class MongoMonitorsRepository implements IMonitorsRepository { create = async (monitor: Monitor, teamId: string, userId: string) => { const monitorModel = new MonitorModel({ ...monitor, teamId, userId }); @@ -231,38 +233,54 @@ class MongoMonitorsRepository implements IMonitorsRepository { } const pipeline = [ { $match: match }, + { + $addFields: { + displayStatus: { + $cond: [ + { + $and: [ + { $eq: [{ $size: { $ifNull: ["$recentChecks", []] } }, 0] }, + { $not: [{ $in: ["$status", [...NON_PENDING_MONITOR_STATUSES]] }] }, + ], + }, + "initializing", + { $ifNull: ["$status", "initializing"] }, + ], + }, + }, + }, { $group: { _id: null, totalMonitors: { $sum: 1 }, upMonitors: { $sum: { - $cond: [{ $eq: ["$status", "up"] }, 1, 0], + $cond: [{ $eq: ["$displayStatus", "up"] }, 1, 0], }, }, downMonitors: { $sum: { - $cond: [{ $eq: ["$status", "down"] }, 1, 0], + $cond: [{ $eq: ["$displayStatus", "down"] }, 1, 0], }, }, pausedMonitors: { $sum: { - $cond: [{ $eq: ["$status", "paused"] }, 1, 0], + $cond: [{ $eq: ["$displayStatus", "paused"] }, 1, 0], }, }, initializingMonitors: { $sum: { - $cond: [{ $eq: ["$status", "initializing"] }, 1, 0], + $cond: [{ $eq: ["$displayStatus", "initializing"] }, 1, 0], }, }, maintenanceMonitors: { $sum: { - $cond: [{ $eq: ["$status", "maintenance"] }, 1, 0], + $cond: [{ $eq: ["$displayStatus", "maintenance"] }, 1, 0], }, }, breachedMonitors: { $sum: { - $cond: [{ $eq: ["$status", "breached"] }, 1, 0], + $cond: [{ $eq: ["$displayStatus", "breached"] }, 1, 0], }, }, }, @@ -338,6 +356,20 @@ class MongoMonitorsRepository implements IMonitorsRepository { return documents.map((doc) => this.toEntity(doc)); }; + private getDisplayStatus = (status: Monitor["status"] | undefined, recentChecks: CheckSnapshot[]): Monitor["status"] => { + const effectiveStatus = status ?? "initializing"; + + if (recentChecks.length > 0) { + return effectiveStatus; + } + + if (NON_PENDING_MONITOR_STATUSES.includes(effectiveStatus as (typeof NON_PENDING_MONITOR_STATUSES)[number])) { + return effectiveStatus; + } + + return "initializing"; + }; + private toEntity = (doc: MonitorDocument): Monitor => { const toStringId = (value: unknown): string => { if (value instanceof mongoose.Types.ObjectId) { @@ -351,6 +383,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification)); + const recentChecks = (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => this.toCheckSnapshot(check)); return { id: toStringId(doc._id), @@ -358,7 +391,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { teamId: toStringId(doc.teamId), name: doc.name, description: doc.description ?? undefined, - status: doc.status ?? "initializing", + status: this.getDisplayStatus(doc.status, recentChecks), statusWindow: doc.statusWindow ?? [], statusWindowSize: doc.statusWindowSize, statusWindowThreshold: doc.statusWindowThreshold, @@ -387,7 +420,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { gameId: doc.gameId ?? undefined, grpcServiceName: doc.grpcServiceName ?? undefined, group: doc.group ?? null, - recentChecks: (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => this.toCheckSnapshot(check)), + recentChecks, geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, @@ -410,6 +443,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification)); + const recentChecks = (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => this.toCheckSnapshot(check)); return { id: toStringId(doc._id), @@ -417,7 +451,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { teamId: toStringId(doc.teamId), name: doc.name, description: doc.description ?? undefined, - status: doc.status ?? "initializing", + status: this.getDisplayStatus(doc.status, recentChecks), statusWindow: doc.statusWindow ?? [], statusWindowSize: doc.statusWindowSize, statusWindowThreshold: doc.statusWindowThreshold, @@ -446,7 +480,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { gameId: doc.gameId ?? undefined, grpcServiceName: doc.grpcServiceName ?? undefined, group: doc.group ?? null, - recentChecks: (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => this.toCheckSnapshot(check)), + recentChecks, geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, diff --git a/server/src/repositories/notifications/MongoNotificationsRepository.ts b/server/src/repositories/notifications/MongoNotificationsRepository.ts index 3c4225b50e..d683deab21 100644 --- a/server/src/repositories/notifications/MongoNotificationsRepository.ts +++ b/server/src/repositories/notifications/MongoNotificationsRepository.ts @@ -32,6 +32,11 @@ class MongoNotificationsRepository implements INotificationsRepository { homeserverUrl: doc.homeserverUrl ?? undefined, roomId: doc.roomId ?? undefined, accessToken: doc.accessToken ?? undefined, + escalationRules: (doc.escalationRules ?? []).map((rule) => ({ + type: rule.type, + delayMinutes: rule.delayMinutes, + trigger: rule.trigger, + })), createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; diff --git a/server/src/service/business/monitorService.ts b/server/src/service/business/monitorService.ts index 71c9d9d906..eae191837d 100644 --- a/server/src/service/business/monitorService.ts +++ b/server/src/service/business/monitorService.ts @@ -25,6 +25,7 @@ import type { ImportedMonitor } from "@/validation/monitorValidation.js"; import { ISuperSimpleQueue } from "../infrastructure/SuperSimpleQueue/SuperSimpleQueue.js"; import { IEmailService } from "../infrastructure/emailService.js"; import { ILogger } from "@/utils/logger.js"; +import { processEscalations } from "@/business/escalationService.js"; const SERVICE_NAME = "MonitorService"; type DateRangeKey = "recent" | "day" | "week" | "month" | "all"; @@ -439,6 +440,9 @@ export class MonitorService implements IMonitorService { editMonitor = async ({ teamId, monitorId, body }: { teamId: string; monitorId: string; body: Partial }) => { const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body); await this.jobQueue.updateJob(editedMonitor); + if (body.notifications !== undefined) { + await processEscalations({ monitorIds: [editedMonitor.id], catchUpBaseNotification: true }); + } return editedMonitor; }; @@ -459,6 +463,9 @@ export class MonitorService implements IMonitorService { if (modifiedCount > 0) { const monitors = await this.monitorsRepository.findByIds(monitorIds); await Promise.all(monitors.map((monitor) => this.jobQueue.updateJob(monitor))); + if (action !== "remove") { + await processEscalations({ monitorIds, catchUpBaseNotification: true }); + } } return modifiedCount; diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts index f4fd5b1e64..9cb65ec7ea 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts @@ -93,6 +93,7 @@ export class SuperSimpleQueue implements ISuperSimpleQueue { this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob()); this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob()); this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob()); + this.scheduler.addTemplate("escalation-check-job", this.helper.getEscalationCheckJob()); const monitors = await this.monitorsRepository.findAll(); if (!monitors) { return true; @@ -106,6 +107,7 @@ export class SuperSimpleQueue implements ISuperSimpleQueue { this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true }); this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 }); + this.scheduler.addJob({ id: "escalation-check", template: "escalation-check-job", active: true, repeat: 60 * 1000 }); return true; } catch (error: unknown) { diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index b6908127b2..d0808d7668 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -23,6 +23,7 @@ import { } from "@/repositories/index.js"; import { ILogger } from "@/utils/logger.js"; import { IBufferService } from "@/service/index.js"; +import { processEscalations } from "@/business/escalationService.js"; export interface ISuperSimpleQueueHelper { readonly serviceName: string; @@ -30,6 +31,7 @@ export interface ISuperSimpleQueueHelper { getHeartbeatGeoJob(): (monitor: Monitor) => Promise; getCleanupOrphanedJob(): () => Promise; getCleanupRetentionJob(): () => Promise; + getEscalationCheckJob(): () => Promise; isInMaintenanceWindow(monitorId: string, teamId: string): Promise; } @@ -155,9 +157,10 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { // Step 5. Get decisions const decision = this.evaluateMonitorAction(statusChangeResult); + const usesIncidentNotificationFlow = decision.shouldCreateIncident; - // Step 6. Handle notifications (best effort, continue even in event of failure, don't wait) - if (decision.shouldSendNotification) { + // Step 6. Handle recovery notifications directly. Incident-causing alerts are sent via the escalation service. + if (decision.shouldSendNotification && !usesIncidentNotificationFlow) { this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: unknown) => { this.logger.error({ message: `Error sending notifications for job ${statusChangeResult.monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, @@ -168,15 +171,28 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { }); } - // Step 7. Handle incidents (best effort, don't wait) - this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status).catch((error: unknown) => { - this.logger.warn({ - message: `Error handling incident for job ${monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, - service: SERVICE_NAME, - method: "getMonitorJob", - stack: error instanceof Error ? error.stack : undefined, + + // Step 7. Handle incidents. If an incident is active, trigger the incident/escalation notification pipeline immediately. + this.incidentService + .handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status) + .then(async (incident) => { + if (!incident || !usesIncidentNotificationFlow || !decision.shouldSendNotification) { + return; + } + + await processEscalations({ + monitorIds: [statusChangeResult.monitor.id], + catchUpBaseNotification: true, + }); + }) + .catch((error: unknown) => { + this.logger.warn({ + message: `Error handling incident for job ${monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "getMonitorJob", + stack: error instanceof Error ? error.stack : undefined, + }); }); - }); } catch (error: unknown) { this.logger.warn({ message: error instanceof Error ? error.message : "Unknown error", @@ -354,6 +370,12 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { }; }; + getEscalationCheckJob = () => { + return async () => { + await processEscalations(); + }; + }; + async isInMaintenanceWindow(monitorId: string, teamId: string) { const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId); // Check for active maintenance window: diff --git a/server/src/service/infrastructure/network/HttpProvider.ts b/server/src/service/infrastructure/network/HttpProvider.ts index bbc0934f8f..d94a44b485 100644 --- a/server/src/service/infrastructure/network/HttpProvider.ts +++ b/server/src/service/infrastructure/network/HttpProvider.ts @@ -10,19 +10,29 @@ import CacheableLookup from "cacheable-lookup"; export class HttpProvider implements IStatusProvider { readonly type = "http"; + private fallbackGot: Got; + private static readonly DEFAULT_HEADERS = { + "user-agent": "Checkmate/1.0 (+https://github.com/bluewave-labs/checkmate)", + accept: "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8", + "accept-language": "en-US,en;q=0.9", + }; constructor( private got: Got, private advancedMatcher: IAdvancedMatcher ) { - const cacheable = new CacheableLookup({ maxTtl: 300, errorTtl: 30 }); - this.got = got.extend({ - dnsCache: cacheable, + this.fallbackGot = got.extend({ timeout: { request: 30000, }, + headers: HttpProvider.DEFAULT_HEADERS, retry: { limit: 1 }, }); + + const cacheable = new CacheableLookup({ maxTtl: 300, errorTtl: 30 }); + this.got = this.fallbackGot.extend({ + dnsCache: cacheable, + }); } supports(type: MonitorType) { @@ -56,6 +66,71 @@ export class HttpProvider implements IStatusProvider { }; } + private isDnsLookupFailure(error: unknown): error is RequestError { + if (!(error instanceof RequestError)) { + return false; + } + + const code = error.code?.toLowerCase?.() ?? ""; + const message = error.message.toLowerCase(); + + return ( + code === "edestruction" || + code === "enotfound" || + code === "eai_again" || + message.includes("edestruction") || + message.includes("enotfound") || + message.includes("eai_again") || + message.includes("getaddrinfo") || + message.includes("querya") + ); + } + + private async executeRequest(client: Got, monitor: Monitor, options: Record): Promise> { + const response = await client(monitor.url, options); + const contentType = response.headers["content-type"] || ""; + const isJson = contentType.includes("application/json"); + + if (monitor.jsonPath && !isJson) { + return { + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: false, + code: response.statusCode, + message: "Response is not JSON", + responseTime: response.timings.phases.total ?? 0, + timings: response.timings, + payload: response.body as unknown as T, + }; + } + + let payload: T; + if (isJson) { + try { + payload = JSON.parse(response.body) as T; + } catch { + payload = response.body as unknown as T; + } + } else { + payload = response.body as unknown as T; + } + + const matchResult = this.advancedMatcher.validate(payload, monitor); + return { + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: response.ok && matchResult.ok, + code: response.statusCode, + message: matchResult.ok ? (response.statusMessage ?? "OK") : matchResult.message, + responseTime: response.timings.phases.total ?? 0, + timings: response.timings, + payload, + extracted: matchResult.extracted, + }; + } + async handle(monitor: Monitor): Promise> { const { url, secret, jsonPath, ignoreTlsErrors } = monitor; @@ -72,49 +147,16 @@ export class HttpProvider implements IStatusProvider { }; try { - const response = await this.got(url, options); - const contentType = response.headers["content-type"] || ""; - const isJson = contentType.includes("application/json"); - - if (jsonPath && !isJson) { - return { - monitorId: monitor.id, - teamId: monitor.teamId, - type: monitor.type, - status: false, - code: response.statusCode, - message: "Response is not JSON", - responseTime: response.timings.phases.total ?? 0, - timings: response.timings, - payload: response.body as unknown as T, - }; - } - - let payload: T; - if (isJson) { + return await this.executeRequest(this.got, monitor, options); + } catch (error: unknown) { + if (this.isDnsLookupFailure(error)) { try { - payload = JSON.parse(response.body) as T; - } catch { - payload = response.body as unknown as T; + return await this.executeRequest(this.fallbackGot, monitor, options); + } catch (fallbackError: unknown) { + return this.handleHttpError(fallbackError, monitor); } - } else { - payload = response.body as unknown as T; } - const matchResult = this.advancedMatcher.validate(payload, monitor); - return { - monitorId: monitor.id, - teamId: monitor.teamId, - type: monitor.type, - status: response.ok && matchResult.ok, - code: response.statusCode, - message: matchResult.ok ? (response.statusMessage ?? "OK") : matchResult.message, - responseTime: response.timings.phases.total ?? 0, - timings: response.timings, - payload, - extracted: matchResult.extracted, - }; - } catch (error: unknown) { return this.handleHttpError(error, monitor); } } diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index c75477c88c..d4ee0561c5 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -14,6 +14,7 @@ export interface INotificationsService { updateById(id: string, teamId: string, updateData: Partial): Promise; deleteById: (id: string, teamId: string) => Promise; handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; + sendNotificationMessage: (notification: Notification, notificationMessage: NotificationMessage) => Promise; sendTestNotification: (notification: Partial) => Promise; testAllNotifications: (notificationIds: string[]) => Promise; @@ -65,18 +66,12 @@ export class NotificationsService implements INotificationsService { this.notificationMessageBuilder = notificationMessageBuilder; } - private send = async ( - notification: Notification, - monitor: Monitor, - monitorStatusResponse: MonitorStatusResponse, - decision: MonitorActionDecision, - notificationMessage: NotificationMessage | undefined - ): Promise => { + sendNotificationMessage = async (notification: Notification, notificationMessage: NotificationMessage): Promise => { if (!notificationMessage) { this.logger.warn({ message: "Notification message not provided", service: SERVICE_NAME, - method: "send", + method: "sendNotificationMessage", }); return false; } @@ -101,12 +96,31 @@ export class NotificationsService implements INotificationsService { this.logger.warn({ message: `Unknown notification type: ${notification.type}`, service: SERVICE_NAME, - method: "send", + method: "sendNotificationMessage", }); return false; } }; + private send = async ( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + notificationMessage: NotificationMessage | undefined + ): Promise => { + if (!notificationMessage) { + this.logger.warn({ + message: "Notification message not provided", + service: SERVICE_NAME, + method: "send", + }); + return false; + } + + return await this.sendNotificationMessage(notification, notificationMessage); + }; + private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { const notificationIds = monitor.notifications ?? []; const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds); diff --git a/server/src/types/incident.ts b/server/src/types/incident.ts index 6b076ff835..d522888925 100644 --- a/server/src/types/incident.ts +++ b/server/src/types/incident.ts @@ -3,6 +3,12 @@ export const IncidentResolutionTypes = ["automatic", "manual", null] as const; export type IncidentResolutionType = (typeof IncidentResolutionTypes)[number]; +export interface EscalationLogEntry { + notificationId: string; + delayMinutes: number; + sentAt: string; +} + export interface Incident { id: string; monitorId: string; @@ -16,6 +22,7 @@ export interface Incident { resolvedBy?: string | null; resolvedByEmail?: string | null; comment?: string | null; + escalationLogs: EscalationLogEntry[]; createdAt: string; updatedAt: string; } diff --git a/server/src/types/notification.ts b/server/src/types/notification.ts index a121cd8e0c..076c2dda1b 100644 --- a/server/src/types/notification.ts +++ b/server/src/types/notification.ts @@ -1,6 +1,15 @@ export const NotificationChannels = ["email", "slack", "discord", "webhook", "pager_duty", "matrix", "teams"] as const; export type NotificationChannel = (typeof NotificationChannels)[number]; +export const EscalationRuleChannels = ["email", "slack", "discord", "webhook"] as const; +export type EscalationRuleChannel = (typeof EscalationRuleChannels)[number]; + +export interface EscalationRule { + type: EscalationRuleChannel; + delayMinutes: number; + trigger: "escalation"; +} + export interface Notification { id: string; userId: string; @@ -12,6 +21,7 @@ export interface Notification { homeserverUrl?: string; roomId?: string; accessToken?: string; + escalationRules: EscalationRule[]; createdAt: string; updatedAt: string; } diff --git a/server/src/utils/demoMonitors.json b/server/src/utils/demoMonitors.json index 3efea8d0ee..9a08086469 100755 --- a/server/src/utils/demoMonitors.json +++ b/server/src/utils/demoMonitors.json @@ -1,7 +1,7 @@ [ { "name": "Google", - "url": "https://www.google.com" + "url": "https://www.google.com/generate_204" }, { "name": "Facebook", diff --git a/server/src/validation/notificationValidation.ts b/server/src/validation/notificationValidation.ts index 2f261d8239..53a88f6587 100644 --- a/server/src/validation/notificationValidation.ts +++ b/server/src/validation/notificationValidation.ts @@ -1,70 +1,124 @@ import { z } from "zod"; +const escalationRuleChannels = ["email", "slack", "discord", "webhook"] as const; + +const escalationRuleValidation = z.object({ + type: z.enum(escalationRuleChannels), + delayMinutes: z.number().int("Delay must be a whole number").positive("Delay must be a positive integer"), + trigger: z.literal("escalation"), +}); + +type EscalationRuleInput = z.infer; +type EscalationValidationInput = { + type: string; + escalationRules?: EscalationRuleInput[]; +}; + +const validateEscalationRules = (schema: z.ZodObject) => + schema + .extend({ + escalationRules: z.array(escalationRuleValidation).optional().default([]), + }) + .superRefine((data, ctx) => { + const normalizedData = data as EscalationValidationInput; + const rules = normalizedData.escalationRules ?? []; + if (rules.length === 0) { + return; + } + + if (!escalationRuleChannels.includes(normalizedData.type as (typeof escalationRuleChannels)[number])) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Escalation rules are only supported for email, slack, discord, and webhook notifications", + path: ["escalationRules"], + }); + } + + for (const [index, rule] of rules.entries()) { + if (rule.type !== normalizedData.type) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Escalation rule type must match the notification type ${normalizedData.type}`, + path: ["escalationRules", index, "type"], + }); + } + + const previousRule = rules[index - 1]; + if (previousRule && rule.delayMinutes <= previousRule.delayMinutes) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Escalation delays must be sorted in strictly increasing order with no duplicates", + path: ["escalationRules", index, "delayMinutes"], + }); + } + } + }); + //**************************************** // Notification Validations //**************************************** export const createNotificationBodyValidation = z.discriminatedUnion("type", [ // Email notification - z.object({ + validateEscalationRules(z.object({ notificationName: z.string().min(1, "Notification name is required"), type: z.literal("email"), address: z.email("Please enter a valid e-mail address"), homeserverUrl: z.union([z.string(), z.literal("")]).optional(), roomId: z.union([z.string(), z.literal("")]).optional(), accessToken: z.union([z.string(), z.literal("")]).optional(), - }), + })), // Webhook notification - z.object({ + validateEscalationRules(z.object({ notificationName: z.string().min(1, "Notification name is required"), type: z.literal("webhook"), address: z.url({ message: "Please enter a valid Webhook URL" }), homeserverUrl: z.union([z.string(), z.literal("")]).optional(), roomId: z.union([z.string(), z.literal("")]).optional(), accessToken: z.union([z.string(), z.literal("")]).optional(), - }), + })), // Slack notification - z.object({ + validateEscalationRules(z.object({ notificationName: z.string().min(1, "Notification name is required"), type: z.literal("slack"), address: z.url({ message: "Please enter a valid Webhook URL" }), homeserverUrl: z.union([z.string(), z.literal("")]).optional(), roomId: z.union([z.string(), z.literal("")]).optional(), accessToken: z.union([z.string(), z.literal("")]).optional(), - }), + })), // Discord notification - z.object({ + validateEscalationRules(z.object({ notificationName: z.string().min(1, "Notification name is required"), type: z.literal("discord"), address: z.url({ message: "Please enter a valid Webhook URL" }), homeserverUrl: z.union([z.string(), z.literal("")]).optional(), roomId: z.union([z.string(), z.literal("")]).optional(), accessToken: z.union([z.string(), z.literal("")]).optional(), - }), + })), // PagerDuty notification - z.object({ + validateEscalationRules(z.object({ notificationName: z.string().min(1, "Notification name is required"), type: z.literal("pager_duty"), address: z.string().min(1, "PagerDuty integration key is required"), homeserverUrl: z.union([z.string(), z.literal("")]).optional(), roomId: z.union([z.string(), z.literal("")]).optional(), accessToken: z.union([z.string(), z.literal("")]).optional(), - }), + })), // Matrix notification - z.object({ + validateEscalationRules(z.object({ notificationName: z.string().min(1, "Notification name is required"), type: z.literal("matrix"), address: z.union([z.string(), z.literal("")]).optional(), homeserverUrl: z.url({ message: "Please enter a valid Homeserver URL" }), roomId: z.string().min(1, "Room ID is required"), accessToken: z.string().min(1, "Access Token is required"), - }), + })), // Teams notification - z.object({ + validateEscalationRules(z.object({ notificationName: z.string().min(1, "Notification name is required"), type: z.literal("teams"), address: z.url({ message: "Please enter a valid Webhook URL" }), - }), + })), ]); export const testNotificationBodyValidation = createNotificationBodyValidation;