From 04df4ce8414d200fe111a69e23db792b3d79c848 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Wed, 4 Nov 2026 10:42:41 +0100 Subject: [PATCH] Group monitor configuration into a readable nested schema --- cli/CLI/Resource.hs | 2 + docs/cli.md | 55 +++++++++ docs/examples/monitors/http-5xx.yaml | 39 +++--- docs/examples/monitors/latency-p99.yaml | 38 +++--- .../monitors/timefusion-flush-failures.yaml | 40 +++--- .../monitors/timefusion-membuffer.yaml | 43 ++++--- .../monitors/timefusion-oldest-bucket.yaml | 45 ++++--- .../monitors/timefusion-unsorted.yaml | 53 ++++---- .../monitors/timefusion-wal-corruption.yaml | 42 +++---- observability/monitors/timefusion-wal.yaml | 44 +++---- shared/monoscope-shared.cabal | 2 + shared/src/Pkg/MonitorConfig.hs | 116 ++++++++++++++++++ src/Models/Apis/MonitorDefinitions.hs | 73 ++++++++++- src/Models/Projects/RepositorySync.hs | 5 +- src/Web/ApiHandlers.hs | 11 +- test/integration/CLI/CLISpec.hs | 50 +++++++- test/integration/Web/ApiV1Spec.hs | 34 +++-- 17 files changed, 503 insertions(+), 189 deletions(-) create mode 100644 shared/src/Pkg/MonitorConfig.hs diff --git a/cli/CLI/Resource.hs b/cli/CLI/Resource.hs index f676f2e02..f254fe24e 100644 --- a/cli/CLI/Resource.hs +++ b/cli/CLI/Resource.hs @@ -24,6 +24,7 @@ import Data.Yaml qualified as Yaml import Effectful import Effectful.Environment (Environment) import Effectful.FileSystem (FileSystem, doesDirectoryExist, doesFileExist, listDirectory) +import Pkg.MonitorConfig (MonitorConfig, encodeMonitorConfig) import System.FilePath (takeExtension, ()) import Web.Operations qualified as Op import Web.Wire (Paged (..)) @@ -51,6 +52,7 @@ runCall cfg mode call = do (_, p : _) -> runApplyResource cfg url p mode (Op.GET, _) | Just l <- call.listing -> runList cfg l url query mode + | "/monitors/" `T.isPrefixOf` o.path, "/yaml" `T.isSuffixOf` o.path -> withResult (apiGetJson @_ @MonitorConfig cfg url []) renderAPIError (liftIO . putBSLn . encodeMonitorConfig) | "/yaml" `T.isSuffixOf` o.path -> withResult (apiGetJson @_ @AE.Value cfg url []) renderAPIError (liftIO . putBSLn . Yaml.encode) | otherwise -> runAPI mode (apiGetJson @_ @AE.Value cfg url query) (Op.DELETE, _) -> withResult (apiDelete cfg url) renderAPIError \() -> putTextLn (url <> " deleted") diff --git a/docs/cli.md b/docs/cli.md index 38cc27a8e..98257be86 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -413,6 +413,61 @@ monoscope monitors bulk mute --ids id1,id2,id3 --duration 60 monoscope monitors bulk resolve --ids id1,id2 ``` + +Monitor files group related configuration under three keys: + +- `condition`: query, unit, thresholds, comparison direction, and recovery thresholds. +- `evaluation`: check interval, lookback window, and sustained threshold duration, all in minutes. +- `notify`: alert text, recipients, repeat interval, and notification limit. + +```yaml +title: High error rate +severity: critical +active: true + +condition: + query: >- + severity.severity_text == "error" + alert_threshold: 10 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 15 + threshold_sustained_for_mins: 2 + +notify: + subject: Error rate elevated + message: Check the error logs. + email_all: true +``` + +The required fields are `title`, `condition.query`, `condition.alert_threshold`, +`condition.trigger_less_than`, `evaluation.check_interval_mins`, and +`evaluation.time_window_mins`. All other fields are optional. +Keep `severity`, `active`, `environment`, `service`, and `visualization_type` at the top level. +Put `unit` inside `condition`. + +Without `notify`, the monitor has no notification recipients. Omitted recipient +lists default to empty. The other defaults match the flat configuration: +`severity: error`, `active: true`, `visualization_type: timeseries`, +`threshold_sustained_for_mins: 0`, `email_all: false`, the title as the subject, +and an empty message. Recovery thresholds and notification repeat limits are unset. + +`create`, `update`, `apply`, and repository sync accept both nested and existing flat files. +A nested file rejects unknown fields and fields placed in the wrong group. +`patch` continues to use flat fields. + +To export a monitor in the nested format, run: + +```bash +monoscope monitors yaml > monitor.yaml +monoscope monitors apply monitor.yaml +``` + +Exports omit unset optional fields, empty recipient lists, and the default +`visualization_type: timeseries`. See [example monitors](examples/monitors/). + --- ## Dashboards diff --git a/docs/examples/monitors/http-5xx.yaml b/docs/examples/monitors/http-5xx.yaml index 726625e51..2a8ead752 100644 --- a/docs/examples/monitors/http-5xx.yaml +++ b/docs/examples/monitors/http-5xx.yaml @@ -1,18 +1,27 @@ title: HTTP 5xx rate -query: "where status_code >= 500 | summarize count(*) by bin_auto(timestamp)" severity: error -subject: "5xx rate elevated" -message: "Server-side error rate has exceeded the alert threshold." -alert_threshold: 10 -warning_threshold: 5 -trigger_less_than: false -check_interval_mins: 5 -time_window_mins: 15 -threshold_sustained_for_mins: 10 -notify_after_mins: 30 -emails: ["oncall@example.com"] -email_all: false -slack_channels: ["#alerts"] -teams: [] -visualization_type: timeseries active: true + +condition: + query: >- + where status_code >= 500 + | summarize count(*) by bin_auto(timestamp) + alert_threshold: 10 + warning_threshold: 5 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 15 + threshold_sustained_for_mins: 10 + +notify: + subject: 5xx rate elevated + message: >- + Server-side error rate has exceeded the alert threshold. + email_all: false + emails: + - oncall@example.com + slack_channels: + - "#alerts" + notify_after_mins: 30 diff --git a/docs/examples/monitors/latency-p99.yaml b/docs/examples/monitors/latency-p99.yaml index 7ff200000..4af7e090c 100644 --- a/docs/examples/monitors/latency-p99.yaml +++ b/docs/examples/monitors/latency-p99.yaml @@ -1,18 +1,26 @@ title: P99 latency -query: "summarize percentile(duration, 99) by bin_auto(timestamp)" severity: warning -subject: "P99 latency exceeded" -message: "Tail latency has breached the configured threshold." -alert_threshold: 1500 -warning_threshold: 1000 -trigger_less_than: false -check_interval_mins: 5 -time_window_mins: 30 -threshold_sustained_for_mins: 15 -notify_after_mins: 60 -emails: ["perf@example.com"] -email_all: false -slack_channels: ["#perf"] -teams: [] -visualization_type: timeseries active: true + +condition: + query: >- + summarize percentile(duration, 99) by bin_auto(timestamp) + alert_threshold: 1500 + warning_threshold: 1000 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 30 + threshold_sustained_for_mins: 15 + +notify: + subject: P99 latency exceeded + message: >- + Tail latency has breached the configured threshold. + email_all: false + emails: + - perf@example.com + slack_channels: + - "#perf" + notify_after_mins: 60 diff --git a/observability/monitors/timefusion-flush-failures.yaml b/observability/monitors/timefusion-flush-failures.yaml index b26078c1f..5285718b2 100644 --- a/observability/monitors/timefusion-flush-failures.yaml +++ b/observability/monitors/timefusion-flush-failures.yaml @@ -1,23 +1,21 @@ -active: true -alert_recovery_threshold: null -alert_threshold: 1.0 -check_interval_mins: 5 -email_all: true -emails: [] -message: Flush failures detected — Delta commits erroring. Check S3/R2 connectivity - and commit coordination. Leading indicator of the bucket-age page. -notify_after_mins: null -query: severity.severity_text == "error" and body has "flush failed" +title: "TF warn: flush failures" severity: warning -slack_channels: [] -stop_after_count: null -subject: TimeFusion Delta flush cycles erroring -teams: [] -threshold_sustained_for_mins: 0 -time_window_mins: 15 -title: 'TF warn: flush failures' -trigger_less_than: false -visualization_type: timeseries -warning_recovery_threshold: null -warning_threshold: null +active: true +condition: + query: >- + severity.severity_text == "error" and body has "flush failed" + alert_threshold: 1 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 15 + threshold_sustained_for_mins: 0 + +notify: + subject: TimeFusion Delta flush cycles erroring + message: >- + Flush failures detected — Delta commits erroring. Check S3/R2 connectivity and + commit coordination. Leading indicator of the bucket-age page. + email_all: true diff --git a/observability/monitors/timefusion-membuffer.yaml b/observability/monitors/timefusion-membuffer.yaml index 3e96d2ad7..921a7ad43 100644 --- a/observability/monitors/timefusion-membuffer.yaml +++ b/observability/monitors/timefusion-membuffer.yaml @@ -1,24 +1,23 @@ -active: true -alert_recovery_threshold: null -alert_threshold: 80.0 -check_interval_mins: 5 -email_all: true -emails: [] -message: MemBuffer pressure above 80% of budget for 2+ min — flush is lagging. Leading - indicator of insert rejections (Memory limit exceeded). -notify_after_mins: null -query: metrics | where metric_name == "timefusion.mem_buffer.pressure_pct" | summarize - max(value) +title: "TF warn: MemBuffer pressure > 80%" severity: warning -slack_channels: [] -stop_after_count: null -subject: TimeFusion approaching buffer memory hard limit -teams: [] -threshold_sustained_for_mins: 2 -time_window_mins: 10 -title: 'TF warn: MemBuffer pressure > 80%' -trigger_less_than: false -visualization_type: timeseries -warning_recovery_threshold: null -warning_threshold: null +active: true +condition: + query: >- + metrics + | where metric_name == "timefusion.mem_buffer.pressure_pct" + | summarize max(value) + alert_threshold: 80 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 10 + threshold_sustained_for_mins: 2 + +notify: + subject: TimeFusion approaching buffer memory hard limit + message: >- + MemBuffer pressure above 80% of budget for 2+ min — flush is lagging. Leading + indicator of insert rejections (Memory limit exceeded). + email_all: true diff --git a/observability/monitors/timefusion-oldest-bucket.yaml b/observability/monitors/timefusion-oldest-bucket.yaml index 8d8021349..3b0e1139b 100644 --- a/observability/monitors/timefusion-oldest-bucket.yaml +++ b/observability/monitors/timefusion-oldest-bucket.yaml @@ -1,25 +1,24 @@ -active: true -alert_recovery_threshold: null -alert_threshold: 1200.0 -check_interval_mins: 5 -email_all: true -emails: [] -message: Oldest bucket older than 2x flush interval (1200s) for 5+ min. Data is not - reaching Delta; ingest will eventually reject. Check flush task / S3 connectivity - / lock convoy. -notify_after_mins: null -query: metrics | where metric_name == "timefusion.mem_buffer.oldest_bucket_age_seconds" - | summarize max(value) +title: "TF page: MemBuffer oldest bucket age > 2x flush interval" severity: critical -slack_channels: [] -stop_after_count: null -subject: TimeFusion MemBuffer not flushing -teams: [] -threshold_sustained_for_mins: 5 -time_window_mins: 10 -title: 'TF page: MemBuffer oldest bucket age > 2x flush interval' -trigger_less_than: false -visualization_type: timeseries -warning_recovery_threshold: null -warning_threshold: null +active: true +condition: + query: >- + metrics + | where metric_name == "timefusion.mem_buffer.oldest_bucket_age_seconds" + | summarize max(value) + alert_threshold: 1200 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 10 + threshold_sustained_for_mins: 5 + +notify: + subject: TimeFusion MemBuffer not flushing + message: >- + Oldest bucket older than 2x flush interval (1200s) for 5+ min. Data is not reaching + Delta; ingest will eventually reject. Check flush task / S3 connectivity / lock + convoy. + email_all: true diff --git a/observability/monitors/timefusion-unsorted.yaml b/observability/monitors/timefusion-unsorted.yaml index a16ff30ff..7ee5e72ae 100644 --- a/observability/monitors/timefusion-unsorted.yaml +++ b/observability/monitors/timefusion-unsorted.yaml @@ -1,29 +1,28 @@ -active: true -alert_recovery_threshold: null -alert_threshold: 1.0 -check_interval_mins: 5 -email_all: true -emails: [] -message: A flush sort failed and wrote its group with no sorting_columns footer. The - reader's derive_common_ordering is all-or-nothing, so ONE such file voids the declared - ordering for every scan touching that partition — DedupExec drops to its unbounded - full-set seen-set (2 GiB/query cap) and the streaming top-N pushdown goes inert. - Nothing re-sorts a converged file, so this never heals on its own. Metric doc says - PAGE if > 0. Check flush_sort pool pressure and group sizes; repair with the off-box - timefusion optimize CLI. -notify_after_mins: null -query: metrics | where metric_name == "timefusion.flush.sort_unsorted_fallbacks" | - summarize max(value) +title: "TF page: flush sort fell back to writing an UNSORTED file" severity: critical -slack_channels: [] -stop_after_count: null -subject: TimeFusion wrote an unsorted parquet file -teams: [] -threshold_sustained_for_mins: 0 -time_window_mins: 60 -title: 'TF page: flush sort fell back to writing an UNSORTED file' -trigger_less_than: false -visualization_type: timeseries -warning_recovery_threshold: null -warning_threshold: null +active: true +condition: + query: >- + metrics + | where metric_name == "timefusion.flush.sort_unsorted_fallbacks" + | summarize max(value) + alert_threshold: 1 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 60 + threshold_sustained_for_mins: 0 + +notify: + subject: TimeFusion wrote an unsorted parquet file + message: >- + A flush sort failed and wrote its group with no sorting_columns footer. The reader's + derive_common_ordering is all-or-nothing, so ONE such file voids the declared + ordering for every scan touching that partition — DedupExec drops to its unbounded + full-set seen-set (2 GiB/query cap) and the streaming top-N pushdown goes inert. + Nothing re-sorts a converged file, so this never heals on its own. Metric doc says + PAGE if > 0. Check flush_sort pool pressure and group sizes; repair with the off-box + timefusion optimize CLI. + email_all: true diff --git a/observability/monitors/timefusion-wal-corruption.yaml b/observability/monitors/timefusion-wal-corruption.yaml index 6658b1e4c..08a88e3f5 100644 --- a/observability/monitors/timefusion-wal-corruption.yaml +++ b/observability/monitors/timefusion-wal-corruption.yaml @@ -1,24 +1,22 @@ -active: true -alert_recovery_threshold: null -alert_threshold: 1.0 -check_interval_mins: 5 -email_all: true -emails: [] -message: WAL replay/corruption errors detected — silent data loss risk. Inspect {data_dir}/wal/quarantine/ - on the host for the offending bytes. -notify_after_mins: null -query: severity.severity_text == "error" and (body has "WAL REPLAY FAILED" or body - has "WAL corruption") +title: "TF page: WAL corruption / replay failures" severity: critical -slack_channels: [] -stop_after_count: null -subject: TimeFusion WAL entries failing to deserialize or replay -teams: [] -threshold_sustained_for_mins: 0 -time_window_mins: 15 -title: 'TF page: WAL corruption / replay failures' -trigger_less_than: false -visualization_type: timeseries -warning_recovery_threshold: null -warning_threshold: null +active: true +condition: + query: >- + severity.severity_text == "error" and (body has "WAL REPLAY FAILED" or body has "WAL + corruption") + alert_threshold: 1 + trigger_less_than: false + +evaluation: + check_interval_mins: 5 + time_window_mins: 15 + threshold_sustained_for_mins: 0 + +notify: + subject: TimeFusion WAL entries failing to deserialize or replay + message: >- + WAL replay/corruption errors detected — silent data loss risk. Inspect + {data_dir}/wal/quarantine/ on the host for the offending bytes. + email_all: true diff --git a/observability/monitors/timefusion-wal.yaml b/observability/monitors/timefusion-wal.yaml index 4265e55b6..f39f4f405 100644 --- a/observability/monitors/timefusion-wal.yaml +++ b/observability/monitors/timefusion-wal.yaml @@ -1,24 +1,24 @@ -active: true -alert_recovery_threshold: null -alert_threshold: 6.442450944e10 -check_interval_mins: 15 -email_all: true -emails: [] -message: WAL dir above 60GB — steady state should be ~retention+20min of ingest (~30GB - at 6MB/s). Either the mtime reaper stopped, walrus leaked across rapid restarts, - or ingest spiked. Check timefusion.wal.gc_deleted_files and the WAL GC log lines. -notify_after_mins: null -query: metrics | where metric_name == "timefusion.wal.disk_bytes" | summarize max(value) +title: "TF warn: WAL disk usage > 60GB" severity: warning -slack_channels: [] -stop_after_count: null -subject: TimeFusion WAL directory growing beyond expected steady state -teams: [] -threshold_sustained_for_mins: 0 -time_window_mins: 30 -title: 'TF warn: WAL disk usage > 60GB' -trigger_less_than: false -visualization_type: timeseries -warning_recovery_threshold: null -warning_threshold: null +active: true +condition: + query: >- + metrics + | where metric_name == "timefusion.wal.disk_bytes" + | summarize max(value) + alert_threshold: 64424509440 # 60 GiB + trigger_less_than: false + +evaluation: + check_interval_mins: 15 + time_window_mins: 30 + threshold_sustained_for_mins: 0 + +notify: + subject: TimeFusion WAL directory growing beyond expected steady state + message: >- + WAL dir above 60GB — steady state should be ~retention+20min of ingest (~30GB at + 6MB/s). Either the mtime reaper stopped, walrus leaked across rapid restarts, or + ingest spiked. Check timefusion.wal.gc_deleted_files and the WAL GC log lines. + email_all: true diff --git a/shared/monoscope-shared.cabal b/shared/monoscope-shared.cabal index cb2f50e92..9f28092b1 100644 --- a/shared/monoscope-shared.cabal +++ b/shared/monoscope-shared.cabal @@ -80,6 +80,7 @@ library Data.Effectful.AgentWorkspace Data.Effectful.AgentWorkspace.Guest Pkg.CLIFormat + Pkg.MonitorConfig Pkg.Deriving Pkg.GitCommand Pkg.Parser.Eval @@ -89,6 +90,7 @@ library Web.Wire build-depends: + , yaml , aeson , base >=4.7 && <5 , cryptonite diff --git a/shared/src/Pkg/MonitorConfig.hs b/shared/src/Pkg/MonitorConfig.hs new file mode 100644 index 000000000..eee39b842 --- /dev/null +++ b/shared/src/Pkg/MonitorConfig.hs @@ -0,0 +1,116 @@ +-- | Database-free monitor document shared by API exports and CLI YAML rendering. +module Pkg.MonitorConfig ( + MonitorConfig (..), + MonitorCondition (..), + MonitorEvaluation (..), + MonitorNotification (..), + encodeMonitorConfig, +) where + +import Data.Aeson qualified as AE +import Data.Default (Default) +import Data.List qualified as L +import Data.OpenApi (ToSchema) +import Data.Yaml.Pretty qualified as Yaml +import Deriving.Aeson qualified as DAE +import Pkg.Deriving (SnakeSchema (..), UUIDId) +import Relude + + +-- | Human-facing apply/export shape. Null options are omitted; required settings +-- stay explicit, and notification lists may be omitted when empty. +data MonitorConfig = MonitorConfig + { title :: Text + , severity :: Maybe Text + , active :: Maybe Bool + , environment :: Maybe Text + , service :: Maybe Text + , condition :: MonitorCondition + , evaluation :: MonitorEvaluation + , notify :: Maybe MonitorNotification + , visualizationType :: Maybe Text + } + deriving stock (Generic, Show) + deriving (AE.FromJSON, AE.ToJSON) via ConfigJSON MonitorConfig + deriving (ToSchema) via SnakeSchema MonitorConfig + + +data MonitorCondition = MonitorCondition + { query :: Text + , unit :: Maybe Text + , alertThreshold :: Double + , warningThreshold :: Maybe Double + , triggerLessThan :: Bool + , alertRecoveryThreshold :: Maybe Double + , warningRecoveryThreshold :: Maybe Double + } + deriving stock (Generic, Show) + deriving (AE.FromJSON, AE.ToJSON) via ConfigJSON MonitorCondition + deriving (ToSchema) via SnakeSchema MonitorCondition + + +data MonitorEvaluation = MonitorEvaluation + { checkIntervalMins :: Int + , timeWindowMins :: Int + , thresholdSustainedForMins :: Maybe Int + } + deriving stock (Generic, Show) + deriving (AE.FromJSON, AE.ToJSON) via ConfigJSON MonitorEvaluation + deriving (ToSchema) via SnakeSchema MonitorEvaluation + + +data MonitorNotification = MonitorNotification + { subject :: Maybe Text + , message :: Maybe Text + , emailAll :: Maybe Bool + , emails :: Maybe [Text] + , slackChannels :: Maybe [Text] + , teams :: Maybe [UUIDId "team"] + , notifyAfterMins :: Maybe Int + , stopAfterCount :: Maybe Int + } + deriving stock (Generic, Show) + deriving anyclass (Default) + deriving (AE.FromJSON, AE.ToJSON) via ConfigJSON MonitorNotification + deriving (ToSchema) via SnakeSchema MonitorNotification + + +type ConfigJSON = DAE.CustomJSON '[DAE.FieldLabelModifier '[DAE.CamelToSnake], DAE.OmitNothingFields, DAE.RejectUnknownFields] + + +-- | Keep the same reading order in CLI dumps and repository sync exports. +encodeMonitorConfig :: MonitorConfig -> ByteString +encodeMonitorConfig = Yaml.encodePretty (Yaml.setConfCompare (comparing rank <> compare) Yaml.defConfig) + where + rank key = + fromMaybe (maxBound :: Int) + $ L.elemIndex + key + [ "title" + , "severity" + , "active" + , "environment" + , "service" + , "condition" + , "query" + , "unit" + , "alert_threshold" + , "warning_threshold" + , "trigger_less_than" + , "alert_recovery_threshold" + , "warning_recovery_threshold" + , "evaluation" + , "check_interval_mins" + , "time_window_mins" + , "threshold_sustained_for_mins" + , "notify" + , "subject" + , "message" + , "email_all" + , "emails" + , "slack_channels" + , "teams" + , "notify_after_mins" + , "stop_after_count" + , "visualization_type" + ] diff --git a/src/Models/Apis/MonitorDefinitions.hs b/src/Models/Apis/MonitorDefinitions.hs index 6368c1e22..ac8e827e9 100644 --- a/src/Models/Apis/MonitorDefinitions.hs +++ b/src/Models/Apis/MonitorDefinitions.hs @@ -1,9 +1,12 @@ -module Models.Apis.MonitorDefinitions (MonitorInput (..), monitorFromInput, monitorToInput, compileAlertSql) where +module Models.Apis.MonitorDefinitions (MonitorInput (..), monitorConfig, monitorFromInput, monitorToInput, compileAlertSql) where +import Control.Lens ((?~)) import Data.Aeson qualified as AE +import Data.Aeson.KeyMap qualified as KM import Data.CaseInsensitive qualified as CI -import Data.Default (Default) -import Data.OpenApi (ToSchema) +import Data.Default (Default (..)) +import Data.OpenApi (ToSchema (..)) +import Data.OpenApi qualified as OpenApi import Data.Text qualified as T import Data.Time (UTCTime) import Data.Vector qualified as V @@ -11,6 +14,7 @@ import Models.Apis.Monitors qualified as Monitors import Models.Projects.ProjectMembers qualified as PM import Models.Projects.Projects qualified as Projects import Pkg.DeriveUtils (SnakeJSON (..), SnakeSchema (..)) +import Pkg.MonitorConfig (MonitorCondition (..), MonitorConfig (..), MonitorEvaluation (..), MonitorNotification (..)) import Pkg.Parser qualified as Parser import Relude @@ -45,7 +49,68 @@ data MonitorInput = MonitorInput } deriving stock (Generic, Show) deriving anyclass (Default) - deriving (AE.FromJSON, AE.ToJSON, ToSchema) via SnakeJSON SnakeSchema MonitorInput + deriving (AE.ToJSON) via SnakeJSON SnakeSchema MonitorInput + + +instance ToSchema MonitorInput where + declareNamedSchema _ = do + OpenApi.NamedSchema _ flat <- declareNamedSchema (Proxy @(SnakeSchema MonitorInput)) + nested <- OpenApi.declareSchemaRef (Proxy @MonitorConfig) + pure $ OpenApi.NamedSchema (Just "MonitorInput") $ mempty & OpenApi.oneOf ?~ [OpenApi.Inline flat, nested] + + +-- | Flat API requests remain valid; as-code documents group related settings. +-- Mixed documents are rejected so an accidentally misplaced field cannot be ignored. +instance AE.FromJSON MonitorInput where + parseJSON = AE.withObject "MonitorInput" \obj -> + if any (`KM.member` obj) ["condition", "evaluation", "notify"] + then configInput <$> AE.parseJSON (AE.Object obj) + else coerce <$> AE.parseJSON @(SnakeJSON SnakeSchema MonitorInput) (AE.Object obj) + where + configInput :: MonitorConfig -> MonitorInput + configInput cfg = + let notification = fromMaybe def cfg.notify + in MonitorInput + { title = cfg.title + , severity = cfg.severity + , active = cfg.active + , environment = cfg.environment + , service = cfg.service + , query = cfg.condition.query + , unit = cfg.condition.unit + , alertThreshold = cfg.condition.alertThreshold + , warningThreshold = cfg.condition.warningThreshold + , triggerLessThan = cfg.condition.triggerLessThan + , alertRecoveryThreshold = cfg.condition.alertRecoveryThreshold + , warningRecoveryThreshold = cfg.condition.warningRecoveryThreshold + , checkIntervalMins = cfg.evaluation.checkIntervalMins + , timeWindowMins = cfg.evaluation.timeWindowMins + , thresholdSustainedForMins = cfg.evaluation.thresholdSustainedForMins + , subject = notification.subject + , message = notification.message + , emailAll = notification.emailAll + , emails = fold notification.emails + , slackChannels = fold notification.slackChannels + , teams = fold notification.teams + , notifyAfterMins = notification.notifyAfterMins + , stopAfterCount = notification.stopAfterCount + , visualizationType = Just $ fromMaybe "timeseries" cfg.visualizationType + } + + +monitorConfig :: MonitorInput -> MonitorConfig +monitorConfig inp = + MonitorConfig + { title = inp.title + , severity = inp.severity + , active = inp.active + , environment = inp.environment + , service = inp.service + , condition = MonitorCondition inp.query inp.unit inp.alertThreshold inp.warningThreshold inp.triggerLessThan inp.alertRecoveryThreshold inp.warningRecoveryThreshold + , evaluation = MonitorEvaluation inp.checkIntervalMins inp.timeWindowMins inp.thresholdSustainedForMins + , notify = Just $ MonitorNotification inp.subject inp.message inp.emailAll (viaNonEmpty toList inp.emails) (viaNonEmpty toList inp.slackChannels) (viaNonEmpty toList inp.teams) inp.notifyAfterMins inp.stopAfterCount + , visualizationType = mfilter (/= "timeseries") inp.visualizationType + } -- | The compiled form of a monitor's KQL, which is what alert evaluation actually diff --git a/src/Models/Projects/RepositorySync.hs b/src/Models/Projects/RepositorySync.hs index 5e8035b24..f26a84dc9 100644 --- a/src/Models/Projects/RepositorySync.hs +++ b/src/Models/Projects/RepositorySync.hs @@ -41,7 +41,7 @@ import Effectful.Time (Time) import Effectful.Time qualified as Time import Hasql.Interpolate qualified as HI import Hasql.Transaction.Sessions qualified as TxS -import Models.Apis.MonitorDefinitions (MonitorInput (..), compileAlertSql, monitorFromInput, monitorToInput) +import Models.Apis.MonitorDefinitions (MonitorInput (..), compileAlertSql, monitorConfig, monitorFromInput, monitorToInput) import Models.Apis.Monitors qualified as Monitors import Models.Apis.PrometheusScrapeConfigs qualified as PromCfg import Models.Projects.Dashboards qualified as Dashboards @@ -49,6 +49,7 @@ import Models.Projects.GitSync qualified as GitSync import Models.Projects.ProjectMembers qualified as Members import Models.Projects.Projects qualified as Projects import Pkg.DeriveUtils (UUIDId (..), WrappedEnumSC (..)) +import Pkg.MonitorConfig (encodeMonitorConfig) import Relude hiding (local) import System.Types (DB) import Web.HttpApiData (FromHttpApiData (..)) @@ -209,7 +210,7 @@ decodeDefinition path body encodeDefinition :: Definition -> ByteString encodeDefinition = \case DashboardDefinition schema -> Yaml.encode schema - QueryDefinition input -> Yaml.encode input + QueryDefinition input -> encodeMonitorConfig (monitorConfig input) value@UptimeDefinitionConfig{} -> Yaml.encode value value@CronDefinitionConfig{} -> Yaml.encode value diff --git a/src/Web/ApiHandlers.hs b/src/Web/ApiHandlers.hs index f0f214236..dada76aa8 100644 --- a/src/Web/ApiHandlers.hs +++ b/src/Web/ApiHandlers.hs @@ -127,7 +127,7 @@ import Models.Apis.ErrorPatterns qualified as ErrorPatterns import Models.Apis.Incidents qualified as Incidents import Models.Apis.Issues qualified as Issues import Models.Apis.LogPatterns qualified as LogPatterns -import Models.Apis.MonitorDefinitions (compileAlertSql, monitorFromInput, monitorToInput) +import Models.Apis.MonitorDefinitions (compileAlertSql, monitorConfig, monitorFromInput, monitorToInput) import Models.Apis.Monitors qualified as Monitors import Models.Apis.SchemaCatalog qualified as SchemaCatalog import Models.Apis.ShareEvents qualified as ShareEvents @@ -144,6 +144,7 @@ import Pages.Replay qualified as Replay import Pkg.Components.TimePicker qualified as TP import Pkg.Components.Widget qualified as Widget import Pkg.DeriveUtils (SnakeSchema (..), UUIDId (..)) +import Pkg.MonitorConfig (MonitorConfig) import Pkg.Parser.Expr qualified as ParserExpr import Pkg.SchemaLearning.Catalog qualified as Fields import Relude hiding (ask, id) @@ -229,8 +230,8 @@ data ApiV1Routes mode = ApiV1Routes , monitorGet :: mode :- "monitors" :> Capture "monitor_id" Monitors.QueryMonitorId :> Get '[JSON] Monitors.QueryMonitor , monitorCreate :: mode :- "monitors" :> ReqBody '[JSON] MonitorInput :> Post '[JSON] Monitors.QueryMonitor , monitorApply :: mode :- "monitors" :> "apply" :> ReqBody '[JSON] MonitorInput :> Post '[JSON] Monitors.QueryMonitor - , monitorYaml :: mode :- "monitors" :> Capture "monitor_id" Monitors.QueryMonitorId :> "yaml" :> Get '[JSON] MonitorInput - -- ^ /yaml returns the applyable MonitorInput as JSON (same for dashboards); + , monitorYaml :: mode :- "monitors" :> Capture "monitor_id" Monitors.QueryMonitorId :> "yaml" :> Get '[JSON] MonitorConfig + -- ^ /yaml returns the applyable configuration as JSON (same for dashboards); -- the CLI renders it as YAML client-side via runYamlDump. , monitorUpdate :: mode :- "monitors" :> Capture "monitor_id" Monitors.QueryMonitorId :> ReqBody '[JSON] MonitorInput :> Put '[JSON] Monitors.QueryMonitor , monitorPatch :: mode :- "monitors" :> Capture "monitor_id" Monitors.QueryMonitorId :> ReqBody '[JSON] MonitorPatch :> Patch '[JSON] Monitors.QueryMonitor @@ -592,10 +593,10 @@ apiMonitorApply pid inp = -- | Inverse of 'monitorFromInput': dump a monitor as the applyable input -- shape, so `monitors yaml ID > m.yaml && monitors apply m.yaml` round-trips. -apiMonitorYaml :: Projects.ProjectId -> Monitors.QueryMonitorId -> ATBaseCtx MonitorInput +apiMonitorYaml :: Projects.ProjectId -> Monitors.QueryMonitorId -> ATBaseCtx MonitorConfig apiMonitorYaml pid mid = do m <- apiMonitorGet pid mid - pure $ monitorToInput m + pure $ monitorConfig $ monitorToInput m -- | PATCH — merge fields into existing monitor. diff --git a/test/integration/CLI/CLISpec.hs b/test/integration/CLI/CLISpec.hs index 0bc104bd8..f671dc41b 100644 --- a/test/integration/CLI/CLISpec.hs +++ b/test/integration/CLI/CLISpec.hs @@ -16,6 +16,7 @@ import Data.Time (UTCTime, addUTCTime, getCurrentTime) import Data.Time.Format.ISO8601 (iso8601Show) import Data.UUID qualified as UUID import Data.UUID.V4 qualified as UUID4 +import Data.Yaml qualified as Yaml import Database.PostgreSQL.Simple.Newtypes (Aeson (..)) import Hasql.Interpolate qualified as HI import Log.Backend.LogList qualified as LogList @@ -274,9 +275,56 @@ spec = around withCliResources $ describe "CLI" do replicateM_ 2 $ cliOk tr ["--json", "monitors", "apply", yamlPath] [mid] <- monitorIds cliOk tr ["--table", "monitors", "list"] >>= (`shouldContainAll` ["Lifecycle error-rate alert", mid]) - cliOk tr ["monitors", "yaml", toString mid] >>= writeFileText dumpPath + writeFileText yamlPath + $ unlines + [ "title: Lifecycle error-rate alert" + , "severity: critical" + , "condition:" + , " query: 'severity.severity_text == \"error\"'" + , " alert_threshold: 20" + , " warning_threshold: 10" + , " trigger_less_than: false" + , " alert_recovery_threshold: 15" + , "evaluation:" + , " check_interval_mins: 2" + , " time_window_mins: 10" + , " threshold_sustained_for_mins: 3" + , "notify:" + , " email_all: true" + , " emails: [ops@example.com]" + , " slack_channels: [\"#alerts\"]" + , " subject: Error rate elevated" + , " message: Check the error logs." + , " notify_after_mins: 30" + , " stop_after_count: 4" + ] + updated <- cliJson tr ["monitors", "apply", yamlPath] + updated ^? key "id" . _String `shouldBe` Just mid + updated ^? key "alert_threshold" . _Number `shouldBe` Just 20 + updated ^? key "threshold_sustained_for_mins" . _Number `shouldBe` Just 3 + updated ^? key "alert_config" . key "email_all" `shouldBe` Just (AE.Bool True) + updated ^? key "alert_config" . key "emails" `shouldBe` Just (AE.toJSON (["ops@example.com"] :: [Text])) + updated ^? key "alert_config" . key "slack_channels" `shouldBe` Just (AE.toJSON (["#alerts"] :: [Text])) + dumped <- cliOk tr ["monitors", "yaml", toString mid] + dumped `shouldSatisfy` T.isPrefixOf "title:" + document <- either (fail . show) pure $ Yaml.decodeEither' @AE.Value (encodeUtf8 dumped) + document ^? key "condition" . key "alert_threshold" . _Number `shouldBe` Just 20 + document ^? key "evaluation" . key "check_interval_mins" . _Number `shouldBe` Just 2 + document ^? key "notify" . key "stop_after_count" . _Number `shouldBe` Just 4 + document ^? key "condition" . key "warning_recovery_threshold" `shouldBe` Nothing + document ^? key "visualization_type" `shouldBe` Nothing + writeFileText dumpPath dumped void $ cliOk tr ["--json", "monitors", "apply", dumpPath] monitorIds `shouldReturn` [mid] + reapplied <- cliJson tr ["monitors", "get", toString mid] + for_ ["log_query", "alert_threshold", "warning_threshold", "alert_recovery_threshold", "warning_recovery_threshold", "trigger_less_than", "check_interval_mins", "time_window_mins", "threshold_sustained_for_mins", "alert_config", "renotify_interval_mins", "stop_after_count"] \field -> + reapplied ^? key field `shouldBe` updated ^? key field + Just documentObject <- pure $ document ^? _Object + writeFileText yamlPath (decodeUtf8 $ AE.encode $ AE.Object $ KM.delete "notify" documentObject) + quiet <- cliJson tr ["monitors", "apply", yamlPath] + quiet ^? key "id" . _String `shouldBe` Just mid + quiet ^? key "alert_config" . key "email_all" `shouldBe` Just (AE.Bool False) + for_ ["emails", "slack_channels"] \field -> quiet ^? key "alert_config" . key field `shouldBe` Just (AE.toJSON ([] :: [Text])) forM_ [["mute", toString mid, "--for", "30"], ["unmute", toString mid], ["delete", toString mid]] \args -> cliOk tr ("--json" : "monitors" : args) monitorIds `shouldReturn` [] diff --git a/test/integration/Web/ApiV1Spec.hs b/test/integration/Web/ApiV1Spec.hs index f6f53810b..30a53f978 100644 --- a/test/integration/Web/ApiV1Spec.hs +++ b/test/integration/Web/ApiV1Spec.hs @@ -43,6 +43,7 @@ import Data.Set qualified as S import Pages.Dashboards qualified as DashPage import Pkg.AI qualified as AI import Pkg.AgentCatalog qualified as Catalog +import Pkg.MonitorConfig (MonitorCondition (unit), MonitorConfig (condition, environment, service)) import Pkg.TestUtils import Servant (NoContent (..)) import Servant qualified @@ -221,7 +222,18 @@ spec = describe "API v1" do (.alertConfig.unit) <$> runB (ApiH.apiMonitorGet testPid created.id) `shouldReturn` Just "s" void $ runB $ ApiH.apiMonitorPatch testPid created.id (def :: ApiT.MonitorPatch){ApiT.unit = Just "ms"} exported <- runB $ ApiH.apiMonitorYaml testPid created.id - (exported.unit, exported.environment, exported.service) `shouldBe` (Just "ms", Just "production", Just "checkout") + (exported.condition.unit, exported.environment, exported.service) `shouldBe` (Just "ms", Just "production", Just "checkout") + apiKey <- createTestAPIKey tr testPid "nested-monitor-config" + Just document <- pure $ AE.toJSON exported ^? _Object + let applyDocument obj = request (topAppWithHTTP (runHTTPGolden "./tests/golden/") tr) H.methodPost "/api/v1/monitors/apply" [demo, json, bearer apiKey] (AE.encode $ AE.Object obj) + invalidDocuments = + [ AEKM.insert "alert_threshold" (AE.Number 999) document + , AEKM.delete "evaluation" document + , AEKM.insert "condition" (AE.object ["query" AE..= ("count(*)" :: Text), "alert_threshold" AE..= ("bad" :: Text), "trigger_less_than" AE..= False]) document + , AEKM.insert "notify" (AE.object ["email_al" AE..= True]) document + ] + for_ invalidDocuments \invalid -> status <$> applyDocument invalid `shouldReturn` 400 + status <$> applyDocument document `shouldReturn` 200 -- PATCH used to keep the OLD compiled SQL, so the monitor kept alerting on its previous query. void $ runB $ ApiH.apiMonitorPatch testPid created.id (def :: ApiT.MonitorPatch){ApiT.query = Just "status_code == 500"} recompiled <- stored created.id @@ -422,8 +434,6 @@ spec = describe "API v1" do rpcCallNamed name args = rpcCall (AE.object ["name" AE..= (name :: Text), "arguments" AE..= args]) - - it "returns an empty dashboard's title and widget list" $ \tr -> do created <- toBaseServantResponse tr $ ApiH.apiDashboardCreate ApiT.ApiKeyPrincipal testPid ApiT.DashboardInput{ApiT.title = "empty dashboard", ApiT.tags = Nothing, ApiT.teams = Nothing, ApiT.filePath = Nothing, ApiT.schema = Nothing} rendered <- toBaseServantResponse tr $ DashPage.apiDashboardData testPid created.summary.id Nothing Nothing (Just "1h") Nothing Nothing [] @@ -478,13 +488,17 @@ spec = describe "API v1" do body name = do input <- find (\tool -> (tool ^? key "name" . _String) == Just name) tools >>= (^? key "inputSchema") schema <- input ^? key "properties" . key "body" - pure $ fromMaybe schema do - reference <- schema ^? key "$ref" . _String - definition <- T.stripPrefix "#/$defs/" reference - input ^? key "$defs" . key (AEK.fromText definition) - for_ ([("create_monitor", ["title", "query", "alert_threshold"]), ("search_events", ["query", "since"])] :: [(Text, [Text])]) $ \(name, fields) -> do - schema <- maybe (fail $ "Missing body schema: " <> toString name) pure $ body name - for_ fields $ \field -> schema ^? key "properties" . key (AEK.fromText field) `shouldSatisfy` isJust + let resolve value = fromMaybe value do + reference <- value ^? key "$ref" . _String + definition <- T.stripPrefix "#/$defs/" reference + input ^? key "$defs" . key (AEK.fromText definition) + resolved = resolve schema + alternatives = resolved ^.. key "oneOf" . _Array . traverse + pure $ map resolve $ if null alternatives then [resolved] else alternatives + for_ ([("create_monitor", [["title", "query", "alert_threshold"], ["title", "condition", "evaluation", "notify"]]), ("search_events", [["query", "since"]])] :: [(Text, [[Text]])]) $ \(name, variants) -> do + schemas <- maybe (fail $ "Missing body schema: " <> toString name) pure $ body name + for_ variants $ \fields -> + schemas `shouldSatisfy` any (\schema -> all (\field -> isJust $ schema ^? key "properties" . key (AEK.fromText field)) fields) for_ (tools ^.. traverse . key "inputSchema") $ \schema -> for_ (schema ^.. cosmos . key "$ref" . _String) $ \reference -> do definition <- maybe (fail $ "Unbound schema reference: " <> toString reference) pure $ T.stripPrefix "#/$defs/" reference