diff --git a/alertmanager/README.md b/alertmanager/README.md index a42a51d..29d5864 100644 --- a/alertmanager/README.md +++ b/alertmanager/README.md @@ -9,12 +9,14 @@ It is deployed using the official docker image. monitoring_instance_name MANDATORY config OPTIONAL slack_url OPTIONAL - slack_channel OPTIONAL + slack_receivers OPTIONAL ``` - **monitoring_instance_name:** A unique name given to the application - **monitoring_space_id:** The Cloud Foundry space you wish to deploy the application to - **config:** The contents of an alertmanager.yml file, as specified in the [documentation](https://prometheus.io/docs/alerting/latest/configuration/). If not specified, a dummy configuration is deployed. +- **slack_url** An incoming webhook created at https://api.slack.com/apps/appid/incoming-webhooks? (replace appid with AppId value). Incoming webhooks are assigned to a single channel so the channel does not need to be specified. The default destination for all alerts that do not satisfy the matches defined in other routes. +- **slack_receivers** A map of route\receiver names and webhook URLs. These can be used to route alerts to different channels based on alert labels. Alerts should include a label called `receiver` whose value is the same as the name of the receiver defined in this config property. ### Example ```hcl @@ -24,7 +26,10 @@ module alertmanager { monitoring_instance_name = "test-alertmanager" config = file("${path.module}/files/alertmanager.yml") slack_url = https://hooks.slack.com/services/XXXXXXXXX/YYYYYYYYYYY/xxxxxxxxxxxxxxxxxxxxxxxx - slack_channel = mychannel + slack_receivers = { + "service_one_tech": "https://hooks.slack.com/services/XXXXXXXXX/YYYYYYYYYYY/xxxxxxxxxxxxxxxxxxxxxxxx", + "service_two_tech": "https://hooks.slack.com/services/XXXXXXXXX/YYYYYYYYYYY/xxxxxxxxxxxxxxxxxxxxxxxx" + } } ``` diff --git a/alertmanager/config/alertmanager.yml.tmpl b/alertmanager/config/alertmanager.yml.tmpl index deddcaa..2a91656 100644 --- a/alertmanager/config/alertmanager.yml.tmpl +++ b/alertmanager/config/alertmanager.yml.tmpl @@ -5,6 +5,7 @@ global: %{ endif ~} route: + # the default route, this will be used if the matcher(s) of any of the child routes are not satisfied receiver: 'slack-notifications' group_interval: 1m repeat_interval: 1h @@ -21,6 +22,13 @@ route: mute_time_intervals: - out-of-hours matchers: [period = 'out-of-hours'] +%{ for receiver, webhook in slack_receivers ~} + - receiver: ${ receiver } + group_interval: 1m + repeat_interval: 1h + matchers: + - receiver="${ receiver }" +%{ endfor ~} mute_time_intervals: - name: out-of-hours @@ -41,13 +49,13 @@ receivers: - name: slack-notifications %{ if slack_url != "" ~} slack_configs: - - channel: '#${slack_channel}' - send_resolved: true + - send_resolved: true title: '{{ template "slack.monzo.title" . }}' pretext: '{{ template "slack.monzo.pretext" . }}' title_link: '{{ template "__alertmanagerURL" . }}' text: '{{ template "slack.monzo.text" . }}' color: '{{ template "slack.monzo.color" . }}' + footer: 'Sent to default route' actions: - type: button text: 'Runbook :green_book:' @@ -62,3 +70,26 @@ receivers: text: 'Silence :no_bell:' url: '{{ template "__alert_silence_link" . }}' %{ endif ~} +%{ for receiver, webhook in slack_receivers ~} +- name: ${ receiver } + slack_configs: + - api_url: ${ webhook } + send_resolved: true + title: '{{ template "slack.monzo.title" . }}' + pretext: '{{ template "slack.monzo.pretext" . }}' + text: '{{ template "slack.monzo.text" . }}' + color: '{{ template "slack.monzo.color" . }}' + actions: + - type: button + text: 'Runbook :green_book:' + url: '{{ (index .Alerts 0).Annotations.runbook }}' + - type: button + text: 'Query :mag:' + url: '{{ (index .Alerts 0).GeneratorURL }}' + - type: button + text: 'Dashboard :bar_chart:' + url: '{{ (index .Alerts 0).Annotations.dashboard }}' + - type: button + text: 'Silence :no_bell:' + url: '{{ template "__alert_silence_link" . }}' +%{ endfor ~} diff --git a/alertmanager/input.tf b/alertmanager/input.tf index 2233e5c..7b2baf6 100644 --- a/alertmanager/input.tf +++ b/alertmanager/input.tf @@ -5,8 +5,11 @@ variable "monitoring_space_id" {} variable "config" { default = "" } variable "slack_url" { default = "" } -variable "slack_channel" { default = "" } variable "slack_template" { default = "" } + +variable "slack_receivers" { + type = map(string) +} variable "docker_credentials" { description = "Credentials for Dockerhub. Map of {username, password}." type = map(any) @@ -18,8 +21,8 @@ variable "docker_credentials" { locals { docker_image_tag = "v0.22.2" alertmanager_variables = { - slack_url = var.slack_url - slack_channel = var.slack_channel + slack_url = var.slack_url + slack_receivers = var.slack_receivers } config = var.config == "" ? templatefile("${path.module}/config/alertmanager.yml.tmpl", local.alertmanager_variables) : var.config slack_template = var.slack_template == "" ? file("${path.module}/config/slack.tmpl") : var.slack_template diff --git a/prometheus_all/input.tf b/prometheus_all/input.tf index b16394d..10e80ae 100644 --- a/prometheus_all/input.tf +++ b/prometheus_all/input.tf @@ -27,9 +27,13 @@ variable "alertmanager_slack_url" { default = "" } variable "alertmanager_slack_channel" { - description = "Slack channel for alert notifications" + description = "DEPRECATED. Slack channel for alert notifications. This is unnecessary if the webhook in alertmanager_slack_url has been assigned to a channel." default = "" } +variable "alertmanager_slack_receivers" { + type = map(string) + default = {} +} variable "alertmanager_slack_template" { description = "Slack message template for alert notifications" default = "" diff --git a/prometheus_all/resources.tf b/prometheus_all/resources.tf index e28d23c..60b2428 100644 --- a/prometheus_all/resources.tf +++ b/prometheus_all/resources.tf @@ -104,7 +104,7 @@ module "alertmanager" { monitoring_space_id = data.cloudfoundry_space.monitoring.id config = var.alertmanager_config slack_url = var.alertmanager_slack_url - slack_channel = var.alertmanager_slack_channel + slack_receivers = var.alertmanager_slack_receivers slack_template = var.alertmanager_slack_template docker_credentials = var.docker_credentials }