Deprecated input & add optional Slack receivers

DEPRECATED INPUT: alertmanager_slack_channel, this input is not
required as Slack incoming webhooks are assigned to a channel.

An optional input accepts a map of alertmanager routes & receivers
along with Slack webhooks that are associated with the receivers.  Each
route is allocated a matcher of the same name so that alerts can be
routed to it.  These routes are in addition to the default routes.

alertmanager/README.md

@@ -9,12 +9,14 @@ It is deployed using the official docker image.
   monitoring_instance_name    MANDATORY
   config                      OPTIONAL
   slack_url                   OPTIONAL
-  slack_channel               OPTIONAL
+  slack_receivers             OPTIONAL
 ```
 
 - **monitoring_instance_name:** A unique name given to the application
 - **monitoring_space_id:** The Cloud Foundry space you wish to deploy the application to
 - **config:**  The contents of an alertmanager.yml file, as specified in the [documentation](https://prometheus.io/docs/alerting/latest/configuration/). If not specified, a dummy configuration is deployed.
+- **slack_url** An incoming webhook created at https://api.slack.com/apps/appid/incoming-webhooks? (replace appid with AppId value).  Incoming webhooks are assigned to a single channel so the channel does not need to be specified.  The default destination for all alerts that do not satisfy the matches defined in other routes.
+- **slack_receivers** A map of route\receiver names and webhook URLs.  These can be used to route alerts to different channels based on alert labels.  Alerts should include a label called `receiver` whose value is the same as the name of the receiver defined in this config property.
 
 ### Example
 ```hcl
@@ -24,7 +26,10 @@ module alertmanager {
   monitoring_instance_name = "test-alertmanager"
   config                   = file("${path.module}/files/alertmanager.yml")
   slack_url                = https://hooks.slack.com/services/XXXXXXXXX/YYYYYYYYYYY/xxxxxxxxxxxxxxxxxxxxxxxx
-  slack_channel            = mychannel
+  slack_receivers          = {
+    "service_one_tech": "https://hooks.slack.com/services/XXXXXXXXX/YYYYYYYYYYY/xxxxxxxxxxxxxxxxxxxxxxxx",
+    "service_two_tech": "https://hooks.slack.com/services/XXXXXXXXX/YYYYYYYYYYY/xxxxxxxxxxxxxxxxxxxxxxxx"
+  }
 }
 ```
 

alertmanager/config/alertmanager.yml.tmpl

@@ -5,6 +5,7 @@ global:
 %{ endif ~}
 
 route:
+  # the default route, this will be used if the matcher(s) of any of the child routes are not satisfied
   receiver: 'slack-notifications'
   group_interval: 1m
   repeat_interval: 1h
@@ -21,6 +22,13 @@ route:
     mute_time_intervals:
     - out-of-hours
     matchers: [period = 'out-of-hours']
+%{ for receiver, webhook in slack_receivers ~}
+  - receiver: ${ receiver }
+    group_interval: 1m
+    repeat_interval: 1h
+    matchers:
+    - receiver="${ receiver }"
+%{ endfor ~}
 
 mute_time_intervals:
 - name: out-of-hours
@@ -41,13 +49,13 @@ receivers:
 - name: slack-notifications
 %{ if slack_url != "" ~}
   slack_configs:
-  - channel: '#${slack_channel}'
-    send_resolved: true
+  - send_resolved: true
     title: '{{ template "slack.monzo.title" . }}'
     pretext: '{{ template "slack.monzo.pretext" . }}'
     title_link: '{{ template "__alertmanagerURL" . }}'
     text: '{{ template "slack.monzo.text" . }}'
     color: '{{ template "slack.monzo.color" . }}'
+    footer: 'Sent to default route'
     actions:
     - type: button
       text: 'Runbook :green_book:'
@@ -62,3 +70,26 @@ receivers:
       text: 'Silence :no_bell:'
       url: '{{ template "__alert_silence_link" . }}'
 %{ endif ~}
+%{ for receiver, webhook in slack_receivers ~}
+- name: ${ receiver }
+  slack_configs:
+  - api_url: ${ webhook }
+    send_resolved: true
+    title: '{{ template "slack.monzo.title" . }}'
+    pretext: '{{ template "slack.monzo.pretext" . }}'
+    text: '{{ template "slack.monzo.text" . }}'
+    color: '{{ template "slack.monzo.color" . }}'
+    actions:
+    - type: button
+      text: 'Runbook :green_book:'
+      url: '{{ (index .Alerts 0).Annotations.runbook }}'
+    - type: button
+      text: 'Query :mag:'
+      url: '{{ (index .Alerts 0).GeneratorURL }}'
+    - type: button
+      text: 'Dashboard :bar_chart:'
+      url: '{{ (index .Alerts 0).Annotations.dashboard }}'
+    - type: button
+      text: 'Silence :no_bell:'
+      url: '{{ template "__alert_silence_link" . }}'
+%{ endfor ~}

alertmanager/input.tf

@@ -5,8 +5,11 @@ variable "monitoring_space_id" {}
 variable "config" { default = "" }
 
 variable "slack_url" { default = "" }
-variable "slack_channel" { default = "" }
 variable "slack_template" { default = "" }
+
+variable "slack_receivers" {
+  type = map(string)
+}
 variable "docker_credentials" {
   description = "Credentials for Dockerhub. Map of {username, password}."
   type        = map(any)
@@ -18,8 +21,8 @@ variable "docker_credentials" {
 locals {
   docker_image_tag = "v0.22.2"
   alertmanager_variables = {
-    slack_url     = var.slack_url
-    slack_channel = var.slack_channel
+    slack_url       = var.slack_url
+    slack_receivers = var.slack_receivers
   }
   config         = var.config == "" ? templatefile("${path.module}/config/alertmanager.yml.tmpl", local.alertmanager_variables) : var.config
   slack_template = var.slack_template == "" ? file("${path.module}/config/slack.tmpl") : var.slack_template

prometheus_all/input.tf

@@ -27,9 +27,13 @@ variable "alertmanager_slack_url" {
   default     = ""
 }
 variable "alertmanager_slack_channel" {
-  description = "Slack channel for alert notifications"
+  description = "DEPRECATED.  Slack channel for alert notifications.  This is unnecessary if the webhook in alertmanager_slack_url has been assigned to a channel."
   default     = ""
 }
+variable "alertmanager_slack_receivers" {
+  type    = map(string)
+  default = {}
+}
 variable "alertmanager_slack_template" {
   description = "Slack message template for alert notifications"
   default     = ""

prometheus_all/resources.tf

@@ -104,7 +104,7 @@ module "alertmanager" {
   monitoring_space_id      = data.cloudfoundry_space.monitoring.id
   config                   = var.alertmanager_config
   slack_url                = var.alertmanager_slack_url
-  slack_channel            = var.alertmanager_slack_channel
+  slack_receivers          = var.alertmanager_slack_receivers
   slack_template           = var.alertmanager_slack_template
   docker_credentials       = var.docker_credentials
 }