Add the ability to delete original alert when matching resolved message arrives

README.md

@@ -9,11 +9,13 @@ Bridge between Prometheus Alertmanager and Gotify
 optional arguments:
   -h, --help            show this help message and exit
   -c CONFIG, --config CONFIG
-                        path to config YAML. (default: alertify.yaml)
+                        path to config YAML. (default: ./alertify.yaml)
   -H, --healthcheck     simply exit with 0 for healthy or 1 when unhealthy
 
 The following environment variables will override any config or default:
+  * DELETE_ONRESOLVE (default: False)
   * DISABLE_RESOLVED (default: False)
+  * GOTIFY_CLIENT    (default: None)
   * GOTIFY_KEY       (default: None)
   * GOTIFY_PORT      (default: 80)
   * GOTIFY_SERVER    (default: localhost)
@@ -24,13 +26,15 @@ The following environment variables will override any config or default:
 
 # Notes
 * Listens on port 8080 by default.
-* Forwards `resolved` alerts, if sent.
+* Forwards `resolved` alerts, if not disabled.
+* Resolved alerts delete the original alert, if enabled.
 * Defaults, if not sent:
   | Field       | Default value |
   |-------------|---------------|
+  | Description | `[nodata]`    |
+  | Instance    | `[unknown]`   |
   | Priority    | `5`           |
-  | Description | `...`         |
+  | Severity    | `Warning`     |
-  | Severity    | `Default`     |
 
 
 # Docker
@@ -43,7 +47,7 @@ docker build . -t 'alertify:latest'
 
 e.g.
 ```bash
-docker run --name alertify -p 8080:8080 -e TZ=Europe/London -e GOTIFY_KEY=XXXXXXXX -e GOTIFY_SERVER=gotify -e GOTIFY_PORT=80 alertify:latest
+docker run --name alertify -p 8080:8080 -e TZ=Europe/London -e GOTIFY_KEY=_APPKEY_ -e GOTIFY_SERVER=gotify -e GOTIFY_PORT=80 alertify:latest
 ```
 
 ## Compose:
@@ -68,7 +72,8 @@ services:
       - "8080:8080"
     environment:
       - TZ=Europe/London
-      - GOTIFY_KEY=XXXXXXXXXXXX
+      - GOTIFY_KEY=_APPKEY_
+      - GOTIFY_CLIENT=_CLIENTKEY_
       - GOTIFY_SERVER=gotify
       - GOTIFY_PORT=80
     restart: unless-stopped

alertify.py

@@ -15,7 +15,9 @@ from http.server import HTTPServer, SimpleHTTPRequestHandler
 import yaml
 
 DEFAULTS = {
+    'delete_onresolve': bool(False),
     'disable_resolved': bool(False),
+    'gotify_client': str(),
     'gotify_key': str(),
     'gotify_port': int(80),
     'gotify_server': str('localhost'),
@@ -24,50 +26,119 @@ DEFAULTS = {
 }
 
 
+class Gotify:
+    """
+    Class to handle Gotify communications
+    """
+    verbose = False
+
+    def __init__(self, server, port, app_key, client_key=None):
+        self.api = http.client.HTTPConnection(server, port)
+        self.app_key = app_key
+        self.client_key = client_key
+        self.base_headers = {
+            'Content-type': 'application/json',
+            'Accept': 'application/json',
+        }
+
+    def _call(self, method, url, body=None):
+        """
+        Method to call Gotify with an app or client key as appropriate
+        """
+        headers = self.base_headers.copy()
+        if method in ['GET', 'DELETE']:
+            headers['X-Gotify-Key'] = self.client_key
+        else:
+            headers['X-Gotify-Key'] = self.app_key
+
+        if self.verbose:
+            print('Sending to Gotify:')
+            print(body)
+
+        try:
+            self.api.request(method, url, body=body, headers=headers)
+            response = self.api.getresponse()
+        except ConnectionRefusedError as error:
+            print(f'ERROR: {error}')
+            return {
+                'status': error.errno,
+                'reason': error.strerror
+            }
+
+        resp_obj = {
+            'status': response.status,
+            'reason': response.reason,
+            'json': None
+        }
+        rawbody = response.read()
+        if len(rawbody) > 0:
+            try:
+                resp_obj['json'] = json.loads(rawbody.decode())
+            except json.decoder.JSONDecodeError as error:
+                print(f'ERROR: {error}')
+
+        if self.verbose:
+            print('Returned from Gotify:')
+            print(json.dumps(resp_obj, indent=2))
+            print('Status: {status}, Reason: {reason}'.format(**resp_obj))
+
+        return resp_obj
+
+    def delete(self, msg_id):
+        """
+        Method to delete a message from the Gotify server
+        """
+        if self.verbose:
+            print(f'Deleting message ID {msg_id}')
+        return self._call('DELETE', f'/message/{msg_id}')
+
+    def find_byfingerprint(self, message):
+        """
+        Method to return the ID of a matching message
+        """
+        try:
+            new_fingerprint = message['fingerprint']
+        except KeyError:
+            if self.verbose:
+                print('No fingerprint found in new message')
+            return None
+
+        for old_message in self.messages():
+            try:
+                old_fingerprint = old_message['extras']['alertify']['fingerprint']
+                if old_fingerprint == new_fingerprint:
+                    return old_message['id']
+            except KeyError:
+                if self.verbose:
+                    print(
+                        f'No fingerprint found in message {old_message["id"]}')
+                continue
+
+        return None
+
+    def messages(self):
+        """
+        Method to return a list of messages from the Gotify server
+        """
+        if self.verbose:
+            print('Fetching existing messages from Gotify')
+        return self._call('GET', '/message')['json'].get('messages', None)
+
+    def send_alert(self, payload):
+        """
+        Method to send a message payload to a Gotify server
+        """
+        if self.verbose:
+            print('Sending message to Gotify')
+        return self._call('POST', '/message', body=json.dumps(payload, indent=2))
+
+
 class HTTPHandler(SimpleHTTPRequestHandler):
     """
     Class to handle the HTTP requests from a client
     """
-
     config = None
 
-    @staticmethod
-    def set_config(config):
-        """
-        Method
-        """
-        HTTPHandler.config = config
-
-    # Override built-in method
-    # pylint: disable=invalid-name
-    def do_GET(self):
-        """
-        Method to handle GET requests
-        """
-        if self.path == '/healthcheck':
-            if not healthy(self.config):
-                print('ERROR: Check requirements')
-                self._respond(500, 'ERR')
-
-            self._respond(200, 'OK')
-
-    # Override built-in method
-    # pylint: disable=invalid-name
-    def do_POST(self):
-        """
-        Method to handle POST requests from AlertManager
-        """
-        if self.path == '/alert':
-            self._alerts()
-
-    def _respond(self, status, message):
-        """
-        Method to output a simple HTTP status and string to the client
-        """
-        self.send_response(int(status) or 500)
-        self.end_headers()
-        self.wfile.write(bytes(str(message).encode()))
-
     def _alerts(self):
         """
         Method to handle the request for alerts
@@ -81,7 +152,7 @@ class HTTPHandler(SimpleHTTPRequestHandler):
         rawdata = self.rfile.read(content_length)
 
         try:
-            alert = json.loads(rawdata.decode())
+            am_msg = json.loads(rawdata.decode())
         except json.decoder.JSONDecodeError as error:
             print(f'ERROR: Bad JSON: {error}')
             self._respond(400, f'Bad JSON: {error}')
@@ -89,76 +160,99 @@ class HTTPHandler(SimpleHTTPRequestHandler):
 
         if self.config.get('verbose'):
             print('Received from Alertmanager:')
-            print(json.dumps(alert, indent=2))
+            print(json.dumps(am_msg, indent=2))
 
-        try:
+        gotify = Gotify(
-            if alert['status'] == 'resolved':
-                if self.config.get('disable_resolved'):
-                    print('Ignoring resolved messages')
-                    self._respond(200, 'Ignored. "resolved" messages are disabled')
-                    return
-                prefix = 'Resolved'
-            else:
-                prefix = alert['commonLabels'].get(
-                    'severity', 'default').capitalize()
-
-            gotify_msg = {
-                'title': '{}: {}'.format(
-                    alert['receiver'],
-                    alert['commonLabels'].get('instance', 'Unknown')
-                ),
-                'message': '{}: {}'.format(
-                    prefix,
-                    alert['commonAnnotations'].get('description', '...')
-                ),
-                'priority': int(alert['commonLabels'].get('priority', 5))
-            }
-        except KeyError as error:
-            print(f'ERROR: KeyError: {error}')
-            self._respond(400, f'Missing field: {error}')
-            return
-
-        if self.config.get('verbose'):
-            print('Sending to Gotify:')
-            print(json.dumps(gotify_msg, indent=2))
-
-        response = gotify_send(
             self.config.get('gotify_server'),
             self.config.get('gotify_port'),
             self.config.get('gotify_key'),
-            gotify_msg
+            self.config.get('gotify_client')
         )
 
         if self.config.get('verbose'):
-            print('Status: {status}, Reason: {reason}'.format(**response))
+            gotify.verbose = True
-        self._respond(response['status'], response['reason'])
 
+        for alert in am_msg['alerts']:
+            try:
+                if alert['status'] == 'resolved':
+                    if self.config.get('disable_resolved'):
+                        print('Ignoring resolved messages')
+                        self._respond(
+                            200, 'Ignored. "resolved" messages are disabled')
+                        continue
+                    if self.config.get('delete_onresolve'):
+                        alert_id = gotify.find_byfingerprint(alert)
+                        if alert_id:
+                            response = gotify.delete(alert_id)
+                        continue
+                    prefix = 'Resolved'
+                else:
+                    prefix = alert['labels'].get(
+                        'severity', 'warning').capitalize()
 
-def gotify_send(server, port, authkey, payload):
+                gotify_msg = {
-    """
+                    'title': '{}: {}'.format(
-    Function to POST data to a Gotify server
+                        prefix,
-    """
+                        alert['annotations'].get('description', '[nodata]'),
+                    ),
+                    'message': '{}: {}'.format(
+                        alert['labels'].get('instance', '[unknown]'),
+                        alert['annotations'].get('summary'),
+                    ),
+                    'priority': int(alert['labels'].get('priority', 5)),
+                    'extras': {
+                        'alertify': {
+                            'fingerprint': alert.get('fingerprint', None)
+                        }
+                    }
+                }
+            except KeyError as error:
+                print(f'ERROR: KeyError: {error}')
+                self._respond(400, f'Missing field: {error}')
+                return
 
-    gotify = http.client.HTTPConnection(server, port)
+            response = gotify.send_alert(gotify_msg)
-    headers = {
-        'X-Gotify-Key': authkey,
-        'Content-type': 'application/json',
-    }
 
-    try:
+        try:
-        gotify.request('POST', '/message', json.dumps(payload), headers)
+            self._respond(response['status'], response['reason'])
-        response = gotify.getresponse()
+        except UnboundLocalError:
-    except ConnectionRefusedError as error:
+            self._respond('204', '')
-        print(f'ERROR: {error}')
-        return {
-            'status': error.errno,
-            'reason': error.strerror
-        }
 
-    return {
+    def _respond(self, status, message):
-        'status': response.status,
+        """
-        'reason': response.reason
+        Method to output a simple HTTP status and string to the client
-    }
+        """
+        self.send_response(int(status) or 500)
+        self.end_headers()
+        self.wfile.write(bytes(str(message).encode()))
+
+    # Override built-in method
+    def do_GET(self):   # pylint: disable=invalid-name
+        """
+        Method to handle GET requests
+        """
+        if self.path == '/healthcheck':
+            if not healthy(self.config):
+                print('ERROR: Check requirements')
+                self._respond(500, 'ERR')
+
+            self._respond(200, 'OK')
+
+    # Override built-in method
+    def do_POST(self):  # pylint: disable=invalid-name
+        """
+        Method to handle POST requests from AlertManager
+        """
+        if self.path == '/alert':
+            self._alerts()
+
+    # FIXME: This isn't right.  A normal method doesn't work, however.
+    @classmethod
+    def set_config(cls, config):
+        """
+        Classmethod to add config to the class
+        """
+        cls.config = config
 
 
 def healthy(config):
@@ -196,7 +290,10 @@ def parse_config(configfile):
             config[key] = type(val)(config[key])
 
     if config['verbose']:
-        print(f'Config:\n{yaml.dump(config, explicit_start=True, default_flow_style=False)}')
+        print(
+            f'Config:\n'
+            f'{yaml.dump(config, explicit_start=True, default_flow_style=False)}'
+        )
     return config
 
 

test.sh

@@ -2,56 +2,86 @@
 
 SERVER=${1:-'localhost:8080'}
 
-name=testAlert-$RANDOM
+NAME=testAlert-$RANDOM
+FINGERPRINT=$(date +%s | md5sum | cut -f1 -d' ')
 URL="http://${SERVER}/alert"
-bold=$(tput bold)
+BOLD=$(tput bold)
-normal=$(tput sgr0)
+NORMAL=$(tput sgr0)
 
 call_alertmanager() {
-    curl -v "${URL}" --header 'Content-type: application/json' --data @<(cat <<EOF
+    VALUE=${1}
+    curl -v "${URL}" --header 'Expect:' --header 'Content-type: application/json' --data @<(cat <<EOF
 {
-  "version": "4",
-  "groupKey": "testGroup",
-  "truncatedAlerts": 0,
-  "status": "${STATUS}",
   "receiver": "alertify",
-  "commonLabels": {
+  "status": "${STATUS}",
-    "alertname": "${name}",
-    "service": "testService",
-    "severity":"warning",
-    "instance": "server.example.net",
-    "namespace": "testNamespace",
-    "label_costcentre": "testCostCentre"
-  },
-  "commonAnnotations": {
-    "summary": "Testing latency is high!",
-    "description": "Testing latency is at ${1}"
-  },
   "alerts": [
     {
       "status": "${STATUS}",
-      "generatorURL": "http://alertmanager.example.net/$name",
+      "labels": {
+        "alertname": "${NAME}",
+        "id": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
+        "instance": "localhost:1234",
+        "job": "test_job",
+        "name": "testserver",
+        "priority": "1",
+        "severity": "low",
+        "value": "${VALUE}"
+      },
+      "annotations": {
+        "description": "testserver: unhealthy",
+        "summary": "Server unhealthy"
+      },
       "startsAt": "${START}",
-      "endsAt": "${END}"
+      "endsAt": "${END}",
+      "generatorURL": "http://example.com/some/url",
+      "fingerprint": "${FINGERPRINT}"
     }
-  ]
+  ],
+  "groupLabels": {
+    "alertname": "${NAME}",
+    "id": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
+    "instance": "localhost:1234",
+    "job": "test_job",
+    "name": "testserver",
+    "priority": "1",
+    "severity": "low",
+    "value": "${VALUE}"
+  },
+  "commonLabels": {
+    "alertname": "${NAME}",
+    "id": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
+    "instance": "localhost:1234",
+    "job": "test_job",
+    "name": "testserver",
+    "priority": "1",
+    "severity": "low",
+    "value": "${VALUE}"
+  },
+  "commonAnnotations": {
+    "description": "testserver: unhealthy",
+    "summary": "Server unhealthy"
+  },
+  "externalURL": "http://1ff297bc31a0:9093",
+  "version": "4",
+  "groupKey": "{}:{alertname=\"${NAME}\", id=\"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b\", instance=\"localhost:1234\", job=\"test_job\", name=\"testserver\", priority=\"1\", severity=\"low\", value=\"${VALUE}\"}",
+  "truncatedAlerts": 0
 }
 EOF
 )
 }
 
-echo "${bold}Firing alert ${name} ${normal}"
+echo "${BOLD}Firing alert ${NAME} ${NORMAL}"
 STATUS='firing'
 START=$(date --rfc-3339=seconds | sed 's/ /T/')
 END="0001-01-01T00:00:00Z"
 call_alertmanager 42
 echo -e "\n"
 
-echo "${bold}Press enter to resolve alert ${name} ${normal}"
+echo "${BOLD}Press enter to resolve alert ${NAME} ${NORMAL}"
 read -r
 
 
-echo "${bold}Sending resolved ${normal}"
+echo "${BOLD}Sending resolved ${NORMAL}"
 STATUS='resolved'
 END=$(date --rfc-3339=seconds | sed 's/ /T/')
 call_alertmanager 0