From 12b39b1c08bd4bc40faac8caacf7f25bb33f1ff5 Mon Sep 17 00:00:00 2001 From: Scott Wallace Date: Fri, 23 Oct 2020 10:06:51 +0100 Subject: [PATCH] Add the ability to delete original alert when matching resolved message arrives --- README.md | 17 +-- alertify.py | 293 ++++++++++++++++++++++++++++++++++------------------ test.sh | 82 ++++++++++----- 3 files changed, 262 insertions(+), 130 deletions(-) diff --git a/README.md b/README.md index 406e803..04ef418 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,13 @@ Bridge between Prometheus Alertmanager and Gotify optional arguments: -h, --help show this help message and exit -c CONFIG, --config CONFIG - path to config YAML. (default: alertify.yaml) + path to config YAML. (default: ./alertify.yaml) -H, --healthcheck simply exit with 0 for healthy or 1 when unhealthy The following environment variables will override any config or default: + * DELETE_ONRESOLVE (default: False) * DISABLE_RESOLVED (default: False) + * GOTIFY_CLIENT (default: None) * GOTIFY_KEY (default: None) * GOTIFY_PORT (default: 80) * GOTIFY_SERVER (default: localhost) @@ -24,13 +26,15 @@ The following environment variables will override any config or default: # Notes * Listens on port 8080 by default. -* Forwards `resolved` alerts, if sent. +* Forwards `resolved` alerts, if not disabled. +* Resolved alerts delete the original alert, if enabled. * Defaults, if not sent: | Field | Default value | |-------------|---------------| + | Description | `[nodata]` | + | Instance | `[unknown]` | | Priority | `5` | - | Description | `...` | - | Severity | `Default` | + | Severity | `Warning` | # Docker @@ -43,7 +47,7 @@ docker build . -t 'alertify:latest' e.g. ```bash -docker run --name alertify -p 8080:8080 -e TZ=Europe/London -e GOTIFY_KEY=XXXXXXXX -e GOTIFY_SERVER=gotify -e GOTIFY_PORT=80 alertify:latest +docker run --name alertify -p 8080:8080 -e TZ=Europe/London -e GOTIFY_KEY=_APPKEY_ -e GOTIFY_SERVER=gotify -e GOTIFY_PORT=80 alertify:latest ``` ## Compose: @@ -68,7 +72,8 @@ services: - "8080:8080" environment: - TZ=Europe/London - - GOTIFY_KEY=XXXXXXXXXXXX + - GOTIFY_KEY=_APPKEY_ + - GOTIFY_CLIENT=_CLIENTKEY_ - GOTIFY_SERVER=gotify - GOTIFY_PORT=80 restart: unless-stopped diff --git a/alertify.py b/alertify.py index 631c63a..2d177b4 100644 --- a/alertify.py +++ b/alertify.py @@ -15,7 +15,9 @@ from http.server import HTTPServer, SimpleHTTPRequestHandler import yaml DEFAULTS = { + 'delete_onresolve': bool(False), 'disable_resolved': bool(False), + 'gotify_client': str(), 'gotify_key': str(), 'gotify_port': int(80), 'gotify_server': str('localhost'), @@ -24,50 +26,119 @@ DEFAULTS = { } +class Gotify: + """ + Class to handle Gotify communications + """ + verbose = False + + def __init__(self, server, port, app_key, client_key=None): + self.api = http.client.HTTPConnection(server, port) + self.app_key = app_key + self.client_key = client_key + self.base_headers = { + 'Content-type': 'application/json', + 'Accept': 'application/json', + } + + def _call(self, method, url, body=None): + """ + Method to call Gotify with an app or client key as appropriate + """ + headers = self.base_headers.copy() + if method in ['GET', 'DELETE']: + headers['X-Gotify-Key'] = self.client_key + else: + headers['X-Gotify-Key'] = self.app_key + + if self.verbose: + print('Sending to Gotify:') + print(body) + + try: + self.api.request(method, url, body=body, headers=headers) + response = self.api.getresponse() + except ConnectionRefusedError as error: + print(f'ERROR: {error}') + return { + 'status': error.errno, + 'reason': error.strerror + } + + resp_obj = { + 'status': response.status, + 'reason': response.reason, + 'json': None + } + rawbody = response.read() + if len(rawbody) > 0: + try: + resp_obj['json'] = json.loads(rawbody.decode()) + except json.decoder.JSONDecodeError as error: + print(f'ERROR: {error}') + + if self.verbose: + print('Returned from Gotify:') + print(json.dumps(resp_obj, indent=2)) + print('Status: {status}, Reason: {reason}'.format(**resp_obj)) + + return resp_obj + + def delete(self, msg_id): + """ + Method to delete a message from the Gotify server + """ + if self.verbose: + print(f'Deleting message ID {msg_id}') + return self._call('DELETE', f'/message/{msg_id}') + + def find_byfingerprint(self, message): + """ + Method to return the ID of a matching message + """ + try: + new_fingerprint = message['fingerprint'] + except KeyError: + if self.verbose: + print('No fingerprint found in new message') + return None + + for old_message in self.messages(): + try: + old_fingerprint = old_message['extras']['alertify']['fingerprint'] + if old_fingerprint == new_fingerprint: + return old_message['id'] + except KeyError: + if self.verbose: + print( + f'No fingerprint found in message {old_message["id"]}') + continue + + return None + + def messages(self): + """ + Method to return a list of messages from the Gotify server + """ + if self.verbose: + print('Fetching existing messages from Gotify') + return self._call('GET', '/message')['json'].get('messages', None) + + def send_alert(self, payload): + """ + Method to send a message payload to a Gotify server + """ + if self.verbose: + print('Sending message to Gotify') + return self._call('POST', '/message', body=json.dumps(payload, indent=2)) + + class HTTPHandler(SimpleHTTPRequestHandler): """ Class to handle the HTTP requests from a client """ - config = None - @staticmethod - def set_config(config): - """ - Method - """ - HTTPHandler.config = config - - # Override built-in method - # pylint: disable=invalid-name - def do_GET(self): - """ - Method to handle GET requests - """ - if self.path == '/healthcheck': - if not healthy(self.config): - print('ERROR: Check requirements') - self._respond(500, 'ERR') - - self._respond(200, 'OK') - - # Override built-in method - # pylint: disable=invalid-name - def do_POST(self): - """ - Method to handle POST requests from AlertManager - """ - if self.path == '/alert': - self._alerts() - - def _respond(self, status, message): - """ - Method to output a simple HTTP status and string to the client - """ - self.send_response(int(status) or 500) - self.end_headers() - self.wfile.write(bytes(str(message).encode())) - def _alerts(self): """ Method to handle the request for alerts @@ -81,7 +152,7 @@ class HTTPHandler(SimpleHTTPRequestHandler): rawdata = self.rfile.read(content_length) try: - alert = json.loads(rawdata.decode()) + am_msg = json.loads(rawdata.decode()) except json.decoder.JSONDecodeError as error: print(f'ERROR: Bad JSON: {error}') self._respond(400, f'Bad JSON: {error}') @@ -89,76 +160,99 @@ class HTTPHandler(SimpleHTTPRequestHandler): if self.config.get('verbose'): print('Received from Alertmanager:') - print(json.dumps(alert, indent=2)) + print(json.dumps(am_msg, indent=2)) - try: - if alert['status'] == 'resolved': - if self.config.get('disable_resolved'): - print('Ignoring resolved messages') - self._respond(200, 'Ignored. "resolved" messages are disabled') - return - prefix = 'Resolved' - else: - prefix = alert['commonLabels'].get( - 'severity', 'default').capitalize() - - gotify_msg = { - 'title': '{}: {}'.format( - alert['receiver'], - alert['commonLabels'].get('instance', 'Unknown') - ), - 'message': '{}: {}'.format( - prefix, - alert['commonAnnotations'].get('description', '...') - ), - 'priority': int(alert['commonLabels'].get('priority', 5)) - } - except KeyError as error: - print(f'ERROR: KeyError: {error}') - self._respond(400, f'Missing field: {error}') - return - - if self.config.get('verbose'): - print('Sending to Gotify:') - print(json.dumps(gotify_msg, indent=2)) - - response = gotify_send( + gotify = Gotify( self.config.get('gotify_server'), self.config.get('gotify_port'), self.config.get('gotify_key'), - gotify_msg + self.config.get('gotify_client') ) if self.config.get('verbose'): - print('Status: {status}, Reason: {reason}'.format(**response)) - self._respond(response['status'], response['reason']) + gotify.verbose = True + for alert in am_msg['alerts']: + try: + if alert['status'] == 'resolved': + if self.config.get('disable_resolved'): + print('Ignoring resolved messages') + self._respond( + 200, 'Ignored. "resolved" messages are disabled') + continue + if self.config.get('delete_onresolve'): + alert_id = gotify.find_byfingerprint(alert) + if alert_id: + response = gotify.delete(alert_id) + continue + prefix = 'Resolved' + else: + prefix = alert['labels'].get( + 'severity', 'warning').capitalize() -def gotify_send(server, port, authkey, payload): - """ - Function to POST data to a Gotify server - """ + gotify_msg = { + 'title': '{}: {}'.format( + prefix, + alert['annotations'].get('description', '[nodata]'), + ), + 'message': '{}: {}'.format( + alert['labels'].get('instance', '[unknown]'), + alert['annotations'].get('summary'), + ), + 'priority': int(alert['labels'].get('priority', 5)), + 'extras': { + 'alertify': { + 'fingerprint': alert.get('fingerprint', None) + } + } + } + except KeyError as error: + print(f'ERROR: KeyError: {error}') + self._respond(400, f'Missing field: {error}') + return - gotify = http.client.HTTPConnection(server, port) - headers = { - 'X-Gotify-Key': authkey, - 'Content-type': 'application/json', - } + response = gotify.send_alert(gotify_msg) - try: - gotify.request('POST', '/message', json.dumps(payload), headers) - response = gotify.getresponse() - except ConnectionRefusedError as error: - print(f'ERROR: {error}') - return { - 'status': error.errno, - 'reason': error.strerror - } + try: + self._respond(response['status'], response['reason']) + except UnboundLocalError: + self._respond('204', '') - return { - 'status': response.status, - 'reason': response.reason - } + def _respond(self, status, message): + """ + Method to output a simple HTTP status and string to the client + """ + self.send_response(int(status) or 500) + self.end_headers() + self.wfile.write(bytes(str(message).encode())) + + # Override built-in method + def do_GET(self): # pylint: disable=invalid-name + """ + Method to handle GET requests + """ + if self.path == '/healthcheck': + if not healthy(self.config): + print('ERROR: Check requirements') + self._respond(500, 'ERR') + + self._respond(200, 'OK') + + # Override built-in method + def do_POST(self): # pylint: disable=invalid-name + """ + Method to handle POST requests from AlertManager + """ + if self.path == '/alert': + self._alerts() + + # FIXME: This isn't right. A normal method doesn't work, however. + @classmethod + def set_config(cls, config): + """ + Classmethod to add config to the class + """ + cls.config = config def healthy(config): @@ -196,7 +290,10 @@ def parse_config(configfile): config[key] = type(val)(config[key]) if config['verbose']: - print(f'Config:\n{yaml.dump(config, explicit_start=True, default_flow_style=False)}') + print( + f'Config:\n' + f'{yaml.dump(config, explicit_start=True, default_flow_style=False)}' + ) return config diff --git a/test.sh b/test.sh index 925c4f8..44300ac 100755 --- a/test.sh +++ b/test.sh @@ -2,56 +2,86 @@ SERVER=${1:-'localhost:8080'} -name=testAlert-$RANDOM +NAME=testAlert-$RANDOM +FINGERPRINT=$(date +%s | md5sum | cut -f1 -d' ') URL="http://${SERVER}/alert" -bold=$(tput bold) -normal=$(tput sgr0) +BOLD=$(tput bold) +NORMAL=$(tput sgr0) call_alertmanager() { - curl -v "${URL}" --header 'Content-type: application/json' --data @<(cat <