Add the ability to delete original alert when matching resolved message arrives
This commit is contained in:
parent
3c57c113cd
commit
12b39b1c08
17
README.md
17
README.md
|
|
@ -9,11 +9,13 @@ Bridge between Prometheus Alertmanager and Gotify
|
|||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-c CONFIG, --config CONFIG
|
||||
path to config YAML. (default: alertify.yaml)
|
||||
path to config YAML. (default: ./alertify.yaml)
|
||||
-H, --healthcheck simply exit with 0 for healthy or 1 when unhealthy
|
||||
|
||||
The following environment variables will override any config or default:
|
||||
* DELETE_ONRESOLVE (default: False)
|
||||
* DISABLE_RESOLVED (default: False)
|
||||
* GOTIFY_CLIENT (default: None)
|
||||
* GOTIFY_KEY (default: None)
|
||||
* GOTIFY_PORT (default: 80)
|
||||
* GOTIFY_SERVER (default: localhost)
|
||||
|
|
@ -24,13 +26,15 @@ The following environment variables will override any config or default:
|
|||
|
||||
# Notes
|
||||
* Listens on port 8080 by default.
|
||||
* Forwards `resolved` alerts, if sent.
|
||||
* Forwards `resolved` alerts, if not disabled.
|
||||
* Resolved alerts delete the original alert, if enabled.
|
||||
* Defaults, if not sent:
|
||||
| Field | Default value |
|
||||
|-------------|---------------|
|
||||
| Description | `[nodata]` |
|
||||
| Instance | `[unknown]` |
|
||||
| Priority | `5` |
|
||||
| Description | `...` |
|
||||
| Severity | `Default` |
|
||||
| Severity | `Warning` |
|
||||
|
||||
|
||||
# Docker
|
||||
|
|
@ -43,7 +47,7 @@ docker build . -t 'alertify:latest'
|
|||
|
||||
e.g.
|
||||
```bash
|
||||
docker run --name alertify -p 8080:8080 -e TZ=Europe/London -e GOTIFY_KEY=XXXXXXXX -e GOTIFY_SERVER=gotify -e GOTIFY_PORT=80 alertify:latest
|
||||
docker run --name alertify -p 8080:8080 -e TZ=Europe/London -e GOTIFY_KEY=_APPKEY_ -e GOTIFY_SERVER=gotify -e GOTIFY_PORT=80 alertify:latest
|
||||
```
|
||||
|
||||
## Compose:
|
||||
|
|
@ -68,7 +72,8 @@ services:
|
|||
- "8080:8080"
|
||||
environment:
|
||||
- TZ=Europe/London
|
||||
- GOTIFY_KEY=XXXXXXXXXXXX
|
||||
- GOTIFY_KEY=_APPKEY_
|
||||
- GOTIFY_CLIENT=_CLIENTKEY_
|
||||
- GOTIFY_SERVER=gotify
|
||||
- GOTIFY_PORT=80
|
||||
restart: unless-stopped
|
||||
|
|
|
|||
293
alertify.py
293
alertify.py
|
|
@ -15,7 +15,9 @@ from http.server import HTTPServer, SimpleHTTPRequestHandler
|
|||
import yaml
|
||||
|
||||
DEFAULTS = {
|
||||
'delete_onresolve': bool(False),
|
||||
'disable_resolved': bool(False),
|
||||
'gotify_client': str(),
|
||||
'gotify_key': str(),
|
||||
'gotify_port': int(80),
|
||||
'gotify_server': str('localhost'),
|
||||
|
|
@ -24,50 +26,119 @@ DEFAULTS = {
|
|||
}
|
||||
|
||||
|
||||
class Gotify:
|
||||
"""
|
||||
Class to handle Gotify communications
|
||||
"""
|
||||
verbose = False
|
||||
|
||||
def __init__(self, server, port, app_key, client_key=None):
|
||||
self.api = http.client.HTTPConnection(server, port)
|
||||
self.app_key = app_key
|
||||
self.client_key = client_key
|
||||
self.base_headers = {
|
||||
'Content-type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
}
|
||||
|
||||
def _call(self, method, url, body=None):
|
||||
"""
|
||||
Method to call Gotify with an app or client key as appropriate
|
||||
"""
|
||||
headers = self.base_headers.copy()
|
||||
if method in ['GET', 'DELETE']:
|
||||
headers['X-Gotify-Key'] = self.client_key
|
||||
else:
|
||||
headers['X-Gotify-Key'] = self.app_key
|
||||
|
||||
if self.verbose:
|
||||
print('Sending to Gotify:')
|
||||
print(body)
|
||||
|
||||
try:
|
||||
self.api.request(method, url, body=body, headers=headers)
|
||||
response = self.api.getresponse()
|
||||
except ConnectionRefusedError as error:
|
||||
print(f'ERROR: {error}')
|
||||
return {
|
||||
'status': error.errno,
|
||||
'reason': error.strerror
|
||||
}
|
||||
|
||||
resp_obj = {
|
||||
'status': response.status,
|
||||
'reason': response.reason,
|
||||
'json': None
|
||||
}
|
||||
rawbody = response.read()
|
||||
if len(rawbody) > 0:
|
||||
try:
|
||||
resp_obj['json'] = json.loads(rawbody.decode())
|
||||
except json.decoder.JSONDecodeError as error:
|
||||
print(f'ERROR: {error}')
|
||||
|
||||
if self.verbose:
|
||||
print('Returned from Gotify:')
|
||||
print(json.dumps(resp_obj, indent=2))
|
||||
print('Status: {status}, Reason: {reason}'.format(**resp_obj))
|
||||
|
||||
return resp_obj
|
||||
|
||||
def delete(self, msg_id):
|
||||
"""
|
||||
Method to delete a message from the Gotify server
|
||||
"""
|
||||
if self.verbose:
|
||||
print(f'Deleting message ID {msg_id}')
|
||||
return self._call('DELETE', f'/message/{msg_id}')
|
||||
|
||||
def find_byfingerprint(self, message):
|
||||
"""
|
||||
Method to return the ID of a matching message
|
||||
"""
|
||||
try:
|
||||
new_fingerprint = message['fingerprint']
|
||||
except KeyError:
|
||||
if self.verbose:
|
||||
print('No fingerprint found in new message')
|
||||
return None
|
||||
|
||||
for old_message in self.messages():
|
||||
try:
|
||||
old_fingerprint = old_message['extras']['alertify']['fingerprint']
|
||||
if old_fingerprint == new_fingerprint:
|
||||
return old_message['id']
|
||||
except KeyError:
|
||||
if self.verbose:
|
||||
print(
|
||||
f'No fingerprint found in message {old_message["id"]}')
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
def messages(self):
|
||||
"""
|
||||
Method to return a list of messages from the Gotify server
|
||||
"""
|
||||
if self.verbose:
|
||||
print('Fetching existing messages from Gotify')
|
||||
return self._call('GET', '/message')['json'].get('messages', None)
|
||||
|
||||
def send_alert(self, payload):
|
||||
"""
|
||||
Method to send a message payload to a Gotify server
|
||||
"""
|
||||
if self.verbose:
|
||||
print('Sending message to Gotify')
|
||||
return self._call('POST', '/message', body=json.dumps(payload, indent=2))
|
||||
|
||||
|
||||
class HTTPHandler(SimpleHTTPRequestHandler):
|
||||
"""
|
||||
Class to handle the HTTP requests from a client
|
||||
"""
|
||||
|
||||
config = None
|
||||
|
||||
@staticmethod
|
||||
def set_config(config):
|
||||
"""
|
||||
Method
|
||||
"""
|
||||
HTTPHandler.config = config
|
||||
|
||||
# Override built-in method
|
||||
# pylint: disable=invalid-name
|
||||
def do_GET(self):
|
||||
"""
|
||||
Method to handle GET requests
|
||||
"""
|
||||
if self.path == '/healthcheck':
|
||||
if not healthy(self.config):
|
||||
print('ERROR: Check requirements')
|
||||
self._respond(500, 'ERR')
|
||||
|
||||
self._respond(200, 'OK')
|
||||
|
||||
# Override built-in method
|
||||
# pylint: disable=invalid-name
|
||||
def do_POST(self):
|
||||
"""
|
||||
Method to handle POST requests from AlertManager
|
||||
"""
|
||||
if self.path == '/alert':
|
||||
self._alerts()
|
||||
|
||||
def _respond(self, status, message):
|
||||
"""
|
||||
Method to output a simple HTTP status and string to the client
|
||||
"""
|
||||
self.send_response(int(status) or 500)
|
||||
self.end_headers()
|
||||
self.wfile.write(bytes(str(message).encode()))
|
||||
|
||||
def _alerts(self):
|
||||
"""
|
||||
Method to handle the request for alerts
|
||||
|
|
@ -81,7 +152,7 @@ class HTTPHandler(SimpleHTTPRequestHandler):
|
|||
rawdata = self.rfile.read(content_length)
|
||||
|
||||
try:
|
||||
alert = json.loads(rawdata.decode())
|
||||
am_msg = json.loads(rawdata.decode())
|
||||
except json.decoder.JSONDecodeError as error:
|
||||
print(f'ERROR: Bad JSON: {error}')
|
||||
self._respond(400, f'Bad JSON: {error}')
|
||||
|
|
@ -89,76 +160,99 @@ class HTTPHandler(SimpleHTTPRequestHandler):
|
|||
|
||||
if self.config.get('verbose'):
|
||||
print('Received from Alertmanager:')
|
||||
print(json.dumps(alert, indent=2))
|
||||
print(json.dumps(am_msg, indent=2))
|
||||
|
||||
try:
|
||||
if alert['status'] == 'resolved':
|
||||
if self.config.get('disable_resolved'):
|
||||
print('Ignoring resolved messages')
|
||||
self._respond(200, 'Ignored. "resolved" messages are disabled')
|
||||
return
|
||||
prefix = 'Resolved'
|
||||
else:
|
||||
prefix = alert['commonLabels'].get(
|
||||
'severity', 'default').capitalize()
|
||||
|
||||
gotify_msg = {
|
||||
'title': '{}: {}'.format(
|
||||
alert['receiver'],
|
||||
alert['commonLabels'].get('instance', 'Unknown')
|
||||
),
|
||||
'message': '{}: {}'.format(
|
||||
prefix,
|
||||
alert['commonAnnotations'].get('description', '...')
|
||||
),
|
||||
'priority': int(alert['commonLabels'].get('priority', 5))
|
||||
}
|
||||
except KeyError as error:
|
||||
print(f'ERROR: KeyError: {error}')
|
||||
self._respond(400, f'Missing field: {error}')
|
||||
return
|
||||
|
||||
if self.config.get('verbose'):
|
||||
print('Sending to Gotify:')
|
||||
print(json.dumps(gotify_msg, indent=2))
|
||||
|
||||
response = gotify_send(
|
||||
gotify = Gotify(
|
||||
self.config.get('gotify_server'),
|
||||
self.config.get('gotify_port'),
|
||||
self.config.get('gotify_key'),
|
||||
gotify_msg
|
||||
self.config.get('gotify_client')
|
||||
)
|
||||
|
||||
if self.config.get('verbose'):
|
||||
print('Status: {status}, Reason: {reason}'.format(**response))
|
||||
self._respond(response['status'], response['reason'])
|
||||
gotify.verbose = True
|
||||
|
||||
for alert in am_msg['alerts']:
|
||||
try:
|
||||
if alert['status'] == 'resolved':
|
||||
if self.config.get('disable_resolved'):
|
||||
print('Ignoring resolved messages')
|
||||
self._respond(
|
||||
200, 'Ignored. "resolved" messages are disabled')
|
||||
continue
|
||||
if self.config.get('delete_onresolve'):
|
||||
alert_id = gotify.find_byfingerprint(alert)
|
||||
if alert_id:
|
||||
response = gotify.delete(alert_id)
|
||||
continue
|
||||
prefix = 'Resolved'
|
||||
else:
|
||||
prefix = alert['labels'].get(
|
||||
'severity', 'warning').capitalize()
|
||||
|
||||
def gotify_send(server, port, authkey, payload):
|
||||
"""
|
||||
Function to POST data to a Gotify server
|
||||
"""
|
||||
gotify_msg = {
|
||||
'title': '{}: {}'.format(
|
||||
prefix,
|
||||
alert['annotations'].get('description', '[nodata]'),
|
||||
),
|
||||
'message': '{}: {}'.format(
|
||||
alert['labels'].get('instance', '[unknown]'),
|
||||
alert['annotations'].get('summary'),
|
||||
),
|
||||
'priority': int(alert['labels'].get('priority', 5)),
|
||||
'extras': {
|
||||
'alertify': {
|
||||
'fingerprint': alert.get('fingerprint', None)
|
||||
}
|
||||
}
|
||||
}
|
||||
except KeyError as error:
|
||||
print(f'ERROR: KeyError: {error}')
|
||||
self._respond(400, f'Missing field: {error}')
|
||||
return
|
||||
|
||||
gotify = http.client.HTTPConnection(server, port)
|
||||
headers = {
|
||||
'X-Gotify-Key': authkey,
|
||||
'Content-type': 'application/json',
|
||||
}
|
||||
response = gotify.send_alert(gotify_msg)
|
||||
|
||||
try:
|
||||
gotify.request('POST', '/message', json.dumps(payload), headers)
|
||||
response = gotify.getresponse()
|
||||
except ConnectionRefusedError as error:
|
||||
print(f'ERROR: {error}')
|
||||
return {
|
||||
'status': error.errno,
|
||||
'reason': error.strerror
|
||||
}
|
||||
try:
|
||||
self._respond(response['status'], response['reason'])
|
||||
except UnboundLocalError:
|
||||
self._respond('204', '')
|
||||
|
||||
return {
|
||||
'status': response.status,
|
||||
'reason': response.reason
|
||||
}
|
||||
def _respond(self, status, message):
|
||||
"""
|
||||
Method to output a simple HTTP status and string to the client
|
||||
"""
|
||||
self.send_response(int(status) or 500)
|
||||
self.end_headers()
|
||||
self.wfile.write(bytes(str(message).encode()))
|
||||
|
||||
# Override built-in method
|
||||
def do_GET(self): # pylint: disable=invalid-name
|
||||
"""
|
||||
Method to handle GET requests
|
||||
"""
|
||||
if self.path == '/healthcheck':
|
||||
if not healthy(self.config):
|
||||
print('ERROR: Check requirements')
|
||||
self._respond(500, 'ERR')
|
||||
|
||||
self._respond(200, 'OK')
|
||||
|
||||
# Override built-in method
|
||||
def do_POST(self): # pylint: disable=invalid-name
|
||||
"""
|
||||
Method to handle POST requests from AlertManager
|
||||
"""
|
||||
if self.path == '/alert':
|
||||
self._alerts()
|
||||
|
||||
# FIXME: This isn't right. A normal method doesn't work, however.
|
||||
@classmethod
|
||||
def set_config(cls, config):
|
||||
"""
|
||||
Classmethod to add config to the class
|
||||
"""
|
||||
cls.config = config
|
||||
|
||||
|
||||
def healthy(config):
|
||||
|
|
@ -196,7 +290,10 @@ def parse_config(configfile):
|
|||
config[key] = type(val)(config[key])
|
||||
|
||||
if config['verbose']:
|
||||
print(f'Config:\n{yaml.dump(config, explicit_start=True, default_flow_style=False)}')
|
||||
print(
|
||||
f'Config:\n'
|
||||
f'{yaml.dump(config, explicit_start=True, default_flow_style=False)}'
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
|
|
|
|||
82
test.sh
82
test.sh
|
|
@ -2,56 +2,86 @@
|
|||
|
||||
SERVER=${1:-'localhost:8080'}
|
||||
|
||||
name=testAlert-$RANDOM
|
||||
NAME=testAlert-$RANDOM
|
||||
FINGERPRINT=$(date +%s | md5sum | cut -f1 -d' ')
|
||||
URL="http://${SERVER}/alert"
|
||||
bold=$(tput bold)
|
||||
normal=$(tput sgr0)
|
||||
BOLD=$(tput bold)
|
||||
NORMAL=$(tput sgr0)
|
||||
|
||||
call_alertmanager() {
|
||||
curl -v "${URL}" --header 'Content-type: application/json' --data @<(cat <<EOF
|
||||
VALUE=${1}
|
||||
curl -v "${URL}" --header 'Expect:' --header 'Content-type: application/json' --data @<(cat <<EOF
|
||||
{
|
||||
"version": "4",
|
||||
"groupKey": "testGroup",
|
||||
"truncatedAlerts": 0,
|
||||
"status": "${STATUS}",
|
||||
"receiver": "alertify",
|
||||
"commonLabels": {
|
||||
"alertname": "${name}",
|
||||
"service": "testService",
|
||||
"severity":"warning",
|
||||
"instance": "server.example.net",
|
||||
"namespace": "testNamespace",
|
||||
"label_costcentre": "testCostCentre"
|
||||
},
|
||||
"commonAnnotations": {
|
||||
"summary": "Testing latency is high!",
|
||||
"description": "Testing latency is at ${1}"
|
||||
},
|
||||
"status": "${STATUS}",
|
||||
"alerts": [
|
||||
{
|
||||
"status": "${STATUS}",
|
||||
"generatorURL": "http://alertmanager.example.net/$name",
|
||||
"labels": {
|
||||
"alertname": "${NAME}",
|
||||
"id": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
|
||||
"instance": "localhost:1234",
|
||||
"job": "test_job",
|
||||
"name": "testserver",
|
||||
"priority": "1",
|
||||
"severity": "low",
|
||||
"value": "${VALUE}"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "testserver: unhealthy",
|
||||
"summary": "Server unhealthy"
|
||||
},
|
||||
"startsAt": "${START}",
|
||||
"endsAt": "${END}"
|
||||
"endsAt": "${END}",
|
||||
"generatorURL": "http://example.com/some/url",
|
||||
"fingerprint": "${FINGERPRINT}"
|
||||
}
|
||||
]
|
||||
],
|
||||
"groupLabels": {
|
||||
"alertname": "${NAME}",
|
||||
"id": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
|
||||
"instance": "localhost:1234",
|
||||
"job": "test_job",
|
||||
"name": "testserver",
|
||||
"priority": "1",
|
||||
"severity": "low",
|
||||
"value": "${VALUE}"
|
||||
},
|
||||
"commonLabels": {
|
||||
"alertname": "${NAME}",
|
||||
"id": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
|
||||
"instance": "localhost:1234",
|
||||
"job": "test_job",
|
||||
"name": "testserver",
|
||||
"priority": "1",
|
||||
"severity": "low",
|
||||
"value": "${VALUE}"
|
||||
},
|
||||
"commonAnnotations": {
|
||||
"description": "testserver: unhealthy",
|
||||
"summary": "Server unhealthy"
|
||||
},
|
||||
"externalURL": "http://1ff297bc31a0:9093",
|
||||
"version": "4",
|
||||
"groupKey": "{}:{alertname=\"${NAME}\", id=\"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b\", instance=\"localhost:1234\", job=\"test_job\", name=\"testserver\", priority=\"1\", severity=\"low\", value=\"${VALUE}\"}",
|
||||
"truncatedAlerts": 0
|
||||
}
|
||||
EOF
|
||||
)
|
||||
}
|
||||
|
||||
echo "${bold}Firing alert ${name} ${normal}"
|
||||
echo "${BOLD}Firing alert ${NAME} ${NORMAL}"
|
||||
STATUS='firing'
|
||||
START=$(date --rfc-3339=seconds | sed 's/ /T/')
|
||||
END="0001-01-01T00:00:00Z"
|
||||
call_alertmanager 42
|
||||
echo -e "\n"
|
||||
|
||||
echo "${bold}Press enter to resolve alert ${name} ${normal}"
|
||||
echo "${BOLD}Press enter to resolve alert ${NAME} ${NORMAL}"
|
||||
read -r
|
||||
|
||||
|
||||
echo "${bold}Sending resolved ${normal}"
|
||||
echo "${BOLD}Sending resolved ${NORMAL}"
|
||||
STATUS='resolved'
|
||||
END=$(date --rfc-3339=seconds | sed 's/ /T/')
|
||||
call_alertmanager 0
|
||||
|
|
|
|||
Loading…
Reference in a new issue