borgmatic-collective/borgmatic
borgmatic-collective/borgmatic
14
58
Fork 42
Code Issues 62 Pull Requests 2 Actions Packages Releases 122 Activity

Added support for grafana loki #747

Merged
witten merged 5 commits from :main into main 2023-08-25 16:28:20 +00:00
Conversation 4 Commits 5 Files Changed 6 +34 -25
2 changed files with 34 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a3edf757ee - Show all commits

2
borgmatic/hooks/dispatch.py
View File

@ -4,6 +4,7 @@ from borgmatic.hooks import (
cronhub,
cronitor,
healthchecks,
loki,
mariadb,
mongodb,
mysql,
@ -11,7 +12,6 @@ from borgmatic.hooks import (
pagerduty,
postgresql,
sqlite,
loki,
)
logger = logging.getLogger(__name__)

57
borgmatic/hooks/loki.py
View File

@ -1,8 +1,10 @@
import logging
import requests
import json
import time
import logging
import platform
import time
import requests
from borgmatic.hooks import monitor
logger = logging.getLogger(__name__)
@ -16,50 +18,54 @@ MONITOR_STATE_TO_HEALTHCHECKS = {
# Threshold at which logs get flushed to loki
MAX_BUFFER_LINES = 100
class loki_log_buffer():
class loki_log_buffer:
witten marked this conversation as resolved
witten commented 2023-08-22 17:45:36 +00:00
Review
Copy Link

Code style convention: Uppercase first letter of class names, e.g. Loki_log_buffer.

Code style convention: Uppercase first letter of class names, e.g. `Loki_log_buffer`.
'''
A log buffer that allows to output the logs as loki requests in json
'''
def __init__(self, url, dry_run):
self.url = url
self.dry_run = dry_run
self.root = {}
self.root["streams"] = [{}]
self.root["streams"][0]["stream"] = {}
self.root["streams"][0]["values"] = []
self.root['streams'] = [{}]
self.root['streams'][0]['stream'] = {}
self.root['streams'][0]['values'] = []
def add_value(self, value):
timestamp = str(time.time_ns())
self.root["streams"][0]["values"].append((timestamp, value))
self.root['streams'][0]['values'].append((timestamp, value))
def add_label(self, label, value):
self.root["streams"][0]["stream"][label] = value
self.root['streams'][0]['stream'][label] = value
def _to_request(self):
return json.dumps(self.root)
def __len__(self):
return len(self.root["streams"][0]["values"])
return len(self.root['streams'][0]['values'])
def flush(self):
if self.dry_run:
witten marked this conversation as resolved
witten commented 2023-08-22 18:01:23 +00:00
Review
Copy Link

Blank lines after some of these ifs would be nice. Maybe before the try as well. In general I find that helps readability even if it's not required by the language.

Blank lines after some of these `if`s would be nice. Maybe before the `try` as well. In general I find that helps readability even if it's not required by the language.
self.root["streams"][0]["values"] = []
self.root['streams'][0]['values'] = []
return
if len(self) == 0:
return
request_body = self._to_request()
self.root["streams"][0]["values"] = []
request_header = {"Content-Type": "application/json"}
self.root['streams'][0]['values'] = []
request_header = {'Content-Type': 'application/json'}
try:
r = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
r.raise_for_status()
result = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
result.raise_for_status()
witten marked this conversation as resolved
witten commented 2023-08-22 18:28:49 +00:00
Review
Copy Link

This code looks good, but it's a little counter-intuitive to me that it would be here. For instance, I think of a buffer as a data structure for storing stuff, not necessarily as a data structure that also implicitly has the side effect of pushing logs to an external service. Maybe I'm just not used to OOP. 😄 I don't feel super strongly or anything, but this might be less surprising if the push to Loki took place elsewhere like in ping_monitor().

I'm guessing part of the reason you're doing it this way though is so that logs get sent to Loki as borgmatic runs rather than all at the end once ping_monitor() is called..? The Healthchecks hook for example only sends logs at the end, but the rationale there is that it's explicitly logging the success/failure status of the backup rather than only logs along the way. So the requirements may be a little different.

Anyway, let me know your thoughts. (And then maybe put some of them into docstrings. 😄)

This code looks good, but it's a little counter-intuitive to me that it would be here. For instance, I think of a buffer as a data structure for storing stuff, not necessarily as a data structure that also implicitly has the side effect of pushing logs to an external service. Maybe I'm just not used to OOP. 😄 I don't feel super strongly or anything, but this might be less surprising if the push to Loki took place elsewhere like in `ping_monitor()`. I'm guessing part of the reason you're doing it this way though is so that logs get sent to Loki as borgmatic runs rather than all at the end once `ping_monitor()` is called..? The Healthchecks hook for example only sends logs at the end, but the rationale there is that it's explicitly logging the success/failure status of the backup rather than only logs along the way. So the requirements may be a little different. Anyway, let me know your thoughts. (And then maybe put some of them into docstrings. 😄)
IBims1NicerTobi commented 2023-08-22 20:41:25 +00:00
Review
Copy Link

Well I am trying to not hit the max request size limits of a lot of loki instances that run behind e.g. nginx. I think it is much better for large volumes of logs to be pushed incrementally instead of pushing it as one who knows how big request in the end.

Well I am trying to not hit the max request size limits of a lot of loki instances that run behind e.g. nginx. I think it is much better for large volumes of logs to be pushed incrementally instead of pushing it as one who knows how big request in the end.
witten commented 2023-08-23 05:08:55 +00:00
Review
Copy Link

Gotcha. The Healthchecks hook "solves" that particular problem by reverse truncating the logs so that older messages are not sent if the logs get too big by the time ping_monitor() is called. However I can see why you might not want to do that in this case, since loki seems much more about log aggregation than simply tracking service status.

Gotcha. The Healthchecks hook "solves" that particular problem by reverse truncating the logs so that older messages are not sent if the logs get too big by the time `ping_monitor()` is called. However I can see why you might not want to do that in this case, since loki seems much more about log aggregation than simply tracking service status.
except requests.RequestException:
logger.warn("Failed to upload logs to loki")
logger.warn('Failed to upload logs to loki')
class loki_log_handeler(logging.Handler):
'''
A log handler that sends logs to loki
'''
def __init__(self, url, dry_run):
super().__init__()
self.buffer = loki_log_buffer(url, dry_run)
@ -79,23 +85,25 @@ class loki_log_handeler(logging.Handler):
if len(self.buffer) > 0:
self.buffer.flush()
def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
'''
Add a handler to the root logger to regularly send the logs to loki
IBims1NicerTobi marked this conversation as resolved
witten commented 2023-08-22 18:30:56 +00:00
Review
Copy Link

Just a code style convention nit: Period at the end of sentences in docstrings.

Just a code style convention nit: Period at the end of sentences in docstrings.
'''
url = hook_config.get('url')
loki = loki_log_handeler(url, dry_run)
for k, v in hook_config.get('labels').items():
if v == '__hostname':
loki.add_label(k, platform.node())
elif v == '__config':
loki.add_label(k, config_filename.split('/')[-1])
elif v == '__config_path':
loki.add_label(k, config_filename)
for key, value in hook_config.get('labels').items():
if value == '__hostname':
loki.add_label(key, platform.node())
elif value == '__config':
loki.add_label(key, config_filename.split('/')[-1])
elif value == '__config_path':
loki.add_label(key, config_filename)
else:
loki.add_label(k, v)
loki.add_label(key, value)
logging.getLogger().addHandler(loki)
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
'''
Adds an entry to the loki logger with the current state
@ -106,6 +114,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
if state in MONITOR_STATE_TO_HEALTHCHECKS.keys():
handler.raw(f'{config_filename} {MONITOR_STATE_TO_HEALTHCHECKS[state]} backup')
def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
'''
Remove the monitor handler that was added to the root logger.