From e576403b64c3c8734d7e46896e73909163493662 Mon Sep 17 00:00:00 2001 From: Tobias Hodapp Date: Tue, 22 Aug 2023 03:13:39 +0200 Subject: [PATCH 1/5] Added support for grafana loki --- borgmatic/config/schema.yaml | 30 +++++++++ borgmatic/hooks/dispatch.py | 2 + borgmatic/hooks/loki.py | 117 +++++++++++++++++++++++++++++++++++ borgmatic/hooks/monitor.py | 2 +- 4 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 borgmatic/hooks/loki.py diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 752c75ab..a2ba64eb 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1403,3 +1403,33 @@ properties: Configuration for a monitoring integration with Crunhub. Create an account at https://cronhub.io if you'd like to use this service. See borgmatic monitoring documentation for details. + loki: + type: object + required: ['url', 'labels'] + additionalProperties: false + properties: + url: + type: string + description: | + Grafana loki log URL to notify when a backup begins, + ends, or fails. + example: "http://localhost:3100/loki/api/v1/push" + labels: + type: object + additionalProperties: + type: string + description: | + Allows setting custom labels for the logging stream. At + least one label is required. "__hostname" gets replaced by + the machine hostname automatically. "__config" gets replaced + by just the name of the configuration file. "__config_path" + gets replaced by the full path of the configuration file. + example: + app: "borgmatic" + config: "__config" + hostname: "__hostname" + description: | + Configuration for a monitoring integration with Grafana loki. You + can send the logs to a self-hosted instance or create an account at + https://grafana.com/auth/sign-up/create-user. See borgmatic + monitoring documentation for details. diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 0c003e33..4f7706fc 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -11,6 +11,7 @@ from borgmatic.hooks import ( pagerduty, postgresql, sqlite, + loki, ) logger = logging.getLogger(__name__) @@ -26,6 +27,7 @@ HOOK_NAME_TO_MODULE = { 'pagerduty': pagerduty, 'postgresql_databases': postgresql, 'sqlite_databases': sqlite, + 'loki': loki, } diff --git a/borgmatic/hooks/loki.py b/borgmatic/hooks/loki.py new file mode 100644 index 00000000..a26f9625 --- /dev/null +++ b/borgmatic/hooks/loki.py @@ -0,0 +1,117 @@ +import logging +import requests +import json +import time +import platform +from borgmatic.hooks import monitor + +logger = logging.getLogger(__name__) + +MONITOR_STATE_TO_HEALTHCHECKS = { + monitor.State.START: 'Started', + monitor.State.FINISH: 'Finished', + monitor.State.FAIL: 'Failed', +} + +# Threshold at which logs get flushed to loki +MAX_BUFFER_LINES = 100 + +class loki_log_buffer(): + ''' + A log buffer that allows to output the logs as loki requests in json + ''' + def __init__(self, url, dry_run): + self.url = url + self.dry_run = dry_run + self.root = {} + self.root["streams"] = [{}] + self.root["streams"][0]["stream"] = {} + self.root["streams"][0]["values"] = [] + + def add_value(self, value): + timestamp = str(time.time_ns()) + self.root["streams"][0]["values"].append((timestamp, value)) + + def add_label(self, label, value): + self.root["streams"][0]["stream"][label] = value + + def _to_request(self): + return json.dumps(self.root) + + def __len__(self): + return len(self.root["streams"][0]["values"]) + + def flush(self): + if self.dry_run: + self.root["streams"][0]["values"] = [] + return + if len(self) == 0: + return + request_body = self._to_request() + self.root["streams"][0]["values"] = [] + request_header = {"Content-Type": "application/json"} + try: + r = requests.post(self.url, headers=request_header, data=request_body, timeout=5) + r.raise_for_status() + except requests.RequestException: + logger.warn("Failed to upload logs to loki") + +class loki_log_handeler(logging.Handler): + ''' + A log handler that sends logs to loki + ''' + def __init__(self, url, dry_run): + super().__init__() + self.buffer = loki_log_buffer(url, dry_run) + + def emit(self, record): + self.raw(record.getMessage()) + + def add_label(self, key, value): + self.buffer.add_label(key, value) + + def raw(self, msg): + self.buffer.add_value(msg) + if len(self.buffer) > MAX_BUFFER_LINES: + self.buffer.flush() + + def flush(self): + if len(self.buffer) > 0: + self.buffer.flush() + +def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): + ''' + Add a handler to the root logger to regularly send the logs to loki + ''' + url = hook_config.get('url') + loki = loki_log_handeler(url, dry_run) + for k, v in hook_config.get('labels').items(): + if v == '__hostname': + loki.add_label(k, platform.node()) + elif v == '__config': + loki.add_label(k, config_filename.split('/')[-1]) + elif v == '__config_path': + loki.add_label(k, config_filename) + else: + loki.add_label(k, v) + logging.getLogger().addHandler(loki) + +def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): + ''' + Adds an entry to the loki logger with the current state + ''' + if not dry_run: + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, loki_log_handeler): + if state in MONITOR_STATE_TO_HEALTHCHECKS.keys(): + handler.raw(f'{config_filename} {MONITOR_STATE_TO_HEALTHCHECKS[state]} backup') + +def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): + ''' + Remove the monitor handler that was added to the root logger. + ''' + logger = logging.getLogger() + for handler in tuple(logger.handlers): + if isinstance(handler, loki_log_handeler): + handler.flush() + logger.removeHandler(handler) diff --git a/borgmatic/hooks/monitor.py b/borgmatic/hooks/monitor.py index c0168178..118639f5 100644 --- a/borgmatic/hooks/monitor.py +++ b/borgmatic/hooks/monitor.py @@ -1,6 +1,6 @@ from enum import Enum -MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy') +MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy', 'loki') class State(Enum): From a3edf757ee3f29ebf2366d45b3e7d6fadb7fc46c Mon Sep 17 00:00:00 2001 From: Tobias Hodapp Date: Tue, 22 Aug 2023 13:40:05 +0200 Subject: [PATCH 2/5] Added changes of formatting tools --- borgmatic/hooks/dispatch.py | 2 +- borgmatic/hooks/loki.py | 57 +++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 4f7706fc..24793b5a 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -4,6 +4,7 @@ from borgmatic.hooks import ( cronhub, cronitor, healthchecks, + loki, mariadb, mongodb, mysql, @@ -11,7 +12,6 @@ from borgmatic.hooks import ( pagerduty, postgresql, sqlite, - loki, ) logger = logging.getLogger(__name__) diff --git a/borgmatic/hooks/loki.py b/borgmatic/hooks/loki.py index a26f9625..53ff2081 100644 --- a/borgmatic/hooks/loki.py +++ b/borgmatic/hooks/loki.py @@ -1,8 +1,10 @@ -import logging -import requests import json -import time +import logging import platform +import time + +import requests + from borgmatic.hooks import monitor logger = logging.getLogger(__name__) @@ -16,50 +18,54 @@ MONITOR_STATE_TO_HEALTHCHECKS = { # Threshold at which logs get flushed to loki MAX_BUFFER_LINES = 100 -class loki_log_buffer(): + +class loki_log_buffer: ''' A log buffer that allows to output the logs as loki requests in json ''' + def __init__(self, url, dry_run): self.url = url self.dry_run = dry_run self.root = {} - self.root["streams"] = [{}] - self.root["streams"][0]["stream"] = {} - self.root["streams"][0]["values"] = [] + self.root['streams'] = [{}] + self.root['streams'][0]['stream'] = {} + self.root['streams'][0]['values'] = [] def add_value(self, value): timestamp = str(time.time_ns()) - self.root["streams"][0]["values"].append((timestamp, value)) + self.root['streams'][0]['values'].append((timestamp, value)) def add_label(self, label, value): - self.root["streams"][0]["stream"][label] = value + self.root['streams'][0]['stream'][label] = value def _to_request(self): return json.dumps(self.root) def __len__(self): - return len(self.root["streams"][0]["values"]) + return len(self.root['streams'][0]['values']) def flush(self): if self.dry_run: - self.root["streams"][0]["values"] = [] + self.root['streams'][0]['values'] = [] return if len(self) == 0: return request_body = self._to_request() - self.root["streams"][0]["values"] = [] - request_header = {"Content-Type": "application/json"} + self.root['streams'][0]['values'] = [] + request_header = {'Content-Type': 'application/json'} try: - r = requests.post(self.url, headers=request_header, data=request_body, timeout=5) - r.raise_for_status() + result = requests.post(self.url, headers=request_header, data=request_body, timeout=5) + result.raise_for_status() except requests.RequestException: - logger.warn("Failed to upload logs to loki") + logger.warn('Failed to upload logs to loki') + class loki_log_handeler(logging.Handler): ''' A log handler that sends logs to loki ''' + def __init__(self, url, dry_run): super().__init__() self.buffer = loki_log_buffer(url, dry_run) @@ -79,23 +85,25 @@ class loki_log_handeler(logging.Handler): if len(self.buffer) > 0: self.buffer.flush() + def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): ''' Add a handler to the root logger to regularly send the logs to loki ''' url = hook_config.get('url') loki = loki_log_handeler(url, dry_run) - for k, v in hook_config.get('labels').items(): - if v == '__hostname': - loki.add_label(k, platform.node()) - elif v == '__config': - loki.add_label(k, config_filename.split('/')[-1]) - elif v == '__config_path': - loki.add_label(k, config_filename) + for key, value in hook_config.get('labels').items(): + if value == '__hostname': + loki.add_label(key, platform.node()) + elif value == '__config': + loki.add_label(key, config_filename.split('/')[-1]) + elif value == '__config_path': + loki.add_label(key, config_filename) else: - loki.add_label(k, v) + loki.add_label(key, value) logging.getLogger().addHandler(loki) + def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' Adds an entry to the loki logger with the current state @@ -106,6 +114,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if state in MONITOR_STATE_TO_HEALTHCHECKS.keys(): handler.raw(f'{config_filename} {MONITOR_STATE_TO_HEALTHCHECKS[state]} backup') + def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): ''' Remove the monitor handler that was added to the root logger. From 7e419ec995a18c6701a8f7f34af786f928e267fa Mon Sep 17 00:00:00 2001 From: Tobias Hodapp Date: Tue, 22 Aug 2023 23:03:14 +0200 Subject: [PATCH 3/5] Fixed spelling errors Added documentation Added log messages for dry run --- borgmatic/hooks/loki.py | 75 +++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/borgmatic/hooks/loki.py b/borgmatic/hooks/loki.py index 53ff2081..ac7047e8 100644 --- a/borgmatic/hooks/loki.py +++ b/borgmatic/hooks/loki.py @@ -1,5 +1,6 @@ import json import logging +import os import platform import time @@ -9,7 +10,7 @@ from borgmatic.hooks import monitor logger = logging.getLogger(__name__) -MONITOR_STATE_TO_HEALTHCHECKS = { +MONITOR_STATE_TO_LOKI = { monitor.State.START: 'Started', monitor.State.FINISH: 'Finished', monitor.State.FAIL: 'Failed', @@ -19,84 +20,107 @@ MONITOR_STATE_TO_HEALTHCHECKS = { MAX_BUFFER_LINES = 100 -class loki_log_buffer: +class Loki_log_buffer: ''' - A log buffer that allows to output the logs as loki requests in json + A log buffer that allows to output the logs as loki requests in json. Allows + adding labels to the log stream and takes care of communication with loki. ''' def __init__(self, url, dry_run): self.url = url self.dry_run = dry_run - self.root = {} - self.root['streams'] = [{}] - self.root['streams'][0]['stream'] = {} - self.root['streams'][0]['values'] = [] + self.root = {'streams': [{'stream': {}, 'values': []}]} def add_value(self, value): + ''' + Add a log entry to the stream. + ''' timestamp = str(time.time_ns()) self.root['streams'][0]['values'].append((timestamp, value)) def add_label(self, label, value): + ''' + Add a label to the logging stream. + ''' self.root['streams'][0]['stream'][label] = value - def _to_request(self): + def to_request(self): return json.dumps(self.root) def __len__(self): + ''' + Gets the number of lines currently in the buffer. + ''' return len(self.root['streams'][0]['values']) def flush(self): if self.dry_run: + # Just empty the buffer and skip self.root['streams'][0]['values'] = [] + logger.info('Skipped uploading logs to loki due to dry run') return + if len(self) == 0: + # Skip as there are not logs to send yet return - request_body = self._to_request() + + request_body = self.to_request() self.root['streams'][0]['values'] = [] request_header = {'Content-Type': 'application/json'} try: result = requests.post(self.url, headers=request_header, data=request_body, timeout=5) result.raise_for_status() except requests.RequestException: - logger.warn('Failed to upload logs to loki') + logger.warning('Failed to upload logs to loki') -class loki_log_handeler(logging.Handler): +class Loki_log_handler(logging.Handler): ''' - A log handler that sends logs to loki + A log handler that sends logs to loki. ''' def __init__(self, url, dry_run): super().__init__() - self.buffer = loki_log_buffer(url, dry_run) + self.buffer = Loki_log_buffer(url, dry_run) def emit(self, record): + ''' + Add a log record from the logging module to the stream. + ''' self.raw(record.getMessage()) def add_label(self, key, value): + ''' + Add a label to the logging stream. + ''' self.buffer.add_label(key, value) def raw(self, msg): + ''' + Add an arbitrary string as a log entry to the stream. + ''' self.buffer.add_value(msg) if len(self.buffer) > MAX_BUFFER_LINES: self.buffer.flush() def flush(self): - if len(self.buffer) > 0: - self.buffer.flush() + ''' + Send the logs to loki and empty the buffer. + ''' + self.buffer.flush() def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): ''' - Add a handler to the root logger to regularly send the logs to loki + Add a handler to the root logger to regularly send the logs to loki. ''' url = hook_config.get('url') - loki = loki_log_handeler(url, dry_run) + loki = Loki_log_handler(url, dry_run) for key, value in hook_config.get('labels').items(): if value == '__hostname': loki.add_label(key, platform.node()) elif value == '__config': - loki.add_label(key, config_filename.split('/')[-1]) + loki.add_label(key, os.path.basename(config_filename)) elif value == '__config_path': loki.add_label(key, config_filename) else: @@ -106,13 +130,14 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' - Adds an entry to the loki logger with the current state + Add an entry to the loki logger with the current state. ''' - if not dry_run: - for handler in tuple(logging.getLogger().handlers): - if isinstance(handler, loki_log_handeler): - if state in MONITOR_STATE_TO_HEALTHCHECKS.keys(): - handler.raw(f'{config_filename} {MONITOR_STATE_TO_HEALTHCHECKS[state]} backup') + if dry_run: + return + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, Loki_log_handler): + if state in MONITOR_STATE_TO_LOKI.keys(): + handler.raw(f'{config_filename}: {MONITOR_STATE_TO_LOKI[state]} backup') def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): @@ -121,6 +146,6 @@ def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, ''' logger = logging.getLogger() for handler in tuple(logger.handlers): - if isinstance(handler, loki_log_handeler): + if isinstance(handler, Loki_log_handler): handler.flush() logger.removeHandler(handler) From 9e2674ea5ac0659c1bfbdb1998f0741a85a15f07 Mon Sep 17 00:00:00 2001 From: Tobias Hodapp Date: Wed, 23 Aug 2023 17:17:23 +0200 Subject: [PATCH 4/5] Added unit tests Removed useless dry run check --- borgmatic/hooks/loki.py | 2 - tests/unit/hooks/test_loki.py | 124 ++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 tests/unit/hooks/test_loki.py diff --git a/borgmatic/hooks/loki.py b/borgmatic/hooks/loki.py index ac7047e8..9a5dae8f 100644 --- a/borgmatic/hooks/loki.py +++ b/borgmatic/hooks/loki.py @@ -132,8 +132,6 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ''' Add an entry to the loki logger with the current state. ''' - if dry_run: - return for handler in tuple(logging.getLogger().handlers): if isinstance(handler, Loki_log_handler): if state in MONITOR_STATE_TO_LOKI.keys(): diff --git a/tests/unit/hooks/test_loki.py b/tests/unit/hooks/test_loki.py new file mode 100644 index 00000000..0de2ced2 --- /dev/null +++ b/tests/unit/hooks/test_loki.py @@ -0,0 +1,124 @@ +from flexmock import flexmock +from borgmatic.hooks import loki +import json +import platform +import logging +import requests + + +def test_log_handler_gets_added(): + hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} + config_filename = 'test.yaml' + dry_run = True + loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, loki.Loki_log_handler): + assert True + return + assert False + + +def test_ping(): + hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} + config_filename = 'test.yaml' + dry_run = True + loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) + loki.ping_monitor(hook_config, '', config_filename, loki.monitor.State.FINISH, '', dry_run) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, loki.Loki_log_handler): + assert len(handler.buffer) <= 1 + return + assert False + + +def test_log_handler_gets_removed(): + hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} + config_filename = 'test.yaml' + dry_run = True + loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) + loki.destroy_monitor(hook_config, '', config_filename, '', dry_run) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, loki.Loki_log_handler): + assert False + + +def test_log_handler_gets_labels(): + buffer = loki.Loki_log_buffer('', False) + buffer.add_label('test', 'label') + assert buffer.root['streams'][0]['stream']['test'] == 'label' + buffer.add_label('test2', 'label2') + assert buffer.root['streams'][0]['stream']['test2'] == 'label2' + + +def test_log_handler_label_replacment(): + hook_config = { + 'url': 'http://localhost:3100/loki/api/v1/push', + 'labels': {'hostname': '__hostname', 'config': '__config', 'config_full': '__config_path'}, + } + config_filename = '/mock/path/test.yaml' + dry_run = True + loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, loki.Loki_log_handler): + assert handler.buffer.root['streams'][0]['stream']['hostname'] == platform.node() + assert handler.buffer.root['streams'][0]['stream']['config'] == 'test.yaml' + assert handler.buffer.root['streams'][0]['stream']['config_full'] == config_filename + return + assert False + + +def test_log_handler_gets_logs(): + buffer = loki.Loki_log_buffer('', False) + assert len(buffer) == 0 + buffer.add_value('Some test log line') + assert len(buffer) == 1 + buffer.add_value('Another test log line') + assert len(buffer) == 2 + + +def test_log_handler_gets_raw(): + handler = loki.Loki_log_handler('', False) + handler.emit(flexmock(getMessage=lambda: 'Some test log line')) + assert len(handler.buffer) == 1 + + +def test_log_handler_json(): + buffer = loki.Loki_log_buffer('', False) + assert json.loads(buffer.to_request()) == json.loads('{"streams":[{"stream":{},"values":[]}]}') + + +def test_log_handler_json_labels(): + buffer = loki.Loki_log_buffer('', False) + buffer.add_label('test', 'label') + assert json.loads(buffer.to_request()) == json.loads( + '{"streams":[{"stream":{"test": "label"},"values":[]}]}' + ) + + +def test_log_handler_json_log_lines(): + buffer = loki.Loki_log_buffer('', False) + buffer.add_value('Some test log line') + assert json.loads(buffer.to_request())['streams'][0]['values'][0][1] == 'Some test log line' + + +def test_log_handler_post(): + handler = loki.Loki_log_handler('', False) + flexmock(loki.requests).should_receive('post').and_return( + flexmock(raise_for_status=lambda: '') + ).once() + for x in range(150): + handler.raw(x) + + +def test_post_failiure(): + handler = loki.Loki_log_handler('', False) + flexmock(loki.requests).should_receive('post').and_return( + flexmock(raise_for_status=lambda: (_ for _ in ()).throw(requests.RequestException())) + ).once() + for x in range(150): + handler.raw(x) + + +def test_empty_flush(): + handler = loki.Loki_log_handler('', False) + handler.flush() From 099a712e53bf0931c69d74976840d1839bc74d3a Mon Sep 17 00:00:00 2001 From: Tobias Hodapp Date: Thu, 24 Aug 2023 13:17:42 +0200 Subject: [PATCH 5/5] Added more documentation to the test Split tests to integration tests --- tests/integration/hooks/test_loki.py | 82 +++++++++++++++++ tests/unit/hooks/test_loki.py | 130 +++++++++++---------------- 2 files changed, 134 insertions(+), 78 deletions(-) create mode 100644 tests/integration/hooks/test_loki.py diff --git a/tests/integration/hooks/test_loki.py b/tests/integration/hooks/test_loki.py new file mode 100644 index 00000000..3eac29d3 --- /dev/null +++ b/tests/integration/hooks/test_loki.py @@ -0,0 +1,82 @@ +import logging +import platform + +from flexmock import flexmock + +from borgmatic.hooks import loki as module + + +def test_log_handler_label_replacment(): + ''' + Assert that label placeholders get replaced + ''' + hook_config = { + 'url': 'http://localhost:3100/loki/api/v1/push', + 'labels': {'hostname': '__hostname', 'config': '__config', 'config_full': '__config_path'}, + } + config_filename = '/mock/path/test.yaml' + dry_run = True + module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, module.Loki_log_handler): + assert handler.buffer.root['streams'][0]['stream']['hostname'] == platform.node() + assert handler.buffer.root['streams'][0]['stream']['config'] == 'test.yaml' + assert handler.buffer.root['streams'][0]['stream']['config_full'] == config_filename + return + assert False + + +def test_initalize_adds_log_handler(): + ''' + Assert that calling initialize_monitor adds our logger to the root logger + ''' + hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} + module.initialize_monitor( + hook_config, + flexmock(), + config_filename='test.yaml', + monitoring_log_level=flexmock(), + dry_run=True, + ) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, module.Loki_log_handler): + return + assert False + + +def test_ping_adds_log_message(): + ''' + Assert that calling ping_monitor adds a message to our logger + ''' + hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} + config_filename = 'test.yaml' + dry_run = True + module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run) + module.ping_monitor( + hook_config, flexmock(), config_filename, module.monitor.State.FINISH, flexmock(), dry_run + ) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, module.Loki_log_handler): + assert any( + map( + lambda log: log + == f'{config_filename}: {module.MONITOR_STATE_TO_LOKI[module.monitor.State.FINISH]} backup', + map(lambda x: x[1], handler.buffer.root['streams'][0]['values']), + ) + ) + return + assert False + + +def test_log_handler_gets_removed(): + ''' + Assert that destroy_monitor removes the logger from the root logger + ''' + hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} + config_filename = 'test.yaml' + dry_run = True + module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run) + module.destroy_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run) + for handler in tuple(logging.getLogger().handlers): + if isinstance(handler, module.Loki_log_handler): + assert False diff --git a/tests/unit/hooks/test_loki.py b/tests/unit/hooks/test_loki.py index 0de2ced2..33e35ecd 100644 --- a/tests/unit/hooks/test_loki.py +++ b/tests/unit/hooks/test_loki.py @@ -1,74 +1,27 @@ -from flexmock import flexmock -from borgmatic.hooks import loki import json -import platform -import logging + import requests +from flexmock import flexmock - -def test_log_handler_gets_added(): - hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} - config_filename = 'test.yaml' - dry_run = True - loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) - for handler in tuple(logging.getLogger().handlers): - if isinstance(handler, loki.Loki_log_handler): - assert True - return - assert False - - -def test_ping(): - hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} - config_filename = 'test.yaml' - dry_run = True - loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) - loki.ping_monitor(hook_config, '', config_filename, loki.monitor.State.FINISH, '', dry_run) - for handler in tuple(logging.getLogger().handlers): - if isinstance(handler, loki.Loki_log_handler): - assert len(handler.buffer) <= 1 - return - assert False - - -def test_log_handler_gets_removed(): - hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}} - config_filename = 'test.yaml' - dry_run = True - loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) - loki.destroy_monitor(hook_config, '', config_filename, '', dry_run) - for handler in tuple(logging.getLogger().handlers): - if isinstance(handler, loki.Loki_log_handler): - assert False +from borgmatic.hooks import loki as module def test_log_handler_gets_labels(): - buffer = loki.Loki_log_buffer('', False) + ''' + Assert that adding labels works + ''' + buffer = module.Loki_log_buffer(flexmock(), False) buffer.add_label('test', 'label') assert buffer.root['streams'][0]['stream']['test'] == 'label' buffer.add_label('test2', 'label2') assert buffer.root['streams'][0]['stream']['test2'] == 'label2' -def test_log_handler_label_replacment(): - hook_config = { - 'url': 'http://localhost:3100/loki/api/v1/push', - 'labels': {'hostname': '__hostname', 'config': '__config', 'config_full': '__config_path'}, - } - config_filename = '/mock/path/test.yaml' - dry_run = True - loki.initialize_monitor(hook_config, '', config_filename, '', dry_run) - for handler in tuple(logging.getLogger().handlers): - if isinstance(handler, loki.Loki_log_handler): - assert handler.buffer.root['streams'][0]['stream']['hostname'] == platform.node() - assert handler.buffer.root['streams'][0]['stream']['config'] == 'test.yaml' - assert handler.buffer.root['streams'][0]['stream']['config_full'] == config_filename - return - assert False - - -def test_log_handler_gets_logs(): - buffer = loki.Loki_log_buffer('', False) +def test_log_buffer_gets_raw(): + ''' + Assert that adding values to the log buffer increases it's length + ''' + buffer = module.Loki_log_buffer(flexmock(), False) assert len(buffer) == 0 buffer.add_value('Some test log line') assert len(buffer) == 1 @@ -76,49 +29,70 @@ def test_log_handler_gets_logs(): assert len(buffer) == 2 -def test_log_handler_gets_raw(): - handler = loki.Loki_log_handler('', False) +def test_log_buffer_gets_log_messages(): + ''' + Assert that adding log records works + ''' + handler = module.Loki_log_handler(flexmock(), False) handler.emit(flexmock(getMessage=lambda: 'Some test log line')) assert len(handler.buffer) == 1 -def test_log_handler_json(): - buffer = loki.Loki_log_buffer('', False) +def test_log_buffer_json(): + ''' + Assert that the buffer correctly serializes when empty + ''' + buffer = module.Loki_log_buffer(flexmock(), False) assert json.loads(buffer.to_request()) == json.loads('{"streams":[{"stream":{},"values":[]}]}') -def test_log_handler_json_labels(): - buffer = loki.Loki_log_buffer('', False) +def test_log_buffer_json_labels(): + ''' + Assert that the buffer correctly serializes with labels + ''' + buffer = module.Loki_log_buffer(flexmock(), False) buffer.add_label('test', 'label') assert json.loads(buffer.to_request()) == json.loads( '{"streams":[{"stream":{"test": "label"},"values":[]}]}' ) -def test_log_handler_json_log_lines(): - buffer = loki.Loki_log_buffer('', False) +def test_log_buffer_json_log_lines(): + ''' + Assert that log lines end up in the correct place in the log buffer + ''' + buffer = module.Loki_log_buffer(flexmock(), False) buffer.add_value('Some test log line') assert json.loads(buffer.to_request())['streams'][0]['values'][0][1] == 'Some test log line' def test_log_handler_post(): - handler = loki.Loki_log_handler('', False) - flexmock(loki.requests).should_receive('post').and_return( + ''' + Assert that the flush function sends a post request after a certain limit + ''' + handler = module.Loki_log_handler(flexmock(), False) + flexmock(module.requests).should_receive('post').and_return( flexmock(raise_for_status=lambda: '') ).once() - for x in range(150): - handler.raw(x) + for num in range(int(module.MAX_BUFFER_LINES * 1.5)): + handler.raw(num) -def test_post_failiure(): - handler = loki.Loki_log_handler('', False) - flexmock(loki.requests).should_receive('post').and_return( +def test_log_handler_post_failiure(): + ''' + Assert that the flush function catches request exceptions + ''' + handler = module.Loki_log_handler(flexmock(), False) + flexmock(module.requests).should_receive('post').and_return( flexmock(raise_for_status=lambda: (_ for _ in ()).throw(requests.RequestException())) ).once() - for x in range(150): - handler.raw(x) + for num in range(int(module.MAX_BUFFER_LINES * 1.5)): + handler.raw(num) -def test_empty_flush(): - handler = loki.Loki_log_handler('', False) +def test_log_handler_empty_flush_noop(): + ''' + Test that flushing an empty buffer does indeed nothing + ''' + handler = module.Loki_log_handler(flexmock(), False) handler.flush()