From 1fdec480d6e2d77749abd760dbc26a8c9096f710 Mon Sep 17 00:00:00 2001 From: "Jelle @ Samson-IT" <71821148+Jelle-SamsonIT@users.noreply.github.com> Date: Wed, 13 Jul 2022 13:29:45 +0200 Subject: [PATCH 01/39] Added some info about fetching mysql database size --- docs/how-to/inspect-your-backups.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 318eb6ee..e4574d7d 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -75,6 +75,16 @@ example, to search only the last five archives: borgmatic list --find foo.txt --last 5 ``` +## Monitoring mysql backup size + +If you have enabled borgmatic's native mysql hook you can fetch the size of your individual sql backups from the backup target itself even when using an append-only access key like you can use on borgbase.com. + +For example: +```bash +borgmatic list --archive latest --no-color | grep root/.borgmatic/mysql_databases/localhost/ +``` + +Note that the `localhost` part of the path in this case is fully dependent on how your config looks. If you connect to an external database from this host, change to the address in config accordingly. ## Logging From 3720f222345fa6fbbf7f432c49f71d1bdae851e0 Mon Sep 17 00:00:00 2001 From: "Jelle @ Samson-IT" <71821148+Jelle-SamsonIT@users.noreply.github.com> Date: Wed, 13 Jul 2022 22:03:51 +0200 Subject: [PATCH 02/39] reworded and added 'all' caveat --- docs/how-to/inspect-your-backups.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index e4574d7d..1fa70317 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -77,14 +77,17 @@ borgmatic list --find foo.txt --last 5 ## Monitoring mysql backup size -If you have enabled borgmatic's native mysql hook you can fetch the size of your individual sql backups from the backup target itself even when using an append-only access key like you can use on borgbase.com. +If you have enabled borgmatic's native mysql hook you can query the size of your sql backups from the host you're backing up itself. This works even when using an append-only access key like you can use on borgbase.com. For example: ```bash borgmatic list --archive latest --no-color | grep root/.borgmatic/mysql_databases/localhost/ ``` -Note that the `localhost` part of the path in this case is fully dependent on how your config looks. If you connect to an external database from this host, change to the address in config accordingly. +Note that the `localhost` part of the path in the regex is dependent on how your config looks. If you connect to an external database your config, change to the regexp accordingly because the path will be different. + +An additional caveat is that when you specify "all" for your database config, there will be one file named "all.sql" in the localhost folder. +Specify your database names in config individually to have one file per database. ## Logging From 69f6695253170fa4e09738c6ca0b31b0a95b7aae Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Sun, 5 Mar 2023 19:27:32 +0530 Subject: [PATCH 03/39] Add support for healthchecks "log" feature #628 Signed-off-by: Soumik Dutta --- borgmatic/commands/borgmatic.py | 18 ++++++++++++++++++ borgmatic/config/schema.yaml | 1 + borgmatic/hooks/cronhub.py | 1 + borgmatic/hooks/cronitor.py | 1 + borgmatic/hooks/healthchecks.py | 3 ++- borgmatic/hooks/monitor.py | 1 + borgmatic/hooks/ntfy.py | 1 + 7 files changed, 25 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index e08df976..b5e86648 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -152,6 +152,24 @@ def run_configuration(config_filename, config, arguments): encountered_error = error error_repository = repository_path + try: + # send logs irrespective of error + dispatch.call_hooks( + 'ping_monitor', + hooks, + config_filename, + monitor.MONITOR_HOOK_NAMES, + monitor.State.LOG, + monitoring_log_level, + global_arguments.dry_run, + ) + except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(config_filename, error): + return + + encountered_error = error + yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error) + if not encountered_error: try: if using_primary_action: diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 2f873b7a..0b2a378f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1199,6 +1199,7 @@ properties: - start - finish - fail + - log uniqueItems: true description: | List of one or more monitoring states to ping for: diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index b93788e4..b3a3521c 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -10,6 +10,7 @@ MONITOR_STATE_TO_CRONHUB = { monitor.State.START: 'start', monitor.State.FINISH: 'finish', monitor.State.FAIL: 'fail', + monitor.State.LOG: 'log', } diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 8866a6ac..c01d5f6d 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -10,6 +10,7 @@ MONITOR_STATE_TO_CRONITOR = { monitor.State.START: 'run', monitor.State.FINISH: 'complete', monitor.State.FAIL: 'fail', + monitor.State.LOG: 'log', } diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 03d012a8..6ad8449f 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -10,6 +10,7 @@ MONITOR_STATE_TO_HEALTHCHECKS = { monitor.State.START: 'start', monitor.State.FINISH: None, # Healthchecks doesn't append to the URL for the finished state. monitor.State.FAIL: 'fail', + monitor.State.LOG: 'log', } PAYLOAD_TRUNCATION_INDICATOR = '...\n' @@ -117,7 +118,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ ) logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url)) - if state in (monitor.State.FINISH, monitor.State.FAIL): + if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG): payload = format_buffered_logs_for_payload() else: payload = '' diff --git a/borgmatic/hooks/monitor.py b/borgmatic/hooks/monitor.py index 846fca15..c0168178 100644 --- a/borgmatic/hooks/monitor.py +++ b/borgmatic/hooks/monitor.py @@ -7,3 +7,4 @@ class State(Enum): START = 1 FINISH = 2 FAIL = 3 + LOG = 4 diff --git a/borgmatic/hooks/ntfy.py b/borgmatic/hooks/ntfy.py index 99ed254a..fda912ca 100644 --- a/borgmatic/hooks/ntfy.py +++ b/borgmatic/hooks/ntfy.py @@ -10,6 +10,7 @@ MONITOR_STATE_TO_NTFY = { monitor.State.START: None, monitor.State.FINISH: None, monitor.State.FAIL: None, + monitor.State.LOG: None, } From 1573d68fe243d030006271f7c17a6515e2fec407 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Sun, 5 Mar 2023 21:57:13 +0530 Subject: [PATCH 04/39] update schema.yaml description also add monitor.State.LOG to cronitor. Signed-off-by: Soumik Dutta --- borgmatic/config/schema.yaml | 8 ++++---- borgmatic/hooks/cronitor.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0b2a378f..92423909 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -951,9 +951,9 @@ properties: name: type: string description: | - This is used to tag the database dump file - with a name. It is not the path to the database - file itself. The name "all" has no special + This is used to tag the database dump file + with a name. It is not the path to the database + file itself. The name "all" has no special meaning for SQLite databases. example: users path: @@ -1168,7 +1168,7 @@ properties: type: string description: | Healthchecks ping URL or UUID to notify when a - backup begins, ends, or errors. + backup begins, ends, errors or just to send logs. example: https://hc-ping.com/your-uuid-here verify_tls: type: boolean diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index c01d5f6d..f3ab13bf 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -10,7 +10,7 @@ MONITOR_STATE_TO_CRONITOR = { monitor.State.START: 'run', monitor.State.FINISH: 'complete', monitor.State.FAIL: 'fail', - monitor.State.LOG: 'log', + monitor.State.LOG: 'ok', } From 45256ae33f39c40481a402b5b8dbf0ba7b34ec6f Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 6 Mar 2023 03:38:08 +0530 Subject: [PATCH 05/39] add test for healthchecks Signed-off-by: Soumik Dutta --- borgmatic/commands/borgmatic.py | 21 +++++++++++---------- borgmatic/hooks/cronhub.py | 1 - borgmatic/hooks/cronitor.py | 1 - borgmatic/hooks/ntfy.py | 7 ------- tests/unit/hooks/test_healthchecks.py | 17 +++++++++++++++++ 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b5e86648..6bc24561 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -153,16 +153,17 @@ def run_configuration(config_filename, config, arguments): error_repository = repository_path try: - # send logs irrespective of error - dispatch.call_hooks( - 'ping_monitor', - hooks, - config_filename, - monitor.MONITOR_HOOK_NAMES, - monitor.State.LOG, - monitoring_log_level, - global_arguments.dry_run, - ) + if using_primary_action: + # send logs irrespective of error + dispatch.call_hooks( + 'ping_monitor', + hooks, + config_filename, + monitor.MONITOR_HOOK_NAMES, + monitor.State.LOG, + monitoring_log_level, + global_arguments.dry_run, + ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): return diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index b3a3521c..b93788e4 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -10,7 +10,6 @@ MONITOR_STATE_TO_CRONHUB = { monitor.State.START: 'start', monitor.State.FINISH: 'finish', monitor.State.FAIL: 'fail', - monitor.State.LOG: 'log', } diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index f3ab13bf..8866a6ac 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -10,7 +10,6 @@ MONITOR_STATE_TO_CRONITOR = { monitor.State.START: 'run', monitor.State.FINISH: 'complete', monitor.State.FAIL: 'fail', - monitor.State.LOG: 'ok', } diff --git a/borgmatic/hooks/ntfy.py b/borgmatic/hooks/ntfy.py index fda912ca..3f897aad 100644 --- a/borgmatic/hooks/ntfy.py +++ b/borgmatic/hooks/ntfy.py @@ -6,13 +6,6 @@ from borgmatic.hooks import monitor logger = logging.getLogger(__name__) -MONITOR_STATE_TO_NTFY = { - monitor.State.START: None, - monitor.State.FINISH: None, - monitor.State.FAIL: None, - monitor.State.LOG: None, -} - def initialize_monitor( ping_url, config_filename, monitoring_log_level, dry_run diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index ee78e52b..d5779534 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -184,6 +184,23 @@ def test_ping_monitor_hits_ping_url_for_fail_state(): ) +def test_ping_monitor_hits_ping_url_for_log_state(): + hook_config = {'ping_url': 'https://example.com'} + payload = 'data' + flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) + flexmock(module.requests).should_receive('post').with_args( + 'https://example.com/log', data=payload.encode('utf'), verify=True + ).and_return(flexmock(ok=True)) + + module.ping_monitor( + hook_config, + 'config.yaml', + state=module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, + ) + + def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): hook_config = {'ping_url': 'abcd-efgh-ijkl-mnop'} payload = 'data' From e211863cba14bdd3f8f63cdc0712843694d47a4e Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 6 Mar 2023 05:12:24 +0530 Subject: [PATCH 06/39] update test_borgmatic.py Signed-off-by: Soumik Dutta --- tests/unit/commands/test_borgmatic.py | 45 ++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 38b28fd9..e2da3b86 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -41,8 +41,10 @@ def test_run_configuration_logs_monitor_start_error(): flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return( None ).and_return(None) - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) + expected_results = [flexmock(), flexmock()] + flexmock(module).should_receive('log_error_records').and_return( + [expected_results[0]] + ).and_return([expected_results[1]]) flexmock(module).should_receive('run_actions').never() config = {'location': {'repositories': ['foo']}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -99,7 +101,7 @@ def test_run_configuration_bails_for_actions_soft_failure(): assert results == [] -def test_run_configuration_logs_monitor_finish_error(): +def test_run_configuration_logs_monitor_log_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( @@ -116,13 +118,48 @@ def test_run_configuration_logs_monitor_finish_error(): assert results == expected_results +def test_run_configuration_bails_for_monitor_log_soft_failure(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') + flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( + None + ).and_raise(error) + flexmock(module).should_receive('log_error_records').never() + flexmock(module).should_receive('run_actions').and_return([]) + flexmock(module.command).should_receive('considered_soft_failure').and_return(True) + config = {'location': {'repositories': ['foo']}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + + results = list(module.run_configuration('test.yaml', config, arguments)) + + assert results == [] + + +def test_run_configuration_logs_monitor_finish_error(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( + None + ).and_return(None).and_raise(OSError) + expected_results = [flexmock()] + flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('run_actions').and_return([]) + config = {'location': {'repositories': ['foo']}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + + results = list(module.run_configuration('test.yaml', config, arguments)) + + assert results == expected_results + + def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( None - ).and_raise(error) + ).and_raise(None).and_raise(error) flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) From f442aeae9c930aa2d2d4c482861823462f01c877 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 6 Mar 2023 05:21:56 +0530 Subject: [PATCH 07/39] fix logs_monitor_start_error() Signed-off-by: Soumik Dutta --- tests/unit/commands/test_borgmatic.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index e2da3b86..fc3f34dd 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -40,11 +40,9 @@ def test_run_configuration_logs_monitor_start_error(): flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return( None - ).and_return(None) - expected_results = [flexmock(), flexmock()] - flexmock(module).should_receive('log_error_records').and_return( - [expected_results[0]] - ).and_return([expected_results[1]]) + ).and_return(None).and_return(None) + expected_results = [flexmock()] + flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').never() config = {'location': {'repositories': ['foo']}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} From 4fcfddbe0804431be8419d5c7db7635741665c3b Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 6 Mar 2023 19:58:57 +0530 Subject: [PATCH 08/39] return early if unsupported state is passed Signed-off-by: Soumik Dutta --- borgmatic/hooks/cronhub.py | 8 ++++++++ borgmatic/hooks/cronitor.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index b93788e4..8ec5aacc 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -27,6 +27,14 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. ''' + if state not in MONITOR_STATE_TO_CRONHUB: + logger.debug( + '{}: Ignoring unsupported monitoring {} in Cronhub hook'.format( + config_filename, state.name.lower() + ) + ) + return + dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state]) ping_url = ( diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 8866a6ac..f5164e42 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -27,6 +27,14 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. ''' + if state not in MONITOR_STATE_TO_CRONITOR: + logger.debug( + '{}: Ignoring unsupported monitoring {} in Cronitor hook'.format( + config_filename, state.name.lower() + ) + ) + return + dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state]) From 98e429594e33cadf39579b3d271be0b2efb0ccb9 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 6 Mar 2023 20:31:00 +0530 Subject: [PATCH 09/39] added tests to make sure unsupported log states are detected Signed-off-by: Soumik Dutta --- tests/unit/hooks/test_cronhub.py | 10 ++++++++++ tests/unit/hooks/test_cronitor.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index 14e8eb23..9007d2c5 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -102,3 +102,13 @@ def test_ping_monitor_with_other_error_logs_warning(): monitoring_log_level=1, dry_run=False, ) + + +def test_ping_monitor_with_unsupported_monitoring_state(): + hook_config = {'ping_url': 'https://example.com'} + flexmock(module.logger).should_receive("debug").once().with_args( + '{}: Ignoring unsupported monitoring {} in Cronhub hook'.format("config.yaml", "log") + ) + module.ping_monitor( + hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 4b762d85..51fba989 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -87,3 +87,13 @@ def test_ping_monitor_with_other_error_logs_warning(): monitoring_log_level=1, dry_run=False, ) + + +def test_ping_monitor_with_unsupported_monitoring_state(): + hook_config = {'ping_url': 'https://example.com'} + flexmock(module.logger).should_receive("debug").once().with_args( + '{}: Ignoring unsupported monitoring {} in Cronitor hook'.format("config.yaml", "log") + ) + module.ping_monitor( + hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + ) From 66194b73045f8a25b93f5e1fbee31be2f7440d4a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 6 Mar 2023 22:41:43 -0800 Subject: [PATCH 10/39] Update dates in documentation examples. --- docs/how-to/extract-a-backup.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index 62bdc75b..9a3b4626 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -20,15 +20,15 @@ borgmatic rlist That should yield output looking something like: ```text -host-2019-01-01T04:05:06.070809 Tue, 2019-01-01 04:05:06 [...] -host-2019-01-02T04:06:07.080910 Wed, 2019-01-02 04:06:07 [...] +host-2023-01-01T04:05:06.070809 Tue, 2023-01-01 04:05:06 [...] +host-2023-01-02T04:06:07.080910 Wed, 2023-01-02 04:06:07 [...] ``` Assuming that you want to extract the archive with the most up-to-date files and therefore the latest timestamp, run a command like: ```bash -borgmatic extract --archive host-2019-01-02T04:06:07.080910 +borgmatic extract --archive host-2023-01-02T04:06:07.080910 ``` (No borgmatic `extract` action? Upgrade borgmatic!) @@ -54,7 +54,7 @@ But if you have multiple repositories configured, then you'll need to specify the repository path containing the archive to extract. Here's an example: ```bash -borgmatic extract --repository repo.borg --archive host-2019-... +borgmatic extract --repository repo.borg --archive host-2023-... ``` ## Extract particular files From 62ae82f2c0c8e8f17dc6bf8ca801d5454b4bf253 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 6 Mar 2023 22:59:34 -0800 Subject: [PATCH 11/39] Mention searching for files in the extract a backup guide. --- docs/how-to/extract-a-backup.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index 9a3b4626..4285c784 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -74,6 +74,13 @@ run the `extract` command above, borgmatic will extract `/var/path/1` and `/var/path/2`. +### Searching for files + +If you're not sure which archive contains the files you're looking for, you +can [search across +archives](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#searching-for-a-file). + + ## Extract to a particular destination By default, borgmatic extracts files into the current directory. To instead From 044ae7869af4457e017e117abe57ebe998edab5e Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Wed, 8 Mar 2023 03:30:12 +0530 Subject: [PATCH 12/39] fix tests Signed-off-by: Soumik Dutta --- borgmatic/hooks/cronhub.py | 4 +--- borgmatic/hooks/cronitor.py | 4 +--- tests/unit/hooks/test_cronhub.py | 4 +--- tests/unit/hooks/test_cronitor.py | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index 8ec5aacc..cd0ffa5c 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -29,9 +29,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ ''' if state not in MONITOR_STATE_TO_CRONHUB: logger.debug( - '{}: Ignoring unsupported monitoring {} in Cronhub hook'.format( - config_filename, state.name.lower() - ) + f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook' ) return diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index f5164e42..633b4c3c 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -29,9 +29,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ ''' if state not in MONITOR_STATE_TO_CRONITOR: logger.debug( - '{}: Ignoring unsupported monitoring {} in Cronitor hook'.format( - config_filename, state.name.lower() - ) + f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook' ) return diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index 9007d2c5..f470b88e 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -106,9 +106,7 @@ def test_ping_monitor_with_other_error_logs_warning(): def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} - flexmock(module.logger).should_receive("debug").once().with_args( - '{}: Ignoring unsupported monitoring {} in Cronhub hook'.format("config.yaml", "log") - ) + flexmock(module.requests).should_receive('get').never() module.ping_monitor( hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 51fba989..7ec1e2e6 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -91,9 +91,7 @@ def test_ping_monitor_with_other_error_logs_warning(): def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} - flexmock(module.logger).should_receive("debug").once().with_args( - '{}: Ignoring unsupported monitoring {} in Cronitor hook'.format("config.yaml", "log") - ) + flexmock(module.requests).should_receive('get').never() module.ping_monitor( hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, ) From 5d19d86e4a823bfb37f07485979f4979babe4c44 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 7 Mar 2023 14:08:35 -0800 Subject: [PATCH 13/39] Add flake8-quotes to complain about incorrect quoting so I don't have to! --- .flake8 | 1 + borgmatic/borg/check.py | 2 +- borgmatic/borg/rlist.py | 2 +- borgmatic/commands/arguments.py | 4 ++-- borgmatic/config/validate.py | 2 +- setup.cfg | 2 ++ test_requirements.txt | 1 + tests/unit/borg/test_create.py | 2 +- tests/unit/hooks/test_mongodb.py | 2 +- 9 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..5d931745 --- /dev/null +++ b/.flake8 @@ -0,0 +1 @@ +select = Q0 diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 484a4154..d9beaa60 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -139,7 +139,7 @@ def filter_checks_on_frequency( if datetime.datetime.now() < check_time + frequency_delta: remaining = check_time + frequency_delta - datetime.datetime.now() logger.info( - f"Skipping {check} check due to configured frequency; {remaining} until next check" + f'Skipping {check} check due to configured frequency; {remaining} until next check' ) filtered_checks.remove(check) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 9f0de473..2a465fb0 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -17,7 +17,7 @@ def resolve_archive_name( Raise ValueError if "latest" is given but there are no archives in the repository. ''' - if archive != "latest": + if archive != 'latest': return archive lock_wait = storage_config.get('lock_wait', None) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8af9bb56..3acf0d27 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -611,7 +611,7 @@ def make_parsers(): metavar='NAME', nargs='+', dest='databases', - help='Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic\'s configuration', + help="Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic's configuration", ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' @@ -805,7 +805,7 @@ def make_parsers(): 'borg', aliases=SUBPARSER_ALIASES['borg'], help='Run an arbitrary Borg command', - description='Run an arbitrary Borg command based on borgmatic\'s configuration', + description="Run an arbitrary Borg command based on borgmatic's configuration", add_help=False, ) borg_group = borg_parser.add_argument_group('borg arguments') diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 3a2807f9..044e22d8 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -186,5 +186,5 @@ def guard_single_repository_selected(repository, configurations): if count != 1: raise ValueError( - 'Can\'t determine which repository to use. Use --repository to disambiguate' + "Can't determine which repository to use. Use --repository to disambiguate" ) diff --git a/setup.cfg b/setup.cfg index fe6236b6..d8d28f89 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,6 +10,8 @@ filterwarnings = [flake8] ignore = E501,W503 exclude = *.*/* +multiline-quotes = ''' +docstring-quotes = ''' [tool:isort] force_single_line = False diff --git a/test_requirements.txt b/test_requirements.txt index e47c6fd6..9cae8fb4 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -5,6 +5,7 @@ click==7.1.2; python_version >= '3.8' colorama==0.4.4 coverage==5.3 flake8==4.0.1 +flake8-quotes==3.3.2 flexmock==0.10.4 isort==5.9.1 mccabe==0.6.1 diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3c4e4338..10b5a951 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -1916,7 +1916,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log (f'repo::{DEFAULT_ARCHIVE_NAME}',) ) flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('collect_special_file_paths').and_return(("/dev/null",)) + flexmock(module).should_receive('collect_special_file_paths').and_return(('/dev/null',)) create_command = ( 'borg', 'create', diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 9544a09f..f61f3c70 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -72,7 +72,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password(): 'name': 'foo', 'username': 'mongo', 'password': 'trustsome1', - 'authentication_database': "admin", + 'authentication_database': 'admin', } ] process = flexmock() From d88bcc8be9e05402210eb952427febfd7de14699 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 7 Mar 2023 15:45:23 -0800 Subject: [PATCH 14/39] Add Healthchecks "log" state feature to NEWS. --- NEWS | 2 + borgmatic/config/schema.yaml | 5 ++- borgmatic/hooks/ntfy.py | 2 - tests/unit/hooks/test_ntfy.py | 69 +++++++++++++++++++++++++---------- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/NEWS b/NEWS index 39cb6c25..c4223875 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ 1.7.9.dev0 * #295: Add a SQLite database dump/restore hook. + * #628: Add Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling + success or failure. 1.7.8 * #620: With the "create" action and the "--list" ("--files") flag, only show excluded files at diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 92423909..480c1dc6 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1180,7 +1180,8 @@ properties: type: boolean description: | Send borgmatic logs to Healthchecks as part the - "finish" state. Defaults to true. + "finish", "fail", and "log" states. Defaults to + true. example: false ping_body_limit: type: integer @@ -1203,7 +1204,7 @@ properties: uniqueItems: true description: | List of one or more monitoring states to ping for: - "start", "finish", and/or "fail". Defaults to + "start", "finish", "fail", and/or "log". Defaults to pinging for all states. example: - finish diff --git a/borgmatic/hooks/ntfy.py b/borgmatic/hooks/ntfy.py index 3f897aad..8a6f0fb8 100644 --- a/borgmatic/hooks/ntfy.py +++ b/borgmatic/hooks/ntfy.py @@ -2,8 +2,6 @@ import logging import requests -from borgmatic.hooks import monitor - logger = logging.getLogger(__name__) diff --git a/tests/unit/hooks/test_ntfy.py b/tests/unit/hooks/test_ntfy.py index ea3f3c1d..9731df7a 100644 --- a/tests/unit/hooks/test_ntfy.py +++ b/tests/unit/hooks/test_ntfy.py @@ -2,6 +2,7 @@ from enum import Enum from flexmock import flexmock +import borgmatic.hooks.monitor from borgmatic.hooks import ntfy as module default_base_url = 'https://ntfy.sh' @@ -37,12 +38,16 @@ def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic} flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.FAIL), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.FAIL), auth=None, ).and_return(flexmock(ok=True)).once() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + borgmatic.hooks.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -54,12 +59,16 @@ def test_ping_monitor_with_auth_hits_hosted_ntfy_on_fail(): } flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.FAIL), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.FAIL), auth=module.requests.auth.HTTPBasicAuth('testuser', 'fakepassword'), ).and_return(flexmock(ok=True)).once() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + borgmatic.hooks.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -67,13 +76,17 @@ def test_ping_monitor_auth_with_no_username_warning(): hook_config = {'topic': topic, 'password': 'fakepassword'} flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.FAIL), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.FAIL), auth=None, ).and_return(flexmock(ok=True)).once() flexmock(module.logger).should_receive('warning').once() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + borgmatic.hooks.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -81,13 +94,17 @@ def test_ping_monitor_auth_with_no_password_warning(): hook_config = {'topic': topic, 'username': 'testuser'} flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.FAIL), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.FAIL), auth=None, ).and_return(flexmock(ok=True)).once() flexmock(module.logger).should_receive('warning').once() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + borgmatic.hooks.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -98,7 +115,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): module.ping_monitor( hook_config, 'config.yaml', - module.monitor.State.START, + borgmatic.hooks.monitor.State.START, monitoring_log_level=1, dry_run=False, ) @@ -111,7 +128,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): module.ping_monitor( hook_config, 'config.yaml', - module.monitor.State.FINISH, + borgmatic.hooks.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, ) @@ -121,12 +138,16 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): hook_config = {'topic': topic, 'server': custom_base_url} flexmock(module.requests).should_receive('post').with_args( f'{custom_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.FAIL), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.FAIL), auth=None, ).and_return(flexmock(ok=True)).once() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + borgmatic.hooks.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -135,7 +156,11 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): flexmock(module.requests).should_receive('post').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True + hook_config, + 'config.yaml', + borgmatic.hooks.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=True, ) @@ -146,7 +171,11 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): ).and_return(flexmock(ok=True)).once() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + borgmatic.hooks.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -154,14 +183,14 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): hook_config = {'topic': topic, 'states': ['start', 'fail']} flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.START), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.START), auth=None, ).and_return(flexmock(ok=True)).once() module.ping_monitor( hook_config, 'config.yaml', - module.monitor.State.START, + borgmatic.hooks.monitor.State.START, monitoring_log_level=1, dry_run=False, ) @@ -171,7 +200,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): hook_config = {'topic': topic} flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.FAIL), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.FAIL), auth=None, ).and_raise(module.requests.exceptions.ConnectionError) flexmock(module.logger).should_receive('warning').once() @@ -179,7 +208,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.ping_monitor( hook_config, 'config.yaml', - module.monitor.State.FAIL, + borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, ) @@ -193,7 +222,7 @@ def test_ping_monitor_with_other_error_logs_warning(): ) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', - headers=return_default_message_headers(module.monitor.State.FAIL), + headers=return_default_message_headers(borgmatic.hooks.monitor.State.FAIL), auth=None, ).and_return(response) flexmock(module.logger).should_receive('warning').once() @@ -201,7 +230,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.ping_monitor( hook_config, 'config.yaml', - module.monitor.State.FAIL, + borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, ) From 9db31bd1e9c0d001dda959574ddd1e6783b0f5ef Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 8 Mar 2023 13:19:41 -0800 Subject: [PATCH 15/39] Run any command-line actions in the order specified instead of using a fixed ordering (#304). --- NEWS | 3 +- borgmatic/commands/arguments.py | 9 +- borgmatic/commands/borgmatic.py | 305 ++++++++++---------- docs/how-to/deal-with-very-large-backups.md | 14 +- tests/unit/commands/test_arguments.py | 22 ++ tests/unit/commands/test_borgmatic.py | 27 ++ 6 files changed, 222 insertions(+), 158 deletions(-) diff --git a/NEWS b/NEWS index c4223875..aa62d5e5 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.7.9.dev0 * #295: Add a SQLite database dump/restore hook. - * #628: Add Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling + * #304: Run any command-line actions in the order specified instead of using a fixed ordering. + * #628: Add a Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling success or failure. 1.7.8 diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3acf0d27..9b882711 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -46,11 +46,12 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if 'borg' in unparsed_arguments: subparsers = {'borg': subparsers['borg']} - for subparser_name, subparser in subparsers.items(): - if subparser_name not in remaining_arguments: - continue + for argument in remaining_arguments: + canonical_name = alias_to_subparser_name.get(argument, argument) + subparser = subparsers.get(canonical_name) - canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name) + if not subparser: + continue # If a parsed value happens to be the same as the name of a subparser, remove it from the # remaining arguments. This prevents, for instance, "check --only extract" from triggering diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 6bc24561..4cf60119 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -281,155 +281,162 @@ def run_actions( **hook_context, ) - if 'rcreate' in arguments: - borgmatic.actions.rcreate.run_rcreate( - repository, - storage, - local_borg_version, - arguments['rcreate'], - global_arguments, - local_path, - remote_path, - ) - if 'transfer' in arguments: - borgmatic.actions.transfer.run_transfer( - repository, - storage, - local_borg_version, - arguments['transfer'], - global_arguments, - local_path, - remote_path, - ) - if 'prune' in arguments: - borgmatic.actions.prune.run_prune( - config_filename, - repository, - storage, - retention, - hooks, - hook_context, - local_borg_version, - arguments['prune'], - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - if 'compact' in arguments: - borgmatic.actions.compact.run_compact( - config_filename, - repository, - storage, - retention, - hooks, - hook_context, - local_borg_version, - arguments['compact'], - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - if 'create' in arguments: - yield from borgmatic.actions.create.run_create( - config_filename, - repository, - location, - storage, - hooks, - hook_context, - local_borg_version, - arguments['create'], - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency): - borgmatic.actions.check.run_check( - config_filename, - repository, - location, - storage, - consistency, - hooks, - hook_context, - local_borg_version, - arguments['check'], - global_arguments, - local_path, - remote_path, - ) - if 'extract' in arguments: - borgmatic.actions.extract.run_extract( - config_filename, - repository, - location, - storage, - hooks, - hook_context, - local_borg_version, - arguments['extract'], - global_arguments, - local_path, - remote_path, - ) - if 'export-tar' in arguments: - borgmatic.actions.export_tar.run_export_tar( - repository, - storage, - local_borg_version, - arguments['export-tar'], - global_arguments, - local_path, - remote_path, - ) - if 'mount' in arguments: - borgmatic.actions.mount.run_mount( - repository, storage, local_borg_version, arguments['mount'], local_path, remote_path, - ) - if 'restore' in arguments: - borgmatic.actions.restore.run_restore( - repository, - location, - storage, - hooks, - local_borg_version, - arguments['restore'], - global_arguments, - local_path, - remote_path, - ) - if 'rlist' in arguments: - yield from borgmatic.actions.rlist.run_rlist( - repository, storage, local_borg_version, arguments['rlist'], local_path, remote_path, - ) - if 'list' in arguments: - yield from borgmatic.actions.list.run_list( - repository, storage, local_borg_version, arguments['list'], local_path, remote_path, - ) - if 'rinfo' in arguments: - yield from borgmatic.actions.rinfo.run_rinfo( - repository, storage, local_borg_version, arguments['rinfo'], local_path, remote_path, - ) - if 'info' in arguments: - yield from borgmatic.actions.info.run_info( - repository, storage, local_borg_version, arguments['info'], local_path, remote_path, - ) - if 'break-lock' in arguments: - borgmatic.actions.break_lock.run_break_lock( - repository, - storage, - local_borg_version, - arguments['break-lock'], - local_path, - remote_path, - ) - if 'borg' in arguments: - borgmatic.actions.borg.run_borg( - repository, storage, local_borg_version, arguments['borg'], local_path, remote_path, - ) + for (action_name, action_arguments) in arguments.items(): + if action_name == 'rcreate': + borgmatic.actions.rcreate.run_rcreate( + repository, + storage, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'transfer': + borgmatic.actions.transfer.run_transfer( + repository, + storage, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'prune': + borgmatic.actions.prune.run_prune( + config_filename, + repository, + storage, + retention, + hooks, + hook_context, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'compact': + borgmatic.actions.compact.run_compact( + config_filename, + repository, + storage, + retention, + hooks, + hook_context, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'create': + yield from borgmatic.actions.create.run_create( + config_filename, + repository, + location, + storage, + hooks, + hook_context, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'check': + if checks.repository_enabled_for_checks(repository, consistency): + borgmatic.actions.check.run_check( + config_filename, + repository, + location, + storage, + consistency, + hooks, + hook_context, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'extract': + borgmatic.actions.extract.run_extract( + config_filename, + repository, + location, + storage, + hooks, + hook_context, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'export-tar': + borgmatic.actions.export_tar.run_export_tar( + repository, + storage, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'mount': + borgmatic.actions.mount.run_mount( + repository, + storage, + local_borg_version, + arguments['mount'], + local_path, + remote_path, + ) + elif action_name == 'restore': + borgmatic.actions.restore.run_restore( + repository, + location, + storage, + hooks, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'rlist': + yield from borgmatic.actions.rlist.run_rlist( + repository, storage, local_borg_version, action_arguments, local_path, remote_path, + ) + elif action_name == 'list': + yield from borgmatic.actions.list.run_list( + repository, storage, local_borg_version, action_arguments, local_path, remote_path, + ) + elif action_name == 'rinfo': + yield from borgmatic.actions.rinfo.run_rinfo( + repository, storage, local_borg_version, action_arguments, local_path, remote_path, + ) + elif action_name == 'info': + yield from borgmatic.actions.info.run_info( + repository, storage, local_borg_version, action_arguments, local_path, remote_path, + ) + elif action_name == 'break-lock': + borgmatic.actions.break_lock.run_break_lock( + repository, + storage, + local_borg_version, + arguments['break-lock'], + local_path, + remote_path, + ) + elif action_name == 'borg': + borgmatic.actions.borg.run_borg( + repository, storage, local_borg_version, action_arguments, local_path, remote_path, + ) command.execute_hook( hooks.get('after_actions'), diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index 720d4334..343f6923 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -36,10 +36,16 @@ skipping certain actions while running others. For instance, this skips borgmatic create check ``` -Or, you can make backups with `create` on a frequent schedule (e.g. with -`borgmatic create` called from one cron job), while only running expensive -consistency checks with `check` on a much less frequent basis (e.g. with -`borgmatic check` called from a separate cron job). +New in version 1.7.9 borgmatic +now respects your specified command-line action order, running actions in the +order you specify. In previous versions, borgmatic ran your specified actions +in a fixed ordering regardless of the order they appeared on the command-line. + +But instead of running actions together, another option is to run backups with +`create` on a frequent schedule (e.g. with `borgmatic create` called from one +cron job), while only running expensive consistency checks with `check` on a +much less frequent basis (e.g. with `borgmatic check` called from a separate +cron job). ### Consistency check configuration diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 640d6ade..9354cf5e 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -1,3 +1,5 @@ +import collections + from flexmock import flexmock from borgmatic.commands import arguments as module @@ -70,6 +72,26 @@ def test_parse_subparser_arguments_consumes_multiple_subparser_arguments(): assert remaining_arguments == [] +def test_parse_subparser_arguments_respects_command_line_action_ordering(): + other_namespace = flexmock() + action_namespace = flexmock(foo=True) + subparsers = { + 'action': flexmock( + parse_known_args=lambda arguments: (action_namespace, ['action', '--foo', 'true']) + ), + 'other': flexmock(parse_known_args=lambda arguments: (other_namespace, ['other'])), + } + + arguments, remaining_arguments = module.parse_subparser_arguments( + ('other', '--foo', 'true', 'action'), subparsers + ) + + assert arguments == collections.OrderedDict( + [('other', other_namespace), ('action', action_namespace)] + ) + assert remaining_arguments == [] + + def test_parse_subparser_arguments_applies_default_subparsers(): prune_namespace = flexmock() compact_namespace = flexmock() diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index fc3f34dd..a6899d97 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -778,6 +778,33 @@ def test_run_actions_runs_borg(): ) +def test_run_actions_runs_multiple_actions_in_argument_order(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.borg).should_receive('run_borg').once().ordered() + flexmock(borgmatic.actions.restore).should_receive('run_restore').once().ordered() + + tuple( + module.run_actions( + arguments={ + 'global': flexmock(dry_run=False), + 'borg': flexmock(), + 'restore': flexmock(), + }, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository_path='repo', + ) + ) + + def test_load_configurations_collects_parsed_configurations_and_logs(): configuration = flexmock() other_configuration = flexmock() From b343363bb8cbe37d75e37e086ba4e11fb8d13461 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 8 Mar 2023 14:05:06 -0800 Subject: [PATCH 16/39] Change the default action order to: "create", "prune", "compact", "check" (#304). --- NEWS | 3 ++ borgmatic/commands/arguments.py | 8 ++-- borgmatic/commands/borgmatic.py | 36 ++++++++-------- borgmatic/config/schema.yaml | 16 +++---- docs/how-to/deal-with-very-large-backups.md | 28 ++++++------ docs/how-to/monitor-your-backups.md | 12 +++--- tests/unit/commands/test_borgmatic.py | 48 ++++++++++----------- 7 files changed, 79 insertions(+), 72 deletions(-) diff --git a/NEWS b/NEWS index aa62d5e5..4e05e072 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,8 @@ 1.7.9.dev0 * #295: Add a SQLite database dump/restore hook. + * #304: Change the default action order when no actions are specified on the command-line to: + "create", "prune", "compact", "check". If you'd like to retain the old ordering ("prune" and + "compact" first), then specify actions explicitly on the command-line. * #304: Run any command-line actions in the order specified instead of using a fixed ordering. * #628: Add a Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling success or failure. diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 9b882711..cc9431db 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -68,9 +68,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): arguments[canonical_name] = parsed - # If no actions are explicitly requested, assume defaults: prune, compact, create, and check. + # If no actions are explicitly requested, assume defaults. if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments: - for subparser_name in ('prune', 'compact', 'create', 'check'): + for subparser_name in ('create', 'prune', 'compact', 'check'): subparser = subparsers[subparser_name] parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments) arguments[subparser_name] = parsed @@ -216,7 +216,7 @@ def make_parsers(): top_level_parser = ArgumentParser( description=''' Simple, configuration-driven backup software for servers and workstations. If none of - the action options are given, then borgmatic defaults to: prune, compact, create, and + the action options are given, then borgmatic defaults to: create, prune, compact, and check. ''', parents=[global_parser], @@ -225,7 +225,7 @@ def make_parsers(): subparsers = top_level_parser.add_subparsers( title='actions', metavar='', - help='Specify zero or more actions. Defaults to prune, compact, create, and check. Use --help with action for details:', + help='Specify zero or more actions. Defaults to creat, prune, compact, and check. Use --help with action for details:', ) rcreate_parser = subparsers.add_parser( 'rcreate', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 4cf60119..fe07981e 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -44,8 +44,8 @@ LEGACY_CONFIG_PATH = '/etc/borgmatic/config' def run_configuration(config_filename, config, arguments): ''' Given a config filename, the corresponding parsed config dict, and command-line arguments as a - dict from subparser name to a namespace of parsed arguments, execute the defined prune, compact, - create, check, and/or other actions. + dict from subparser name to a namespace of parsed arguments, execute the defined create, prune, + compact, check, and/or other actions. Yield a combination of: @@ -64,7 +64,7 @@ def run_configuration(config_filename, config, arguments): retry_wait = storage.get('retry_wait', 0) encountered_error = None error_repository = '' - using_primary_action = {'prune', 'compact', 'create', 'check'}.intersection(arguments) + using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) try: @@ -302,6 +302,21 @@ def run_actions( local_path, remote_path, ) + elif action_name == 'create': + yield from borgmatic.actions.create.run_create( + config_filename, + repository, + location, + storage, + hooks, + hook_context, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) elif action_name == 'prune': borgmatic.actions.prune.run_prune( config_filename, @@ -332,21 +347,6 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'create': - yield from borgmatic.actions.create.run_create( - config_filename, - repository, - location, - storage, - hooks, - hook_context, - local_borg_version, - action_arguments, - global_arguments, - dry_run_label, - local_path, - remote_path, - ) elif action_name == 'check': if checks.repository_enabled_for_checks(repository, consistency): borgmatic.actions.check.run_check( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 480c1dc6..f7161563 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -369,6 +369,11 @@ properties: description: | Extra command-line options to pass to "borg init". example: "--extra-option" + create: + type: string + description: | + Extra command-line options to pass to "borg create". + example: "--extra-option" prune: type: string description: | @@ -379,11 +384,6 @@ properties: description: | Extra command-line options to pass to "borg compact". example: "--extra-option" - create: - type: string - description: | - Extra command-line options to pass to "borg create". - example: "--extra-option" check: type: string description: | @@ -663,11 +663,11 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - when an exception occurs during a "prune", "compact", - "create", or "check" action or an associated before/after + when an exception occurs during a "create", "prune", + "compact", or "check" action or an associated before/after hook. example: - - echo "Error during prune/compact/create/check." + - echo "Error during create/prune/compact/check." before_everything: type: array items: diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index 343f6923..c02f718c 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -9,28 +9,32 @@ eleventyNavigation: Borg itself is great for efficiently de-duplicating data across successive backup archives, even when dealing with very large repositories. But you may -find that while borgmatic's default mode of `prune`, `compact`, `create`, and -`check` works well on small repositories, it's not so great on larger ones. -That's because running the default pruning, compact, and consistency checks -take a long time on large repositories. +find that while borgmatic's default actions of `create`, `prune`, `compact`, +and `check` works well on small repositories, it's not so great on larger +ones. That's because running the default pruning, compact, and consistency +checks take a long time on large repositories. + +Prior to version 1.7.9 The +default action ordering was `prune`, `compact`, `create`, and `check`. ### A la carte actions -If you find yourself in this situation, you have some options. First, you can -run borgmatic's `prune`, `compact`, `create`, or `check` actions separately. -For instance, the following optional actions are available: +If you find yourself wanting to customize the actions, you have some options. +First, you can run borgmatic's `prune`, `compact`, `create`, or `check` +actions separately. For instance, the following optional actions are +available (among others): ```bash +borgmatic create borgmatic prune borgmatic compact -borgmatic create borgmatic check ``` -You can run with only one of these actions provided, or you can mix and match -any number of them in a single borgmatic run. This supports approaches like -skipping certain actions while running others. For instance, this skips -`prune` and `compact` and only runs `create` and `check`: +You can run borgmatic with only one of these actions provided, or you can mix +and match any number of them in a single borgmatic run. This supports +approaches like skipping certain actions while running others. For instance, +this skips `prune` and `compact` and only runs `create` and `check`: ```bash borgmatic create check diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 575f4af4..eb7a6200 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -83,7 +83,7 @@ tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). ## Error hooks -When an error occurs during a `prune`, `compact`, `create`, or `check` action, +When an error occurs during a `create`, `prune`, `compact`, or `check` action, borgmatic can run configurable shell commands to fire off custom error notifications or take other actions, so you can get alerted as soon as something goes wrong. Here's a not-so-useful example: @@ -116,8 +116,8 @@ the repository. Here's the full set of supported variables you can use here: * `output`: output of the command that failed (may be blank if an error occurred without running a command) -Note that borgmatic runs the `on_error` hooks only for `prune`, `compact`, -`create`, or `check` actions or hooks in which an error occurs, and not other +Note that borgmatic runs the `on_error` hooks only for `create`, `prune`, +`compact`, or `check` actions or hooks in which an error occurs, and not other actions. borgmatic does not run `on_error` hooks if an error occurs within a `before_everything` or `after_everything` hook. For more about hooks, see the [borgmatic hooks @@ -144,7 +144,7 @@ With this hook in place, borgmatic pings your Healthchecks project when a backup begins, ends, or errors. Specifically, after the `before_backup` hooks run, borgmatic lets Healthchecks know that it has started if any of -the `prune`, `compact`, `create`, or `check` actions are run. +the `create`, `prune`, `compact`, or `check` actions are run. Then, if the actions complete successfully, borgmatic notifies Healthchecks of the success after the `after_backup` hooks run, and includes borgmatic logs in @@ -154,8 +154,8 @@ in the Healthchecks UI, although be aware that Healthchecks currently has a If an error occurs during any action or hook, borgmatic notifies Healthchecks after the `on_error` hooks run, also tacking on logs including the error -itself. But the logs are only included for errors that occur when a `prune`, -`compact`, `create`, or `check` action is run. +itself. But the logs are only included for errors that occur when a `create`, +`prune`, `compact`, or `check` action is run. You can customize the verbosity of the logs that are sent to Healthchecks with borgmatic's `--monitoring-verbosity` flag. The `--list` and `--stats` flags diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index a6899d97..19ac00de 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -436,6 +436,30 @@ def test_run_actions_runs_transfer(): ) +def test_run_actions_runs_create(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + expected = flexmock() + flexmock(borgmatic.actions.create).should_receive('run_create').and_yield(expected).once() + + result = tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False), 'create': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository_path='repo', + ) + ) + assert result == (expected,) + + def test_run_actions_runs_prune(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') @@ -480,30 +504,6 @@ def test_run_actions_runs_compact(): ) -def test_run_actions_runs_create(): - flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.command).should_receive('execute_hook') - expected = flexmock() - flexmock(borgmatic.actions.create).should_receive('run_create').and_yield(expected).once() - - result = tuple( - module.run_actions( - arguments={'global': flexmock(dry_run=False), 'create': flexmock()}, - config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, - local_path=flexmock(), - remote_path=flexmock(), - local_borg_version=flexmock(), - repository_path='repo', - ) - ) - assert result == (expected,) - - def test_run_actions_runs_check_when_repository_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') From d3086788ebe58744515d2515a90ba7a049aefe04 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 8 Mar 2023 16:09:41 -0800 Subject: [PATCH 17/39] Document how to list database dumps in an archive. --- docs/Dockerfile | 2 +- docs/how-to/backup-your-databases.md | 5 ++++- docs/how-to/inspect-your-backups.md | 14 +++++++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index ef29da82..8800cc1f 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in rcreate transfer prune compact create check extract export-tar mount umount restore rlist list rinfo info break-lock borg; do \ + && for action in rcreate transfer create prune compact check extract export-tar mount umount restore rlist list rinfo info break-lock borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic "$action" --help >> /command-line.txt; done diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 602f6488..bc21b659 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -316,7 +316,10 @@ user and you're extracting to `/tmp`, then the dump will be in `/tmp/root/.borgmatic`. After extraction, you can manually restore the dump file using native database -commands like `pg_restore`, `mysql`, `mongorestore` or similar. +commands like `pg_restore`, `mysql`, `mongorestore`, `sqlite`, or similar. + +Also see the documentation on [listing database +dumps](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#listing-database-dumps). ## Preparation and cleanup hooks diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 77e9bea2..57a2381c 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -91,19 +91,19 @@ example, to search only the last five archives: borgmatic list --find foo.txt --last 5 ``` -## Monitoring mysql backup size +## Listing database dumps -If you have enabled borgmatic's native mysql hook you can query the size of your sql backups from the host you're backing up itself. This works even when using an append-only access key like you can use on borgbase.com. +If you have enabled borgmatic's [database +hooks](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/), you +can list backed up database dumps via borgmatic. For example: -For example: ```bash -borgmatic list --archive latest --no-color | grep root/.borgmatic/mysql_databases/localhost/ +borgmatic list --archive latest --find .borgmatic/*_databases ``` -Note that the `localhost` part of the path in the regex is dependent on how your config looks. If you connect to an external database your config, change to the regexp accordingly because the path will be different. +This gives you a listing of all database dump files contained in the latest +archive, complete with file sizes. -An additional caveat is that when you specify "all" for your database config, there will be one file named "all.sql" in the localhost folder. -Specify your database names in config individually to have one file per database. ## Logging From 8cec7c74d8ff4604a3066dc3808f6851371487c2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 9 Mar 2023 10:09:16 -0800 Subject: [PATCH 18/39] Add "--strip-components all" on the "extract" action to remove leading path components (#647). --- NEWS | 2 ++ borgmatic/borg/extract.py | 7 +++++ borgmatic/commands/arguments.py | 5 ++-- tests/unit/borg/test_extract.py | 51 +++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 4e05e072..154e6f3c 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,8 @@ * #304: Run any command-line actions in the order specified instead of using a fixed ordering. * #628: Add a Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling success or failure. + * #647: Add "--strip-components all" feature on the "extract" action to remove leading path + components of files you extract. Must be used with the "--path" flag. 1.7.8 * #620: With the "create" action and the "--list" ("--files") flag, only show excluded files at diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 8ea8bbbd..bbf36edf 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -87,6 +87,13 @@ def extract_archive( else: numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else () + if strip_components == 'all': + if not paths: + raise ValueError('The --strip-components flag with "all" requires at least one --path') + + # Calculate the maximum number of leading path components of the given paths. + strip_components = max(0, *(len(path.split(os.path.sep)) - 1 for path in paths)) + full_command = ( (local_path, 'extract') + (('--remote-path', remote_path) if remote_path else ()) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index cc9431db..3f56b7cd 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -476,10 +476,9 @@ def make_parsers(): ) extract_group.add_argument( '--strip-components', - type=int, + type=lambda number: number if number == 'all' else int(number), metavar='NUMBER', - dest='strip_components', - help='Number of leading path components to remove from each extracted path. Skip paths with fewer elements', + help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements', ) extract_group.add_argument( '--progress', diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index 64d16d35..d27026e4 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -312,6 +312,57 @@ def test_extract_archive_calls_borg_with_strip_components(): ) +def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ( + 'borg', + 'extract', + '--strip-components', + '2', + 'repo::archive', + 'foo/bar/baz.txt', + 'foo/bar.txt', + ) + ) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=['foo/bar/baz.txt', 'foo/bar.txt'], + location_config={}, + storage_config={}, + local_borg_version='1.2.3', + strip_components='all', + ) + + +def test_extract_archive_with_strip_components_all_and_no_paths_raises(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('execute_command').never() + + with pytest.raises(ValueError): + module.extract_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + location_config={}, + storage_config={}, + local_borg_version='1.2.3', + strip_components='all', + ) + + def test_extract_archive_calls_borg_with_progress_parameter(): flexmock(module.os.path).should_receive('abspath').and_return('repo') flexmock(module.environment).should_receive('make_environment') From c6829782a3a84293019b6473dc79fdc1e2219c6b Mon Sep 17 00:00:00 2001 From: Nain Date: Wed, 15 Mar 2023 06:50:47 -0400 Subject: [PATCH 19/39] Fix --editable (mode) option given --user as arg --user option should be before, or after `--editable .` not in between. Before seems better. --- docs/how-to/develop-on-borgmatic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 80ecf304..fbc1d244 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -26,7 +26,7 @@ make sure your changes work. ```bash cd borgmatic/ -pip3 install --editable --user . +pip3 install --user --editable . ``` Note that this will typically install the borgmatic commands into From 5fad2bd4089af0f5ee572ad9a254753d0601ce27 Mon Sep 17 00:00:00 2001 From: Nain Date: Wed, 15 Mar 2023 07:54:49 -0400 Subject: [PATCH 20/39] Better indicate position of additional docs on page On wide screens, the position of the documentation (how-to and reference guide) is at same level as #it's-your-data.-keep-it-that-way. So the jump due to anchor link makes it seem like we're taken to top aka main content. Indicate that links are to the left so reader doesn't recurse. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3b92dd37..27fc6cd2 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,8 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). Your first step is to [install and configure borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/). -For additional documentation, check out the links above for borgmatic how-to and +For additional documentation, check out the links above (left panel on wide screens) +for borgmatic how-to and reference guides. From cdbe6cdf3ab5d26d0d688ddc9b44bf90a383c3fa Mon Sep 17 00:00:00 2001 From: Nain Date: Wed, 15 Mar 2023 12:39:54 -0400 Subject: [PATCH 21/39] Add "--repository" flag to the "prune" action part of ticket #564 --- borgmatic/actions/prune.py | 6 +++ borgmatic/commands/arguments.py | 4 ++ tests/unit/actions/test_prune.py | 72 +++++++++++++++++++++++++++++++- 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 2d214a18..ca098ce4 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -1,6 +1,7 @@ import logging import borgmatic.borg.prune +import borgmatic.config.validate import borgmatic.hooks.command logger = logging.getLogger(__name__) @@ -23,6 +24,11 @@ def run_prune( ''' Run the "prune" action for the given repository. ''' + if prune_arguments.repository and not borgmatic.config.validate.repositories_match( + repository, prune_arguments.repository + ): + return + borgmatic.hooks.command.execute_hook( hooks.get('before_prune'), hooks.get('umask'), diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3f56b7cd..a8e8aac2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -333,6 +333,10 @@ def make_parsers(): add_help=False, ) prune_group = prune_parser.add_argument_group('prune arguments') + prune_group.add_argument( + '--repository', + help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file)', + ) prune_group.add_argument( '--stats', dest='stats', diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index d34a9c84..21e82600 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -7,7 +7,77 @@ def test_run_prune_calls_hooks(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) - prune_arguments = flexmock(stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_prune( + config_filename='test.yaml', + repository='repo', + storage={}, + retention={}, + hooks={}, + hook_context={}, + local_borg_version=None, + prune_arguments=prune_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + +def test_run_prune_runs_with_no_explicit_repository(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') + prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_prune( + config_filename='test.yaml', + repository='repo', + storage={}, + retention={}, + hooks={}, + hook_context={}, + local_borg_version=None, + prune_arguments=prune_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + +def test_run_prune_runs_with_select_repository(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') + prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_prune( + config_filename='test.yaml', + repository='repo', + storage={}, + retention={}, + hooks={}, + hook_context={}, + local_borg_version=None, + prune_arguments=prune_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + +def test_run_prune_bails_if_repository_does_not_match(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( + False + ) + flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').never() + prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( From 7de9260b0d14a7b68aee31404b5b757d8460d0a9 Mon Sep 17 00:00:00 2001 From: Nain Date: Wed, 15 Mar 2023 14:59:12 -0400 Subject: [PATCH 22/39] Remove test now that --repository isn't expected to error As discussed #652#issuecomment-5579 --- tests/integration/commands/test_arguments.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index cb001c10..754564b2 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -254,13 +254,6 @@ def test_parse_arguments_allows_init_and_create(): module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') -def test_parse_arguments_disallows_repository_unless_action_consumes_it(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg') - - def test_parse_arguments_allows_repository_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) From ce0ce4cd1c7016e302f7066f249d873a9b579e56 Mon Sep 17 00:00:00 2001 From: Nain Date: Thu, 16 Mar 2023 08:23:21 -0400 Subject: [PATCH 23/39] Merge mostly repetetive tests --- tests/unit/actions/test_prune.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index 21e82600..6c09e0c9 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -3,7 +3,7 @@ from flexmock import flexmock from borgmatic.actions import prune as module -def test_run_prune_calls_hooks(): +def test_run_prune_calls_hooks_of_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) @@ -26,28 +26,6 @@ def test_run_prune_calls_hooks(): ) -def test_run_prune_runs_with_no_explicit_repository(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') - prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - module.run_prune( - config_filename='test.yaml', - repository='repo', - storage={}, - retention={}, - hooks={}, - hook_context={}, - local_borg_version=None, - prune_arguments=prune_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - - def test_run_prune_runs_with_select_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) From 480addd7ce9f34996712f3f250d56d8d20dd3abd Mon Sep 17 00:00:00 2001 From: Nain Date: Thu, 16 Mar 2023 08:34:52 -0400 Subject: [PATCH 24/39] Add "--repository" flag to the "check" action --- borgmatic/actions/check.py | 6 ++++ borgmatic/commands/arguments.py | 4 +++ tests/unit/actions/test_check.py | 60 ++++++++++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 580c70ef..f3572395 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -1,6 +1,7 @@ import logging import borgmatic.borg.check +import borgmatic.config.validate import borgmatic.hooks.command logger = logging.getLogger(__name__) @@ -23,6 +24,11 @@ def run_check( ''' Run the "check" action for the given repository. ''' + if check_arguments.repository and not borgmatic.config.validate.repositories_match( + repository, check_arguments.repository + ): + return + borgmatic.hooks.command.execute_hook( hooks.get('before_check'), hooks.get('umask'), diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index a8e8aac2..fbf71685 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -419,6 +419,10 @@ def make_parsers(): add_help=False, ) check_group = check_parser.add_argument_group('check arguments') + check_group.add_argument( + '--repository', + help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file)', + ) check_group.add_argument( '--progress', dest='progress', diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 0007ee3f..db031403 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -3,7 +3,7 @@ from flexmock import flexmock from borgmatic.actions import check as module -def test_run_check_calls_hooks(): +def test_run_check_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.checks).should_receive( 'repository_enabled_for_checks' @@ -11,7 +11,63 @@ def test_run_check_calls_hooks(): flexmock(module.borgmatic.borg.check).should_receive('check_archives') flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( - progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() + repository=None, progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_check( + config_filename='test.yaml', + repository='repo', + location={'repositories': ['repo']}, + storage={}, + consistency={}, + hooks={}, + hook_context={}, + local_borg_version=None, + check_arguments=check_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_check_runs_with_select_repository(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.check).should_receive('check_archives') + check_arguments = flexmock( + repository='repo', progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_check( + config_filename='test.yaml', + repository='repo', + location={'repositories': ['repo']}, + storage={}, + consistency={}, + hooks={}, + hook_context={}, + local_borg_version=None, + check_arguments=check_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_check_bails_if_repository_does_not_match(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( + False + ) + flexmock(module.borgmatic.borg.check).should_receive('check_archives').never() + check_arguments = flexmock( + repository='repo2', + progress=flexmock(), + repair=flexmock(), + only=flexmock(), + force=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) From a8aeace5b52e9d3bfff4a7860f34388f35454e77 Mon Sep 17 00:00:00 2001 From: Nain Date: Thu, 16 Mar 2023 11:13:45 -0400 Subject: [PATCH 25/39] Add "--repository" flag to the "compact" action --- borgmatic/actions/compact.py | 6 ++++ borgmatic/commands/arguments.py | 4 +++ tests/unit/actions/test_compact.py | 58 ++++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 00585b0b..7a25b829 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -2,6 +2,7 @@ import logging import borgmatic.borg.compact import borgmatic.borg.feature +import borgmatic.config.validate import borgmatic.hooks.command logger = logging.getLogger(__name__) @@ -24,6 +25,11 @@ def run_compact( ''' Run the "compact" action for the given repository. ''' + if compact_arguments.repository and not borgmatic.config.validate.repositories_match( + repository, compact_arguments.repository + ): + return + borgmatic.hooks.command.execute_hook( hooks.get('before_compact'), hooks.get('umask'), diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index fbf71685..6991db58 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -357,6 +357,10 @@ def make_parsers(): add_help=False, ) compact_group = compact_parser.add_argument_group('compact arguments') + compact_group.add_argument( + '--repository', + help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file)', + ) compact_group.add_argument( '--progress', dest='progress', diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index bc2b9406..fab21f32 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -3,13 +3,67 @@ from flexmock import flexmock from borgmatic.actions import compact as module -def test_compact_actions_calls_hooks(): +def test_compact_actions_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments') flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) compact_arguments = flexmock( - progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=None, progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_compact( + config_filename='test.yaml', + repository='repo', + storage={}, + retention={}, + hooks={}, + hook_context={}, + local_borg_version=None, + compact_arguments=compact_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + +def test_compact_runs_with_select_repository(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments') + compact_arguments = flexmock( + repository='repo', progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_compact( + config_filename='test.yaml', + repository='repo', + storage={}, + retention={}, + hooks={}, + hook_context={}, + local_borg_version=None, + compact_arguments=compact_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + +def test_compact_bails_if_repository_does_not_match(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( + False + ) + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').never() + compact_arguments = flexmock( + repository='repo2', progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) From 5f87ea3ec5146f07ce639580d1752344646ff925 Mon Sep 17 00:00:00 2001 From: Nain Date: Thu, 16 Mar 2023 13:15:49 -0400 Subject: [PATCH 26/39] Add "--repository" flag to the "create" action --- borgmatic/actions/create.py | 6 +++ borgmatic/commands/arguments.py | 4 ++ tests/unit/actions/test_create.py | 74 ++++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index c882032a..96a48521 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -2,6 +2,7 @@ import json import logging import borgmatic.borg.create +import borgmatic.config.validate import borgmatic.hooks.command import borgmatic.hooks.dispatch import borgmatic.hooks.dump @@ -28,6 +29,11 @@ def run_create( If create_arguments.json is True, yield the JSON output from creating the archive. ''' + if create_arguments.repository and not borgmatic.config.validate.repositories_match( + repository, create_arguments.repository + ): + return + borgmatic.hooks.command.execute_hook( hooks.get('before_backup'), hooks.get('umask'), diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6991db58..d5dc6af4 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -393,6 +393,10 @@ def make_parsers(): add_help=False, ) create_group = create_parser.add_argument_group('create arguments') + create_group.add_argument( + '--repository', + help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file)', + ) create_group.add_argument( '--progress', dest='progress', diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 915f0ae1..04d7abc0 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -5,14 +5,84 @@ from borgmatic.actions import create as module def test_run_create_executes_and_calls_hooks(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.borg.create).should_receive('create_archive') + flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( 'call_hooks_even_if_unconfigured' ).and_return({}) create_arguments = flexmock( - progress=flexmock(), stats=flexmock(), json=flexmock(), list_files=flexmock() + repository=flexmock(), + progress=flexmock(), + stats=flexmock(), + json=flexmock(), + list_files=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + list( + module.run_create( + config_filename='test.yaml', + repository='repo', + location={}, + storage={}, + hooks={}, + hook_context={}, + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_runs_with_select_repository(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + stats=flexmock(), + json=flexmock(), + list_files=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + list( + module.run_create( + config_filename='test.yaml', + repository='repo', + location={}, + storage={}, + hooks={}, + hook_context={}, + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_bails_if_repository_does_not_match(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(False) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + stats=flexmock(), + json=flexmock(), + list_files=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) From 3e2241461331283d4332d08675611c6294288532 Mon Sep 17 00:00:00 2001 From: Nain Date: Thu, 16 Mar 2023 14:01:29 -0400 Subject: [PATCH 27/39] Update tests Make them more explicit. Also formatting. --- tests/unit/actions/test_check.py | 27 +++++++++++++++++---------- tests/unit/actions/test_compact.py | 19 +++++++++++-------- tests/unit/actions/test_create.py | 5 +++-- tests/unit/actions/test_prune.py | 15 +++++++++------ 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index db031403..bd0cf1c7 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -8,10 +8,11 @@ def test_run_check_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.config.checks).should_receive( 'repository_enabled_for_checks' ).and_return(True) - flexmock(module.borgmatic.borg.check).should_receive('check_archives') + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() + flexmock(module.borgmatic.borg.check).should_receive('check_archives').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( - repository=None, progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() + repository=None, progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -33,16 +34,22 @@ def test_run_check_calls_hooks_for_configured_repository(): def test_run_check_runs_with_select_repository(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borgmatic.borg.check).should_receive('check_archives') + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.check).should_receive('check_archives').once() check_arguments = flexmock( - repository='repo', progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() + repository=flexmock(), + progress=flexmock(), + repair=flexmock(), + only=flexmock(), + force=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_check( config_filename='test.yaml', - repository='repo', + repository=flexmock(), location={'repositories': ['repo']}, storage={}, consistency={}, @@ -58,12 +65,12 @@ def test_run_check_runs_with_select_repository(): def test_run_check_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( - False - ) + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(False) flexmock(module.borgmatic.borg.check).should_receive('check_archives').never() check_arguments = flexmock( - repository='repo2', + repository=flexmock(), progress=flexmock(), repair=flexmock(), only=flexmock(), diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index fab21f32..97f3d5b7 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -6,7 +6,8 @@ from borgmatic.actions import compact as module def test_compact_actions_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) - flexmock(module.borgmatic.borg.compact).should_receive('compact_segments') + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) compact_arguments = flexmock( repository=None, progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() @@ -31,11 +32,13 @@ def test_compact_actions_calls_hooks_for_configured_repository(): def test_compact_runs_with_select_repository(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) - flexmock(module.borgmatic.borg.compact).should_receive('compact_segments') + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository='repo', progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -58,12 +61,12 @@ def test_compact_runs_with_select_repository(): def test_compact_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) - flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( - False - ) + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(False) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').never() compact_arguments = flexmock( - repository='repo2', progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 04d7abc0..6eb18c2c 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -3,8 +3,9 @@ from flexmock import flexmock from borgmatic.actions import create as module -def test_run_create_executes_and_calls_hooks(): +def test_run_create_executes_and_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) @@ -12,7 +13,7 @@ def test_run_create_executes_and_calls_hooks(): 'call_hooks_even_if_unconfigured' ).and_return({}) create_arguments = flexmock( - repository=flexmock(), + repository=None, progress=flexmock(), stats=flexmock(), json=flexmock(), diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index 6c09e0c9..b9119898 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -5,7 +5,8 @@ from borgmatic.actions import prune as module def test_run_prune_calls_hooks_of_configured_repository(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() + flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -28,8 +29,10 @@ def test_run_prune_calls_hooks_of_configured_repository(): def test_run_prune_runs_with_select_repository(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -51,9 +54,9 @@ def test_run_prune_runs_with_select_repository(): def test_run_prune_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( - False - ) + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(False) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').never() prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) From 7605838bfefbdac0f072456ea9e59627ada5e5aa Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 16 Mar 2023 13:27:08 -0700 Subject: [PATCH 28/39] Add "--repository" flag to all actions where it makes sense (#564). --- NEWS | 2 ++ tests/unit/actions/test_check.py | 2 +- tests/unit/actions/test_compact.py | 2 +- tests/unit/actions/test_create.py | 2 +- tests/unit/actions/test_prune.py | 4 ++-- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 154e6f3c..0e8dd5ed 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ "create", "prune", "compact", "check". If you'd like to retain the old ordering ("prune" and "compact" first), then specify actions explicitly on the command-line. * #304: Run any command-line actions in the order specified instead of using a fixed ordering. + * #564: Add "--repository" flag to all actions where it makes sense, so you can run borgmatic on + a single configured repository instead of all of them. * #628: Add a Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling success or failure. * #647: Add "--strip-components all" feature on the "extract" action to remove leading path diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index bd0cf1c7..3e1a9c2f 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -32,7 +32,7 @@ def test_run_check_calls_hooks_for_configured_repository(): ) -def test_run_check_runs_with_select_repository(): +def test_run_check_runs_with_selected_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( 'repositories_match' diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 97f3d5b7..4dae903e 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -30,7 +30,7 @@ def test_compact_actions_calls_hooks_for_configured_repository(): ) -def test_compact_runs_with_select_repository(): +def test_compact_runs_with_selected_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( 'repositories_match' diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 6eb18c2c..8a9d0b4e 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -39,7 +39,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): ) -def test_run_create_runs_with_select_repository(): +def test_run_create_runs_with_selected_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( 'repositories_match' diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index b9119898..db9c1247 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -3,7 +3,7 @@ from flexmock import flexmock from borgmatic.actions import prune as module -def test_run_prune_calls_hooks_of_configured_repository(): +def test_run_prune_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() @@ -27,7 +27,7 @@ def test_run_prune_calls_hooks_of_configured_repository(): ) -def test_run_prune_runs_with_select_repository(): +def test_run_prune_runs_with_selected_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( 'repositories_match' From ca4461820dc98ae9f737d058b5ca936ae52a306d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 16 Mar 2023 13:29:37 -0700 Subject: [PATCH 29/39] Add support for Python 3.11. --- NEWS | 1 + tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 0e8dd5ed..1bf69455 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,7 @@ success or failure. * #647: Add "--strip-components all" feature on the "extract" action to remove leading path components of files you extract. Must be used with the "--path" flag. + * Add support for Python 3.11. 1.7.8 * #620: With the "create" action and the "--list" ("--files") flag, only show excluded files at diff --git a/tox.ini b/tox.ini index a58c871d..17b7a9da 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37,py38,py39,py310 +envlist = py37,py38,py39,py310,py311 skip_missing_interpreters = True skipsdist = True minversion = 3.14.1 @@ -13,7 +13,7 @@ whitelist_externals = passenv = COVERAGE_FILE commands = pytest {posargs} - py38,py39,py310: black --check . + py38,py39,py310,py311: black --check . isort --check-only --settings-path setup.cfg . flake8 borgmatic tests From bdfe4b61eb7e05f56f659135d10ac1fe87ee8781 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 16 Mar 2023 13:42:15 -0700 Subject: [PATCH 30/39] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 1bf69455..86b5fcf7 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.9.dev0 +1.7.9 * #295: Add a SQLite database dump/restore hook. * #304: Change the default action order when no actions are specified on the command-line to: "create", "prune", "compact", "check". If you'd like to retain the old ordering ("prune" and diff --git a/setup.py b/setup.py index 7ddb8117..ec68b049 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.9.dev0' +VERSION = '1.7.9' setup( From e6605c868d76c0342b9e34475f916368fa3b9ef5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 17 Mar 2023 10:09:36 -0700 Subject: [PATCH 31/39] Clarify check frequency default behavior (#653). --- docs/how-to/deal-with-very-large-backups.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index c02f718c..e5962c1e 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -57,8 +57,8 @@ cron job). Another option is to customize your consistency checks. By default, if you omit consistency checks from configuration, borgmatic runs full-repository checks (`repository`) and per-archive checks (`archives`) within each -repository, no more than once a month. This is equivalent to what `borg check` -does if run without options. +repository. (Although see below about check frequency.) This is equivalent to +what `borg check` does if run without options. But if you find that archive checks are too slow, for example, you can configure borgmatic to run repository checks only. Configure this in the @@ -70,8 +70,9 @@ consistency: - name: repository ``` -Prior to version 1.6.2 `checks` -was a plain list of strings without the `name:` part. For example: +Prior to version 1.6.2 The +`checks` option was a plain list of strings without the `name:` part, and +borgmatic ran each configured check every time checks were run. For example: ```yaml consistency: @@ -112,8 +113,13 @@ consistency: This tells borgmatic to run the `repository` consistency check at most once every two weeks for a given repository and the `archives` check at most once a month. The `frequency` value is a number followed by a unit of time, e.g. "3 -days", "1 week", "2 months", etc. The `frequency` defaults to `always`, which -means run this check every time checks run. +days", "1 week", "2 months", etc. + +The `frequency` defaults to `always` for a check configured without a +`frequency`, which means run this check every time checks run. But if you omit +consistency checks from configuration entirely, borgmatic runs full-repository +checks (`repository`) and per-archive checks (`archives`) within each +repository, at most once a month. Unlike a real scheduler like cron, borgmatic only makes a best effort to run checks on the configured frequency. It compares that frequency with how long From 0db137efdfd785eb403d8531a47fb17637c9296c Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Sat, 18 Mar 2023 01:48:24 +0530 Subject: [PATCH 32/39] add option to set borg_files_cache_ttl in config Signed-off-by: Soumik Dutta --- borgmatic/borg/environment.py | 3 ++- borgmatic/config/schema.yaml | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 235c674d..1b14369a 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -2,6 +2,7 @@ OPTION_TO_ENVIRONMENT_VARIABLE = { 'borg_base_directory': 'BORG_BASE_DIR', 'borg_config_directory': 'BORG_CONFIG_DIR', 'borg_cache_directory': 'BORG_CACHE_DIR', + 'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL', 'borg_security_directory': 'BORG_SECURITY_DIR', 'borg_keys_directory': 'BORG_KEYS_DIR', 'encryption_passcommand': 'BORG_PASSCOMMAND', @@ -27,7 +28,7 @@ def make_environment(storage_config): value = storage_config.get(option_name) if value: - environment[environment_variable_name] = value + environment[environment_variable_name] = str(value) for ( option_name, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f7161563..af0051b7 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -315,6 +315,12 @@ properties: Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg example: /path/to/base/cache + borg_files_cache_ttl: + type: integer + description: | + Maximum time to live (ttl) for entries in the borg files + cache. + example: 20 borg_security_directory: type: string description: | From fb9677230bb66af2f1b6175b52fd29a217f252dc Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Sat, 18 Mar 2023 02:57:56 +0530 Subject: [PATCH 33/39] add test to ensure integers are converted to string before setting them up to be environment variable values Signed-off-by: Soumik Dutta --- borgmatic/config/schema.yaml | 2 +- tests/unit/borg/test_environment.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index af0051b7..2cb6dbb3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -318,7 +318,7 @@ properties: borg_files_cache_ttl: type: integer description: | - Maximum time to live (ttl) for entries in the borg files + Maximum time to live (ttl) for entries in the Borg files cache. example: 20 borg_security_directory: diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index c5fce80d..4cef39b9 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -32,3 +32,8 @@ def test_make_environment_with_relocated_repo_access_should_override_default(): environment = module.make_environment({'relocated_repo_access_is_ok': True}) assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes' + + +def test_make_environment_with_integer_variable_value(): + environment = module.make_environment({'borg_files_cache_ttl': 40}) + assert environment.get('BORG_FILES_CACHE_TTL') == '40' From d17b2c74dbf8dba351b9664955901ea33c8c34b5 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 18 Mar 2023 04:35:55 +0530 Subject: [PATCH 34/39] feat: add optional check for existence of source directories --- borgmatic/borg/create.py | 18 ++++++++++++++++++ borgmatic/config/schema.yaml | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 0889c3d1..f3c3c69b 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -306,6 +306,22 @@ def collect_special_file_paths( ) +def check_all_source_directories_exist(source_directories): + ''' + Given a sequence of source directories, check that they all exist. If any do not, raise an + exception. + ''' + missing_directories = [ + source_directory + for source_directory in source_directories + if not os.path.exists(source_directory) + ] + if missing_directories: + raise ValueError( + 'Source directories do not exist: {}'.format(', '.join(missing_directories)) + ) + + def create_archive( dry_run, repository, @@ -331,6 +347,8 @@ def create_archive( borgmatic_source_directories = expand_directories( collect_borgmatic_source_directories(location_config.get('borgmatic_source_directory')) ) + if location_config.get('source_directories_must_exist', False): + check_all_source_directories_exist(location_config.get('source_directories')) sources = deduplicate_directories( map_directories_to_devices( expand_directories( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 2f873b7a..0135299c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -202,6 +202,12 @@ properties: path prevents "borgmatic restore" from finding any database dumps created before the change. Defaults to ~/.borgmatic example: /tmp/borgmatic + source_directories_must_exist: + type: boolean + description: | + If true, then source directories must exist, otherwise an + error is raised. Defaults to false. + example: true storage: type: object description: | From c84b26499b2baa03e712921c1bc1209ebae4ec6c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 17 Mar 2023 19:29:10 -0700 Subject: [PATCH 35/39] Add "borg_files_cache_ttl" option to NEWS. --- NEWS | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 86b5fcf7..7e8e6f9f 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.7.10.dev0 + * #618: Support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in + borgmatic's storage configuration. + 1.7.9 * #295: Add a SQLite database dump/restore hook. * #304: Change the default action order when no actions are specified on the command-line to: diff --git a/setup.py b/setup.py index ec68b049..5ea3c2e4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.9' +VERSION = '1.7.10.dev0' setup( From 997f60b3e660a4b219c611d83d628c9bcf55626a Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 18 Mar 2023 17:24:21 +0530 Subject: [PATCH 36/39] add tests --- borgmatic/borg/create.py | 8 +++--- tests/unit/borg/test_create.py | 45 ++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index f3c3c69b..f459e51f 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -317,9 +317,7 @@ def check_all_source_directories_exist(source_directories): if not os.path.exists(source_directory) ] if missing_directories: - raise ValueError( - 'Source directories do not exist: {}'.format(', '.join(missing_directories)) - ) + raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}") def create_archive( @@ -509,7 +507,9 @@ def create_archive( ) elif output_log_level is None: return execute_command_and_capture_output( - create_command, working_directory=working_directory, extra_environment=borg_environment, + create_command, + working_directory=working_directory, + extra_environment=borg_environment, ) else: execute_command( diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3c4e4338..835ebed1 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -207,7 +207,6 @@ def test_make_exclude_flags_includes_exclude_patterns_filename_when_given(): def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config(): - exclude_flags = module.make_exclude_flags( location_config={'exclude_from': ['excludes', 'other']} ) @@ -1054,7 +1053,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), + 'feature_available,option_flag', + ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), ) def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_parameters( feature_available, option_flag @@ -1189,7 +1189,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner')), + 'feature_available,option_flag', + ((True, '--numeric-ids'), (False, '--numeric-owner')), ) def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( feature_available, option_flag @@ -1291,7 +1292,12 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize( 'option_name,option_value', - (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), + ( + ('ctime', True), + ('ctime', False), + ('birthtime', True), + ('birthtime', False), + ), ) def test_create_archive_with_basic_option_calls_borg_with_corresponding_parameter( option_name, option_value @@ -1767,7 +1773,12 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info', '--progress',), + ('borg', 'create') + + REPO_ARCHIVE_WITH_PATHS + + ( + '--info', + '--progress', + ), output_log_level=logging.INFO, output_file=module.DO_NOT_CAPTURE, borg_local_path='borg', @@ -2530,3 +2541,27 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read local_borg_version='1.2.3', stream_processes=processes, ) + + +def test_create_archive_with_non_existent_directory_and_source_directories_must_exist_raises_error(): + """ + If a source directory doesn't exist and source_directories_must_exist is True, raise an error. + """ + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.os.path).should_receive('exists').and_return(False) + + with pytest.raises(ValueError): + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + 'source_directories_must_exist': True, + }, + storage_config={}, + local_borg_version='1.2.3', + ) From f803836416f15c50da237b1e419ee4fff1810654 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 18 Mar 2023 17:27:33 +0530 Subject: [PATCH 37/39] reformat --- borgmatic/borg/create.py | 4 +--- tests/unit/borg/test_create.py | 20 ++++---------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index f459e51f..87a0fdd7 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -507,9 +507,7 @@ def create_archive( ) elif output_log_level is None: return execute_command_and_capture_output( - create_command, - working_directory=working_directory, - extra_environment=borg_environment, + create_command, working_directory=working_directory, extra_environment=borg_environment, ) else: execute_command( diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 835ebed1..3a5bddc1 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -1053,8 +1053,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( @pytest.mark.parametrize( - 'feature_available,option_flag', - ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), + 'feature_available,option_flag', ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), ) def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_parameters( feature_available, option_flag @@ -1189,8 +1188,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par @pytest.mark.parametrize( - 'feature_available,option_flag', - ((True, '--numeric-ids'), (False, '--numeric-owner')), + 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner')), ) def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( feature_available, option_flag @@ -1292,12 +1290,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize( 'option_name,option_value', - ( - ('ctime', True), - ('ctime', False), - ('birthtime', True), - ('birthtime', False), - ), + (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), ) def test_create_archive_with_basic_option_calls_borg_with_corresponding_parameter( option_name, option_value @@ -1773,12 +1766,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') - + REPO_ARCHIVE_WITH_PATHS - + ( - '--info', - '--progress', - ), + ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info', '--progress',), output_log_level=logging.INFO, output_file=module.DO_NOT_CAPTURE, borg_local_path='borg', From 55c153409ea08b43ae0aa0e74a2ad3854bcc1cad Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Mar 2023 14:07:38 -0700 Subject: [PATCH 38/39] Add "source_directories_must_exist" option to NEWS (#501). --- NEWS | 2 ++ tests/unit/borg/test_create.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 7e8e6f9f..239b30a0 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.7.10.dev0 + * #501: Optionally error if a source directory does not exist when "source_directories_must_exist" + option in location configuration section is true. * #618: Support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 04afa872..69a3ede2 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -2532,9 +2532,9 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read def test_create_archive_with_non_existent_directory_and_source_directories_must_exist_raises_error(): - """ + ''' If a source directory doesn't exist and source_directories_must_exist is True, raise an error. - """ + ''' flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) From 6351747da58bf82103444300af1e07acffad317a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 19 Mar 2023 09:02:47 -0700 Subject: [PATCH 39/39] Add NixOS package link to installation docs. --- NEWS | 4 ++-- docs/how-to/set-up-backups.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 239b30a0..6eb4f3e4 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 1.7.10.dev0 - * #501: Optionally error if a source directory does not exist when "source_directories_must_exist" - option in location configuration section is true. + * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" + option in borgmatic's location configuration. * #618: Support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index be133e34..52962c34 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -94,6 +94,7 @@ installing borgmatic: * [openSUSE](https://software.opensuse.org/package/borgmatic) * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic) * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/) + * [NixOS](https://search.nixos.org/packages?show=borgmatic&sort=relevance&type=packages&query=borgmatic) * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup) * [virtualenv](https://virtualenv.pypa.io/en/stable/)