From ece56086774d00e9243c2fff78a61bdaa9144840 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 28 Jul 2018 22:27:39 -0700 Subject: [PATCH 1/7] Rev for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index a00e8835f..31c51801a 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.2.1.dev0 +1.2.1 * Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info. * #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/ path. diff --git a/setup.py b/setup.py index a451f513f..1f0ee2558 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.2.1.dev0' +VERSION = '1.2.1' setup( From 7b77fd251036b797c0026f74b90b99169170add6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 11 Aug 2018 13:59:27 -0700 Subject: [PATCH 2/7] Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52 (#85). --- NEWS | 4 ++++ borgmatic/config/validate.py | 4 ++-- setup.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 31c51801a..f4e03adcc 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.2.2 + * #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in + borgmatic as a pykwalify RuleError. + 1.2.1 * Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info. * #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/ diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index bdbbbf782..bb4ab4f37 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -74,8 +74,8 @@ def parse_configuration(config_filename, schema_filename): logging.getLogger('pykwalify').setLevel(logging.ERROR) try: - config = yaml.round_trip_load(open(config_filename)) - schema = yaml.round_trip_load(open(schema_filename)) + config = yaml.safe_load(open(config_filename)) + schema = yaml.safe_load(open(schema_filename)) except yaml.error.YAMLError as error: raise Validation_error(config_filename, (str(error),)) diff --git a/setup.py b/setup.py index 1f0ee2558..329b07cc4 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.2.1' +VERSION = '1.2.2' setup( From 5e2a5494af1c4452a0d12041df6f613f2283249c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Aug 2018 14:07:18 -0700 Subject: [PATCH 3/7] Fix declared pykwalify compatibility version range in setup.py (#88). --- NEWS | 4 ++++ setup.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index f4e03adcc..d55a0efc2 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.2.3.dev0 + * #88: Fix declared pykwalify compatibility version range in setup.py to prevent use of ancient + versions of pykwalify with large version numbers. + 1.2.2 * #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in borgmatic as a pykwalify RuleError. diff --git a/setup.py b/setup.py index 329b07cc4..bc2891e07 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.2.2' +VERSION = '1.2.3.dev0' setup( @@ -32,7 +32,7 @@ setup( 'atticmatic', ], install_requires=( - 'pykwalify>=1.6.0', + 'pykwalify>=1.6.0,<14.06', 'ruamel.yaml>0.15.0,<0.16.0', 'setuptools', ), From c7f4200417ac07a51ac8ec3610e26dff5f253e32 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 19 Aug 2018 11:24:48 -0700 Subject: [PATCH 4/7] Somewhat more robust mechanism to find unsupported Borg arguments. --- scripts/find-unsupported-borg-options | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/find-unsupported-borg-options b/scripts/find-unsupported-borg-options index cab633d79..723aeffbb 100755 --- a/scripts/find-unsupported-borg-options +++ b/scripts/find-unsupported-borg-options @@ -22,22 +22,28 @@ for sub_command in prune create check list info; do sort borgmatic_borg_flags > borgmatic_borg_flags.sorted mv borgmatic_borg_flags.sorted borgmatic_borg_flags - for line in $(borg $sub_command --help | awk -v RS= '/^usage:/') ; do + for word in $(borg $sub_command --help | grep '^ -') ; do # Exclude a bunch of flags that borgmatic actually supports, but don't get exercised by the # generated sample config, and also flags that don't make sense to support. - echo "$line" | grep -- -- | sed -r 's/(\[|\])//g' \ - | grep -v '^-h$' \ + echo "$word" | grep ^-- | sed -e 's/,$//' \ | grep -v '^--archives-only$' \ + | grep -v '^--critical$' \ + | grep -v '^--debug$' \ + | grep -v '^--dry-run$' \ + | grep -v '^--error$' \ + | grep -v '^--help$' \ + | grep -v '^--info$' \ + | grep -v '^--list$' \ + | grep -v '^--nobsdflags$' \ + | grep -v '^--pattern$' \ | grep -v '^--repository-only$' \ | grep -v '^--stats$' \ - | grep -v '^--list$' \ - | grep -v '^--critical$' \ - | grep -v '^--error$' \ + | grep -v '^--verbose$' \ | grep -v '^--warning$' \ - | grep -v '^--info$' \ - | grep -v '^--debug$' \ + | grep -v '^-h$' \ >> all_borg_flags done + sort all_borg_flags > all_borg_flags.sorted mv all_borg_flags.sorted all_borg_flags From da7aed381436f63c25a5d232340e009598ddcbab Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 19 Aug 2018 11:41:49 -0700 Subject: [PATCH 5/7] Support for Borg create --checkpoint-interval (#87). --- NEWS | 2 ++ borgmatic/borg/create.py | 2 ++ borgmatic/config/schema.yaml | 7 +++++++ borgmatic/tests/unit/borg/test_create.py | 20 ++++++++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/NEWS b/NEWS index d55a0efc2..4b0043d2c 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.2.3.dev0 + * #87: Support for Borg create --checkpoint-interval via "checkpoint_interval" option in + borgmatic's storage configuration. * #88: Fix declared pykwalify compatibility version range in setup.py to prevent use of ancient versions of pykwalify with large version numbers. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 632f39c12..3b8c9d9f8 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -115,6 +115,7 @@ def create_archive( pattern_file = _write_pattern_file(location_config.get('patterns')) exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns'))) + checkpoint_interval = storage_config.get('checkpoint_interval', None) compression = storage_config.get('compression', None) remote_rate_limit = storage_config.get('remote_rate_limit', None) umask = storage_config.get('umask', None) @@ -140,6 +141,7 @@ def create_archive( location_config, exclude_file.name if exclude_file else None, ) + + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ()) + (('--compression', compression) if compression else ()) + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()) + (('--one-file-system',) if location_config.get('one_file_system') else ()) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index a7b9a1759..7219a678c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -124,6 +124,13 @@ map: punctuation, so it parses correctly. And backslash any quote or backslash literals as well. example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" + checkpoint_interval: + type: int + desc: | + Number of seconds between each checkpoint during a long-running backup. See + https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there + for details. Defaults to checkpoints every 1800 seconds (30 minutes). + example: 1800 compression: type: scalar desc: | diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 411893c9f..f918ae13d 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -374,6 +374,26 @@ def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats ) +def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters(): + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + insert_subprocess_mock(CREATE_COMMAND + ('--checkpoint-interval', '600')) + + module.create_archive( + verbosity=None, + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'checkpoint_interval': 600}, + ) + + def test_create_archive_with_compression_calls_borg_with_compression_parameters(): flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) From 7d40a448cbdc1d91fa3b05b36e29ebb16e3c9821 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 19 Aug 2018 12:44:40 -0700 Subject: [PATCH 6/7] Pass --show-rc option to Borg when at highest verbosity level (#89). --- NEWS | 1 + borgmatic/borg/check.py | 2 +- borgmatic/borg/create.py | 4 ++-- borgmatic/borg/extract.py | 2 +- borgmatic/borg/info.py | 2 +- borgmatic/borg/list.py | 2 +- borgmatic/borg/prune.py | 2 +- borgmatic/tests/unit/borg/test_check.py | 2 +- borgmatic/tests/unit/borg/test_create.py | 4 ++-- borgmatic/tests/unit/borg/test_extract.py | 4 ++-- borgmatic/tests/unit/borg/test_info.py | 2 +- borgmatic/tests/unit/borg/test_list.py | 2 +- borgmatic/tests/unit/borg/test_prune.py | 2 +- scripts/find-unsupported-borg-options | 1 + 14 files changed, 17 insertions(+), 15 deletions(-) diff --git a/NEWS b/NEWS index 4b0043d2c..4b92e4de5 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ borgmatic's storage configuration. * #88: Fix declared pykwalify compatibility version range in setup.py to prevent use of ancient versions of pykwalify with large version numbers. + * #89: Pass --show-rc option to Borg when at highest verbosity level. 1.2.2 * #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index d5bdb8202..9d340dfe5 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -93,7 +93,7 @@ def check_archives(verbosity, repository, storage_config, consistency_config, lo lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = { VERBOSITY_SOME: ('--info',), - VERBOSITY_LOTS: ('--debug',), + VERBOSITY_LOTS: ('--debug', '--show-rc'), }.get(verbosity, ()) prefix = consistency_config.get('prefix') diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 3b8c9d9f8..1f038780a 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -151,8 +151,8 @@ def create_archive( + (('--umask', str(umask)) if umask else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + { - VERBOSITY_SOME: ('--info',) if dry_run else ('--info', '--stats',), - VERBOSITY_LOTS: ('--debug', '--list',) if dry_run else ('--debug', '--list', '--stats',), + VERBOSITY_SOME: ('--info',) if dry_run else ('--info', '--stats'), + VERBOSITY_LOTS: ('--debug', '--list', '--show-rc') if dry_run else ('--debug', '--list', '--show-rc', '--stats'), }.get(verbosity, ()) + (('--dry-run',) if dry_run else ()) ) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index d3d62a3c8..8318c4c8c 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -17,7 +17,7 @@ def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_pa lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = { VERBOSITY_SOME: ('--info',), - VERBOSITY_LOTS: ('--debug',), + VERBOSITY_LOTS: ('--debug', '--show-rc'), }.get(verbosity, ()) full_list_command = ( diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index b7417a803..1925374b2 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -23,7 +23,7 @@ def display_archives_info( + (('--json',) if json else ()) + { VERBOSITY_SOME: ('--info',), - VERBOSITY_LOTS: ('--debug',), + VERBOSITY_LOTS: ('--debug', '--show-rc'), }.get(verbosity, ()) ) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 30427814f..521ccd53b 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -23,7 +23,7 @@ def list_archives( + (('--json',) if json else ()) + { VERBOSITY_SOME: ('--info',), - VERBOSITY_LOTS: ('--debug',), + VERBOSITY_LOTS: ('--debug', '--show-rc'), }.get(verbosity, ()) ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 09e1c9b3d..2a04b3862 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -56,7 +56,7 @@ def prune_archives(verbosity, dry_run, repository, storage_config, retention_con + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + { VERBOSITY_SOME: ('--info', '--stats',), - VERBOSITY_LOTS: ('--debug', '--stats', '--list'), + VERBOSITY_LOTS: ('--debug', '--stats', '--list', '--show-rc'), }.get(verbosity, ()) + (('--dry-run',) if dry_run else ()) ) diff --git a/borgmatic/tests/unit/borg/test_check.py b/borgmatic/tests/unit/borg/test_check.py index b5d83786f..b6208dd0c 100644 --- a/borgmatic/tests/unit/borg/test_check.py +++ b/borgmatic/tests/unit/borg/test_check.py @@ -173,7 +173,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( - ('borg', 'check', 'repo', '--debug'), + ('borg', 'check', 'repo', '--debug', '--show-rc'), stdout=None, stderr=STDOUT, ) diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index f918ae13d..16bea78e8 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -296,7 +296,7 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter(): flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) - insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) + insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--show-rc', '--stats')) module.create_archive( verbosity=VERBOSITY_LOTS, @@ -359,7 +359,7 @@ def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) - insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--dry-run')) + insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--show-rc', '--dry-run')) module.create_archive( verbosity=VERBOSITY_LOTS, diff --git a/borgmatic/tests/unit/borg/test_extract.py b/borgmatic/tests/unit/borg/test_extract.py index 9811a7864..e8c377d19 100644 --- a/borgmatic/tests/unit/borg/test_extract.py +++ b/borgmatic/tests/unit/borg/test_extract.py @@ -73,11 +73,11 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter(): flexmock(sys.stdout).encoding = 'utf-8' insert_subprocess_check_output_mock( - ('borg', 'list', '--short', 'repo', '--debug'), + ('borg', 'list', '--short', 'repo', '--debug', '--show-rc'), result='archive1\narchive2\n'.encode('utf-8'), ) insert_subprocess_mock( - ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'), + ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--show-rc', '--list'), ) module.extract_last_archive_dry_run( diff --git a/borgmatic/tests/unit/borg/test_info.py b/borgmatic/tests/unit/borg/test_info.py index 43e881e27..1cfc8f3b9 100644 --- a/borgmatic/tests/unit/borg/test_info.py +++ b/borgmatic/tests/unit/borg/test_info.py @@ -35,7 +35,7 @@ def test_display_archives_info_with_verbosity_some_calls_borg_with_info_paramete def test_display_archives_info_with_verbosity_lots_calls_borg_with_debug_parameter(): - insert_subprocess_mock(INFO_COMMAND + ('--debug',)) + insert_subprocess_mock(INFO_COMMAND + ('--debug', '--show-rc')) module.display_archives_info( repository='repo', diff --git a/borgmatic/tests/unit/borg/test_list.py b/borgmatic/tests/unit/borg/test_list.py index 48bd095ab..01eec3061 100644 --- a/borgmatic/tests/unit/borg/test_list.py +++ b/borgmatic/tests/unit/borg/test_list.py @@ -35,7 +35,7 @@ def test_list_archives_with_verbosity_some_calls_borg_with_info_parameter(): def test_list_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): - insert_subprocess_mock(LIST_COMMAND + ('--debug',)) + insert_subprocess_mock(LIST_COMMAND + ('--debug', '--show-rc')) module.list_archives( repository='repo', diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py index 6e3a20df8..dccdfa878 100644 --- a/borgmatic/tests/unit/borg/test_prune.py +++ b/borgmatic/tests/unit/borg/test_prune.py @@ -92,7 +92,7 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list')) + insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list', '--show-rc')) module.prune_archives( repository='repo', diff --git a/scripts/find-unsupported-borg-options b/scripts/find-unsupported-borg-options index 723aeffbb..66188bd57 100755 --- a/scripts/find-unsupported-borg-options +++ b/scripts/find-unsupported-borg-options @@ -37,6 +37,7 @@ for sub_command in prune create check list info; do | grep -v '^--nobsdflags$' \ | grep -v '^--pattern$' \ | grep -v '^--repository-only$' \ + | grep -v '^--show-rc$' \ | grep -v '^--stats$' \ | grep -v '^--verbose$' \ | grep -v '^--warning$' \ From 9b48eb5a615dbc464a15112a40484ea66ed567eb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 19 Aug 2018 12:57:52 -0700 Subject: [PATCH 7/7] Clarify that --json can be used with --info command-line flag. --- borgmatic/commands/borgmatic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index d3e2d0461..e496e3d98 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -81,7 +81,7 @@ def parse_arguments(*arguments): dest='json', default=False, action='store_true', - help='Output results from the --list option as json', + help='Output results from the --list or --info options as json', ) parser.add_argument( '-n', '--dry-run',