From 48f44d2f3d06dffdc5b8627c48385b391314ba7e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 8 Feb 2022 12:05:02 -0800 Subject: [PATCH] Add tests for compact action (#394). --- borgmatic/borg/compact.py | 5 +- borgmatic/commands/arguments.py | 4 +- borgmatic/commands/borgmatic.py | 1 - borgmatic/config/schema.yaml | 10 +-- docs/Dockerfile | 2 +- scripts/run-full-tests | 7 +- test_requirements.txt | 6 +- tests/end-to-end/docker-compose.yaml | 2 +- tests/unit/borg/test_compact.py | 112 ++++++++++++++++++++++++++ tests/unit/commands/test_borgmatic.py | 10 +++ 10 files changed, 141 insertions(+), 18 deletions(-) create mode 100644 tests/unit/borg/test_compact.py diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 9c6f660e3..115fb939c 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -9,7 +9,6 @@ def compact_segments( dry_run, repository, storage_config, - retention_config, local_path='borg', remote_path=None, progress=False, @@ -17,8 +16,8 @@ def compact_segments( threshold=None, ): ''' - Given dry-run flag, a local or remote repository path, a storage config dict, and a - retention config dict, compact Borg segments in a repository. + Given dry-run flag, a local or remote repository path, and a storage config dict, compact Borg + segments in a repository. ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 90061f1d7..5f6a75681 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -287,7 +287,9 @@ def parse_arguments(*unparsed_arguments): dest='threshold', help='Minimum saved space percentage threshold for compacting a segment, defaults to 10', ) - compact_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') + compact_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) create_parser = subparsers.add_parser( 'create', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 3b21b645f..ee92a5a7b 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -337,7 +337,6 @@ def run_actions( global_arguments.dry_run, repository, storage, - retention, local_path=local_path, remote_path=remote_path, progress=arguments['compact'].progress, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index dba6b5f29..684070f18 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -346,27 +346,27 @@ properties: init: type: string description: | - Extra command-line options to pass to "borg init". + Extra command-line options to pass to "borg init". example: "--make-parent-dirs" prune: type: string description: | - Extra command-line options to pass to "borg prune". + Extra command-line options to pass to "borg prune". example: "--save-space" compact: type: string description: | - Extra command-line options to pass to "borg compact". + Extra command-line options to pass to "borg compact". example: "--save-space" create: type: string description: | - Extra command-line options to pass to "borg create". + Extra command-line options to pass to "borg create". example: "--no-files-cache" check: type: string description: | - Extra command-line options to pass to "borg check". + Extra command-line options to pass to "borg check". example: "--save-space" description: | Additional options to pass directly to particular Borg diff --git a/docs/Dockerfile b/docs/Dockerfile index b9f4e22f8..feadcd296 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.8-alpine3.12 as borgmatic COPY . /app -RUN pip install --no-cache ruamel.yaml.clib==0.2.2 /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml +RUN pip install --no-cache ruamel.yaml.clib==0.2.6 /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ && for action in init prune create check extract export-tar mount umount restore list info borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ diff --git a/scripts/run-full-tests b/scripts/run-full-tests index 0e9d02330..31cf3cfec 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -10,11 +10,12 @@ set -e -apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools +apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ + py3-ruamel.yaml.clib # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true -python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0 -pip3 install tox==3.24.4 +python3 -m pip install --no-cache --upgrade pip==22.0.3 setuptools==60.8.1 +pip3 install tox==3.24.5 export COVERAGE_FILE=/tmp/.coverage tox --workdir /tmp/.tox --sitepackages tox --workdir /tmp/.tox --sitepackages -e end-to-end diff --git a/test_requirements.txt b/test_requirements.txt index 6513bcaae..9e56c3473 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -4,15 +4,15 @@ black==19.10b0; python_version >= '3.8' click==7.1.2; python_version >= '3.8' colorama==0.4.4 coverage==5.3 -flake8==3.8.4 +flake8==4.0.1 flexmock==0.10.4 isort==5.9.1 mccabe==0.6.1 pluggy==0.13.1 pathspec==0.8.1; python_version >= '3.8' py==1.10.0 -pycodestyle==2.6.0 -pyflakes==2.2.0 +pycodestyle==2.8.0 +pyflakes==2.4.0 jsonschema==3.2.0 pytest==6.2.5 pytest-cov==3.0.0 diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 094ac8d37..835c2a70a 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -16,7 +16,7 @@ services: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test tests: - image: alpine:3.13 + image: alpine:3.15 volumes: - "../..:/app:ro" tmpfs: diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py new file mode 100644 index 000000000..d6170607c --- /dev/null +++ b/tests/unit/borg/test_compact.py @@ -0,0 +1,112 @@ +import logging + +from flexmock import flexmock + +from borgmatic.borg import compact as module + +from ..test_verbosity import insert_logging_mock + + +def insert_execute_command_mock(compact_command, output_log_level): + flexmock(module).should_receive('execute_command').with_args( + compact_command, output_log_level=output_log_level, borg_local_path=compact_command[0] + ).once() + + +COMPACT_COMMAND = ('borg', 'compact') + + +def test_compact_segments_calls_borg_with_parameters(): + insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.WARNING) + + module.compact_segments(dry_run=False, repository='repo', storage_config={}) + + +def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): + insert_execute_command_mock(COMPACT_COMMAND + ('--info', 'repo'), logging.WARNING) + insert_logging_mock(logging.INFO) + + module.compact_segments(repository='repo', storage_config={}, dry_run=False) + + +def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): + insert_execute_command_mock(COMPACT_COMMAND + ('--debug', '--show-rc', 'repo'), logging.WARNING) + insert_logging_mock(logging.DEBUG) + + module.compact_segments(repository='repo', storage_config={}, dry_run=False) + + +def test_compact_segments_with_dry_run_calls_borg_with_dry_run_parameter(): + insert_execute_command_mock(COMPACT_COMMAND + ('--dry-run', 'repo'), logging.WARNING) + + module.compact_segments(repository='repo', storage_config={}, dry_run=True) + + +def test_compact_segments_with_local_path_calls_borg_via_local_path(): + insert_execute_command_mock(('borg1',) + COMPACT_COMMAND[1:] + ('repo',), logging.WARNING) + + module.compact_segments( + dry_run=False, repository='repo', storage_config={}, local_path='borg1', + ) + + +def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameters(): + insert_execute_command_mock( + COMPACT_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.WARNING + ) + + module.compact_segments( + dry_run=False, repository='repo', storage_config={}, remote_path='borg1', + ) + + +def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): + insert_execute_command_mock(COMPACT_COMMAND + ('--progress', 'repo'), logging.WARNING) + + module.compact_segments( + dry_run=False, repository='repo', storage_config={}, progress=True, + ) + + +def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_parameter(): + insert_execute_command_mock(COMPACT_COMMAND + ('--cleanup-commits', 'repo'), logging.WARNING) + + module.compact_segments( + dry_run=False, repository='repo', storage_config={}, cleanup_commits=True, + ) + + +def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): + insert_execute_command_mock(COMPACT_COMMAND + ('--threshold', '20', 'repo'), logging.WARNING) + + module.compact_segments( + dry_run=False, repository='repo', storage_config={}, threshold=20, + ) + + +def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): + storage_config = {'umask': '077'} + insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.WARNING) + + module.compact_segments( + dry_run=False, repository='repo', storage_config=storage_config, + ) + + +def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): + storage_config = {'lock_wait': 5} + insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.WARNING) + + module.compact_segments( + dry_run=False, repository='repo', storage_config=storage_config, + ) + + +def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options(): + insert_execute_command_mock(COMPACT_COMMAND + ('--extra', '--options', 'repo'), logging.WARNING) + + module.compact_segments( + dry_run=False, + repository='repo', + storage_config={'extra_borg_options': {'compact': '--extra --options'}}, + ) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 83db9c2ed..ba545f8e3 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -33,6 +33,16 @@ def test_run_configuration_calls_hooks_for_prune_action(): list(module.run_configuration('test.yaml', config, arguments)) +def test_run_configuration_calls_hooks_for_compact_action(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook').twice() + flexmock(module).should_receive('run_actions').and_return([]) + config = {'location': {'repositories': ['foo']}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'compact': flexmock()} + + list(module.run_configuration('test.yaml', config, arguments)) + + def test_run_configuration_executes_and_calls_hooks_for_create_action(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook').twice()