borgmatic-collective/borgmatic
borgmatic-collective/borgmatic
10
66
Fork 30
Code Issues 45 Pull Requests 5 Actions Packages Releases 145 Activity

Initial work on delete/rdelete actions (#298).

Browse Source
This commit is contained in:
Dan Helfman 2024-06-28 16:20:10 -07:00
parent fc3b4a653e
commit e9a0226ee0
13 changed files with 490 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,4 +1,5 @@
1.8.13.dev0
* #298: Add "delete" and "rdelete" actions to delete archives or entire repositories.
* #785: Add an "only_run_on" option to consistency checks so you can limit a check to running on
particular days of the week. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-days

50
borgmatic/actions/delete.py Normal file
View File

@ -0,0 +1,50 @@
import logging
import borgmatic.actions.arguments
import borgmatic.borg.delete
import borgmatic.borg.rdelete
import borgmatic.borg.rlist
logger = logging.getLogger(__name__)
def run_delete(
repository,
config,
local_borg_version,
delete_arguments,
global_arguments,
local_path,
remote_path,
):
'''
Run the "delete" action for the given repository and archive(s).
'''
if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
repository, delete_arguments.repository
):
logger.answer(f'{repository.get("label", repository["path"])}: Deleting archives')
archive_name = (
borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
delete_arguments.archive,
config,
local_borg_version,
global_arguments,
local_path,
remote_path,
)
if delete_arguments.archive
else None
)
borgmatic.borg.delete.delete_archives(
repository,
config,
local_borg_version,
borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
global_arguments,
local_path,
remote_path,
)

33
borgmatic/actions/rdelete.py Normal file
View File

@ -0,0 +1,33 @@
import logging
import borgmatic.borg.rdelete
logger = logging.getLogger(__name__)
def run_rdelete(
repository,
config,
local_borg_version,
rdelete_arguments,
global_arguments,
local_path,
remote_path,
):
'''
Run the "rdelete" action for the given repository.
'''
if rdelete_arguments.repository is None or borgmatic.config.validate.repositories_match(
repository, rdelete_arguments.repository
):
logger.answer(f'{repository.get("label", repository["path"])}: Deleting repository')
borgmatic.borg.rdelete.delete_repository(
repository,
config,
local_borg_version,
rdelete_arguments,
global_arguments,
local_path,
remote_path,
)

128
borgmatic/borg/delete.py Normal file
View File

@ -0,0 +1,128 @@
import argparse
import logging
import borgmatic.borg.environment
import borgmatic.borg.feature
import borgmatic.borg.flags
import borgmatic.borg.rdelete
import borgmatic.execute
logger = logging.getLogger(__name__)
def make_delete_command(
repository,
config,
local_borg_version,
delete_arguments,
global_arguments,
local_path,
remote_path,
):
'''
Given a local or remote repository dict, a configuration dict, the local Borg version, the
arguments to the delete action as an argparse.Namespace, and global arguments, return a command
as a tuple to delete archives from the repository.
'''
return (
(local_path, 'delete')
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ borgmatic.borg.flags.make_flags('remote-path', remote_path)
+ borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+ borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+ borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives)
+ (
(('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
if delete_arguments.force
else ()
)
# Ignore match_archives and archive_name_format options from configuration, so the user has
# to be explicit on the command-line about the archives they want to delete.
+ borgmatic.borg.flags.make_match_archives_flags(
delete_arguments.match_archives or delete_arguments.archive,
archive_name_format=None,
local_borg_version=local_borg_version,
default_archive_name_format='*',
)
+ borgmatic.borg.flags.make_flags_from_arguments(
delete_arguments,
excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'),
)
+ borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
)
ARCHIVE_RELATED_ARGUMENT_NAMES = (
'archive',
'match_archives',
'first',
'last',
'oldest',
'newest',
'older',
'newer',
)
def delete_archives(
repository,
config,
local_borg_version,
delete_arguments,
global_arguments,
local_path='borg',
remote_path=None,
):
'''
Given a local or remote repository dict, a configuration dict, the local Borg version, the
arguments to the delete action as an argparse.Namespace, global arguments as an
argparse.Namespace, and local and remote Borg paths, delete the selected archives from the
repository. If no archives are selected, then delete the entire repository.
'''
borgmatic.logger.add_custom_log_levels()
if not any(
getattr(delete_arguments, argument_name) for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
):
if borgmatic.borg.feature.available(
borgmatic.borg.feature.Feature.RDELETE, local_borg_version
):
logger.warning(
'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the rdelete action instead.'
)
rdelete_arguments = argparse.Namespace(
repository=repository['path'],
list_archives=delete_arguments.list_archives,
force=delete_arguments.force,
cache_only=delete_arguments.cache_only,
keep_security_info=delete_arguments.keep_security_info,
)
return borgmatic.borg.rdelete.delete_repository(
repository,
config,
local_borg_version,
rdelete_arguments,
global_arguments,
local_path,
remote_path,
)
command = make_delete_command(
repository,
config,
local_borg_version,
delete_arguments,
global_arguments,
local_path,
remote_path,
)
borgmatic.execute.execute_command(
command,
output_log_level=logging.ANSWER,
extra_environment=borgmatic.borg.environment.make_environment(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

6
borgmatic/borg/feature.py
View File

@ -13,8 +13,9 @@ class Feature(Enum):
RCREATE = 7
RLIST = 8
RINFO = 9
MATCH_ARCHIVES = 10
EXCLUDED_FILES_MINUS = 11
RDELETE = 10
MATCH_ARCHIVES = 11
EXCLUDED_FILES_MINUS = 12
FEATURE_TO_MINIMUM_BORG_VERSION = {
@ -27,6 +28,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
Feature.RCREATE: parse('2.0.0a2'), # borg rcreate
Feature.RLIST: parse('2.0.0a2'), # borg rlist
Feature.RINFO: parse('2.0.0a2'), # borg rinfo
Feature.RDELETE: parse('2.0.0a2'), # borg rdelete
Feature.MATCH_ARCHIVES: parse('2.0.0b3'), # borg --match-archives
Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'), # --list --filter uses "-" for excludes
}

9
borgmatic/borg/flags.py
View File

@ -66,7 +66,12 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003
def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
def make_match_archives_flags(
match_archives,
archive_name_format,
local_borg_version,
default_archive_name_format=DEFAULT_ARCHIVE_NAME_FORMAT,
):
'''
Return match archives flags based on the given match archives value, if any. If it isn't set,
return match archives flags to match archives created with the given (or default) archive name
@ -83,7 +88,7 @@ def make_match_archives_flags(match_archives, archive_name_format, local_borg_ve
return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
derived_match_archives = re.sub(
r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format or DEFAULT_ARCHIVE_NAME_FORMAT
r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format or default_archive_name_format
)
if derived_match_archives == '*':

12
borgmatic/borg/list.py
View File

@ -144,12 +144,12 @@ def list_archive(
remote_path=None,
):
'''
Given a local or remote repository path, a configuration dict, the local Borg version, global
arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace,
and local and remote Borg paths, display the output of listing the files of a Borg archive (or
return JSON output). If list_arguments.find_paths are given, list the files by searching across
multiple archives. If neither find_paths nor archive name are given, instead list the archives
in the given repository.
Given a local or remote repository path, a configuration dict, the local Borg version, the
arguments to the list action as an argparse.Namespace, global arguments as an
argparse.Namespace, and local and remote Borg paths, display the output of listing the files of
a Borg archive (or return JSON output). If list_arguments.find_paths are given, list the files
by searching across multiple archives. If neither find_paths nor archive name are given, instead
list the archives in the given repository.
'''
borgmatic.logger.add_custom_log_levels()

91
borgmatic/borg/rdelete.py Normal file
View File

@ -0,0 +1,91 @@
import logging
import borgmatic.borg.environment
import borgmatic.borg.feature
import borgmatic.borg.flags
import borgmatic.execute
logger = logging.getLogger(__name__)
def make_rdelete_command(
repository,
config,
local_borg_version,
rdelete_arguments,
global_arguments,
local_path,
remote_path,
):
'''
Given a local or remote repository dict, a configuration dict, the local Borg version, the
arguments to the rdelete action as an argparse.Namespace, and global arguments, return a command
as a tuple to rdelete the entire repository.
'''
return (
(local_path,)
+ (
('rdelete',)
if borgmatic.borg.feature.available(
borgmatic.borg.feature.Feature.RDELETE, local_borg_version
)
else ('delete',)
)
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ borgmatic.borg.flags.make_flags('remote-path', remote_path)
+ borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+ borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+ borgmatic.borg.flags.make_flags('list', rdelete_arguments.list_archives)
+ (
(('--force',) + (('--force',) if rdelete_arguments.force >= 2 else ()))
if rdelete_arguments.force
else ()
)
+ borgmatic.borg.flags.make_flags_from_arguments(
rdelete_arguments, excludes=('list_archives', 'force', 'repository')
)
+ borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
)
def delete_repository(
repository,
config,
local_borg_version,
rdelete_arguments,
global_arguments,
local_path='borg',
remote_path=None,
):
'''
Given a local or remote repository dict, a configuration dict, the local Borg version, the
arguments to the rdelete action as an argparse.Namespace, global arguments as an
argparse.Namespace, and local and remote Borg paths, rdelete the entire repository.
'''
borgmatic.logger.add_custom_log_levels()
command = make_rdelete_command(
repository,
config,
local_borg_version,
rdelete_arguments,
global_arguments,
local_path,
remote_path,
)
borgmatic.execute.execute_command(
command,
output_log_level=logging.ANSWER,
# Don't capture output when Borg is expected to prompt for interactive confirmation, or the
# prompt won't work.
output_file=(
None
if rdelete_arguments.force or rdelete_arguments.cache_only
else borgmatic.execute.DO_NOT_CAPTURE
),
extra_environment=borgmatic.borg.environment.make_environment(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

132
borgmatic/commands/arguments.py
View File

@ -12,11 +12,13 @@ ACTION_ALIASES = {
'create': ['-C'],
'check': ['-k'],
'config': [],
'delete': [],
'extract': ['-x'],
'export-tar': [],
'mount': ['-m'],
'umount': ['-u'],
'restore': ['-r'],
'rdelete': [],
'rlist': [],
'list': ['-l'],
'rinfo': [],
@ -538,7 +540,7 @@ def make_parsers():
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive',
help='Display statistics of the pruned archive',
)
prune_group.add_argument(
'--list', dest='list_archives', action='store_true', help='List archives kept/pruned'
@ -689,6 +691,97 @@ def make_parsers():
)
check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
delete_parser = action_parsers.add_parser(
'delete',
aliases=ACTION_ALIASES['delete'],
help='Delete an archive from a repository or delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
description='Delete an archive from a repository or delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
add_help=False,
)
delete_group = delete_parser.add_argument_group('delete arguments')
delete_group.add_argument(
'--repository',
help='Path of repository to delete or delete archives from, defaults to the configured repository if there is only one',
)
delete_group.add_argument(
'--archive',
help='Archive to delete',
)
delete_group.add_argument(
'--list',
dest='list_archives',
action='store_true',
help='Show details for the deleted archives',
)
delete_group.add_argument(
'--stats',
action='store_true',
help='Display statistics for the deleted archives',
)
delete_group.add_argument(
'--cache-only',
action='store_true',
help='Delete only the local cache for the given repository',
)
delete_group.add_argument(
'--force',
action='count',
help='Force deletion of corrupted archives, can be given twice if once does not work',
)
delete_group.add_argument(
'--keep-security-info',
action='store_true',
help='Do not delete the local security info when deleting a repository',
)
delete_group.add_argument(
'--save-space',
action='store_true',
help='Work slower, but using less space',
)
delete_group.add_argument(
'--checkpoint-interval',
type=int,
metavar='SECONDS',
help='Write a checkpoint at the given interval, defaults to 1800 seconds (30 minutes)',
)
delete_group.add_argument(
'-a',
'--match-archives',
'--glob-archives',
metavar='PATTERN',
help='Only delete archives matching this pattern',
)
delete_group.add_argument(
'--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
)
delete_group.add_argument(
'--first', metavar='N', help='Delete first N archives after other filters are applied'
)
delete_group.add_argument(
'--last', metavar='N', help='Delete last N archives after other filters are applied'
)
delete_group.add_argument(
'--oldest',
metavar='TIMESPAN',
help='Delete archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
)
delete_group.add_argument(
'--newest',
metavar='TIMESPAN',
help='Delete archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
)
delete_group.add_argument(
'--older',
metavar='TIMESPAN',
help='Delete archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
)
delete_group.add_argument(
'--newer',
metavar='TIMESPAN',
help='Delete archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
)
delete_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
extract_parser = action_parsers.add_parser(
'extract',
aliases=ACTION_ALIASES['extract'],
@ -977,6 +1070,43 @@ def make_parsers():
)
umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
rdelete_parser = action_parsers.add_parser(
'rdelete',
aliases=ACTION_ALIASES['rdelete'],
help='Delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
description='Delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
add_help=False,
)
rdelete_group = rdelete_parser.add_argument_group('delete arguments')
rdelete_group.add_argument(
'--repository',
help='Path of repository to delete, defaults to the configured repository if there is only one',
)
rdelete_group.add_argument(
'--list',
dest='list_archives',
action='store_true',
help='Show details for the archives in the given repository',
)
rdelete_group.add_argument(
'--force',
action='count',
help='Force deletion of corrupted archives, can be given twice if once does not work',
)
rdelete_group.add_argument(
'--cache-only',
action='store_true',
help='Delete only the local cache for the given repository',
)
rdelete_group.add_argument(
'--keep-security-info',
action='store_true',
help='Do not delete the local security info when deleting a repository',
)
rdelete_group.add_argument(
'-h', '--help', action='help', help='Show this help message and exit'
)
restore_parser = action_parsers.add_parser(
'restore',
aliases=ACTION_ALIASES['restore'],

22
borgmatic/commands/borgmatic.py
View File

@ -18,6 +18,7 @@ import borgmatic.actions.config.bootstrap
import borgmatic.actions.config.generate
import borgmatic.actions.config.validate
import borgmatic.actions.create
import borgmatic.actions.delete
import borgmatic.actions.export_key
import borgmatic.actions.export_tar
import borgmatic.actions.extract
@ -26,6 +27,7 @@ import borgmatic.actions.list
import borgmatic.actions.mount
import borgmatic.actions.prune
import borgmatic.actions.rcreate
import borgmatic.actions.rdelete
import borgmatic.actions.restore
import borgmatic.actions.rinfo
import borgmatic.actions.rlist
@ -479,6 +481,26 @@ def run_actions(
local_path,
remote_path,
)
elif action_name == 'delete' and action_name not in skip_actions:
borgmatic.actions.delete.run_delete(
repository,
config,
local_borg_version,
action_arguments,
global_arguments,
local_path,
remote_path,
)
elif action_name == 'rdelete' and action_name not in skip_actions:
borgmatic.actions.rdelete.run_rdelete(
repository,
config,
local_borg_version,
action_arguments,
global_arguments,
local_path,
remote_path,
)
elif action_name == 'borg' and action_name not in skip_actions:
borgmatic.actions.borg.run_borg(
repository,

2
borgmatic/config/schema.yaml
View File

@ -747,11 +747,13 @@ properties:
- compact
- create
- check
- delete
- extract
- config
- export-tar
- mount
- umount
- rdelete
- restore
- rlist
- list

2
docs/Dockerfile
View File

@ -4,7 +4,7 @@ COPY . /app
RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
RUN borgmatic --help > /command-line.txt \
&& for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" "config validate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \
&& for action in rcreate transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount rdelete restore rlist list rinfo info break-lock borg; do \
echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
&& borgmatic $action --help >> /command-line.txt; done

14
tests/unit/borg/test_flags.py
View File

@ -190,6 +190,20 @@ def test_make_match_archives_flags_makes_flags_with_globs(
)
def test_make_match_archives_flags_accepts_default_archive_name_format():
flexmock(module.feature).should_receive('available').and_return(True)
assert (
module.make_match_archives_flags(
match_archives=None,
archive_name_format=None,
local_borg_version=flexmock(),
default_archive_name_format='*',
)
== ()
)
def test_warn_for_aggressive_archive_flags_without_archive_flags_bails():
flexmock(module.logger).should_receive('warning').never()