2019-02-18 09:30:34 -08:00
import collections
2023-11-07 10:17:16 -08:00
import importlib . metadata
2018-07-28 21:21:38 +00:00
import json
2017-10-25 21:47:33 -07:00
import logging
2016-06-10 11:21:53 -07:00
import os
import sys
2021-11-14 22:15:22 +00:00
import time
2021-07-14 22:46:02 +01:00
from queue import Queue
2021-07-14 23:17:35 +01:00
from subprocess import CalledProcessError
2016-06-10 11:21:53 -07:00
2019-05-13 23:39:10 +02:00
import colorama
2023-04-14 21:21:25 -07:00
2023-01-25 23:31:07 -08:00
import borgmatic . actions . borg
import borgmatic . actions . break_lock
2024-09-01 11:13:39 -07:00
import borgmatic . actions . change_passphrase
2023-01-25 23:31:07 -08:00
import borgmatic . actions . check
import borgmatic . actions . compact
2023-06-06 23:37:09 +05:30
import borgmatic . actions . config . bootstrap
2023-06-21 12:19:49 -07:00
import borgmatic . actions . config . generate
2023-06-23 10:11:41 -07:00
import borgmatic . actions . config . validate
2023-01-25 23:31:07 -08:00
import borgmatic . actions . create
2024-06-28 16:20:10 -07:00
import borgmatic . actions . delete
2023-08-07 12:28:39 -07:00
import borgmatic . actions . export_key
2023-01-25 23:31:07 -08:00
import borgmatic . actions . export_tar
import borgmatic . actions . extract
import borgmatic . actions . info
import borgmatic . actions . list
import borgmatic . actions . mount
import borgmatic . actions . prune
2024-09-09 10:05:32 -07:00
import borgmatic . actions . repo_create
import borgmatic . actions . repo_delete
import borgmatic . actions . repo_info
import borgmatic . actions . repo_list
2023-01-25 23:31:07 -08:00
import borgmatic . actions . restore
import borgmatic . actions . transfer
2023-06-15 10:55:31 -07:00
import borgmatic . commands . completion . bash
import borgmatic . commands . completion . fish
2019-11-25 14:56:20 -08:00
from borgmatic . borg import umount as borg_umount
2022-02-09 14:33:12 -08:00
from borgmatic . borg import version as borg_version
2019-06-22 21:32:27 -07:00
from borgmatic . commands . arguments import parse_arguments
2023-06-20 09:41:26 -07:00
from borgmatic . config import checks , collect , validate
2023-01-25 23:31:07 -08:00
from borgmatic . hooks import command , dispatch , monitor
2023-05-30 23:19:33 -07:00
from borgmatic . logger import DISABLED , add_custom_log_levels , configure_logging , should_do_markup
2017-10-31 21:58:35 -07:00
from borgmatic . signals import configure_signals
2018-09-08 20:53:37 +00:00
from borgmatic . verbosity import verbosity_to_log_level
2017-10-25 21:47:33 -07:00
2019-06-17 11:53:08 -07:00
logger = logging . getLogger ( __name__ )
2016-06-10 11:21:53 -07:00
2023-12-28 10:22:48 -08:00
def get_skip_actions ( config , arguments ) :
'''
Given a configuration dict and command - line arguments as an argparse . Namespace , return a list of
the configured action names to skip . Omit " check " from this list though if " check --force " is
part of the command - like arguments .
'''
skip_actions = config . get ( ' skip_actions ' , [ ] )
if ' check ' in arguments and arguments [ ' check ' ] . force :
return [ action for action in skip_actions if action != ' check ' ]
return skip_actions
2024-01-09 13:47:20 -08:00
def run_configuration ( config_filename , config , config_paths , arguments ) :
2017-10-29 16:44:15 -07:00
'''
2024-01-09 13:47:20 -08:00
Given a config filename , the corresponding parsed config dict , a sequence of loaded
configuration paths , and command - line arguments as a dict from subparser name to a namespace of
parsed arguments , execute the defined create , prune , compact , check , and / or other actions .
2019-04-02 22:30:14 -07:00
2019-09-30 22:19:31 -07:00
Yield a combination of :
* JSON output strings from successfully executing any actions that produce JSON
* logging . LogRecord instances containing errors from any actions or backup hooks that fail
2017-10-29 16:44:15 -07:00
'''
2019-06-22 16:10:07 -07:00
global_arguments = arguments [ ' global ' ]
2017-10-29 16:44:15 -07:00
2023-07-08 23:14:30 -07:00
local_path = config . get ( ' local_path ' , ' borg ' )
remote_path = config . get ( ' remote_path ' )
retries = config . get ( ' retries ' , 0 )
retry_wait = config . get ( ' retry_wait ' , 0 )
2019-10-01 12:23:16 -07:00
encountered_error = None
error_repository = ' '
2023-03-08 14:05:06 -08:00
using_primary_action = { ' create ' , ' prune ' , ' compact ' , ' check ' } . intersection ( arguments )
2020-01-22 15:10:47 -08:00
monitoring_log_level = verbosity_to_log_level ( global_arguments . monitoring_verbosity )
2023-05-12 19:05:52 +05:30
monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED
2023-12-28 10:22:48 -08:00
skip_actions = get_skip_actions ( config , arguments )
2023-10-31 21:54:41 -07:00
if skip_actions :
logger . debug (
f " { config_filename } : Skipping { ' / ' . join ( skip_actions ) } action { ' s ' if len ( skip_actions ) > 1 else ' ' } due to configured skip_actions "
)
2018-07-01 21:09:45 +02:00
2019-12-12 22:54:45 -08:00
try :
2023-07-08 23:14:30 -07:00
local_borg_version = borg_version . local_borg_version ( config , local_path )
2023-11-24 18:47:37 +01:00
logger . debug ( f ' { config_filename } : Borg { local_borg_version } ' )
2022-02-09 14:33:12 -08:00
except ( OSError , CalledProcessError , ValueError ) as error :
2023-03-23 23:11:14 -07:00
yield from log_error_records ( f ' { config_filename } : Error getting local Borg version ' , error )
2022-02-09 14:33:12 -08:00
return
try :
2023-05-12 19:05:52 +05:30
if monitoring_hooks_are_activated :
2019-11-12 15:31:07 -08:00
dispatch . call_hooks (
2020-06-02 14:33:41 -07:00
' initialize_monitor ' ,
2023-07-08 23:14:30 -07:00
config ,
2019-11-12 15:31:07 -08:00
config_filename ,
monitor . MONITOR_HOOK_NAMES ,
2020-01-22 15:10:47 -08:00
monitoring_log_level ,
2019-11-12 15:31:07 -08:00
global_arguments . dry_run ,
2019-11-07 10:08:44 -08:00
)
2023-05-12 19:05:52 +05:30
2020-06-02 14:33:41 -07:00
dispatch . call_hooks (
' ping_monitor ' ,
2023-07-08 23:14:30 -07:00
config ,
2020-06-02 14:33:41 -07:00
config_filename ,
monitor . MONITOR_HOOK_NAMES ,
monitor . State . START ,
monitoring_log_level ,
global_arguments . dry_run ,
)
2019-12-12 22:54:45 -08:00
except ( OSError , CalledProcessError ) as error :
2020-01-24 20:52:48 -08:00
if command . considered_soft_failure ( config_filename , error ) :
return
2019-12-12 22:54:45 -08:00
encountered_error = error
2023-03-23 23:11:14 -07:00
yield from log_error_records ( f ' { config_filename } : Error pinging monitor ' , error )
2017-10-29 16:44:15 -07:00
2019-09-30 22:19:31 -07:00
if not encountered_error :
2021-07-14 22:46:02 +01:00
repo_queue = Queue ( )
2023-07-08 23:14:30 -07:00
for repo in config [ ' repositories ' ] :
2023-04-14 19:35:24 -07:00
repo_queue . put (
( repo , 0 ) ,
)
2021-07-14 22:53:01 +01:00
2021-07-14 22:46:02 +01:00
while not repo_queue . empty ( ) :
2023-03-23 01:01:26 +05:30
repository , retry_num = repo_queue . get ( )
2023-05-16 09:36:50 -07:00
logger . debug (
f ' { repository . get ( " label " , repository [ " path " ] ) } : Running actions for repository '
)
2021-11-15 11:51:17 -08:00
timeout = retry_num * retry_wait
2021-11-14 22:15:22 +00:00
if timeout :
2023-05-16 09:36:50 -07:00
logger . warning (
f ' { repository . get ( " label " , repository [ " path " ] ) } : Sleeping { timeout } s before next retry '
)
2021-11-14 22:15:22 +00:00
time . sleep ( timeout )
2019-09-30 22:19:31 -07:00
try :
yield from run_actions (
arguments = arguments ,
2022-04-21 22:08:25 -07:00
config_filename = config_filename ,
2023-07-08 23:14:30 -07:00
config = config ,
2024-01-09 13:47:20 -08:00
config_paths = config_paths ,
2019-09-30 22:19:31 -07:00
local_path = local_path ,
remote_path = remote_path ,
2022-02-09 14:33:12 -08:00
local_borg_version = local_borg_version ,
2023-03-23 01:01:26 +05:30
repository = repository ,
2019-09-30 22:19:31 -07:00
)
2019-10-31 12:57:36 -07:00
except ( OSError , CalledProcessError , ValueError ) as error :
2021-07-14 22:46:02 +01:00
if retry_num < retries :
2023-04-14 19:35:24 -07:00
repo_queue . put (
( repository , retry_num + 1 ) ,
)
2022-04-02 22:28:41 -07:00
tuple ( # Consume the generator so as to trigger logging.
log_error_records (
2023-05-16 09:36:50 -07:00
f ' { repository . get ( " label " , repository [ " path " ] ) } : Error running actions for repository ' ,
2022-04-02 22:28:41 -07:00
error ,
levelno = logging . WARNING ,
2023-07-10 11:16:18 -07:00
log_command_error_output = True ,
2022-04-02 22:28:41 -07:00
)
)
2021-11-15 11:51:17 -08:00
logger . warning (
2023-05-16 09:36:50 -07:00
f ' { repository . get ( " label " , repository [ " path " ] ) } : Retrying... attempt { retry_num + 1 } / { retries } '
2021-11-15 11:51:17 -08:00
)
2021-07-14 22:46:02 +01:00
continue
2022-04-02 22:28:41 -07:00
2022-04-21 22:08:25 -07:00
if command . considered_soft_failure ( config_filename , error ) :
2024-10-06 17:39:02 -07:00
continue
2022-04-21 22:08:25 -07:00
2022-04-02 22:28:41 -07:00
yield from log_error_records (
2023-05-16 09:36:50 -07:00
f ' { repository . get ( " label " , repository [ " path " ] ) } : Error running actions for repository ' ,
error ,
2022-04-02 22:28:41 -07:00
)
2021-07-14 22:46:02 +01:00
encountered_error = error
2023-03-23 01:01:26 +05:30
error_repository = repository [ ' path ' ]
2017-10-29 16:44:15 -07:00
2023-03-05 19:27:32 +05:30
try :
2023-05-12 19:05:52 +05:30
if monitoring_hooks_are_activated :
2024-03-10 16:18:49 -07:00
# Send logs irrespective of error.
2023-03-06 03:38:08 +05:30
dispatch . call_hooks (
' ping_monitor ' ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-06 03:38:08 +05:30
config_filename ,
monitor . MONITOR_HOOK_NAMES ,
monitor . State . LOG ,
monitoring_log_level ,
global_arguments . dry_run ,
)
2023-03-05 19:27:32 +05:30
except ( OSError , CalledProcessError ) as error :
2024-03-18 23:15:28 -07:00
if not command . considered_soft_failure ( config_filename , error ) :
encountered_error = error
yield from log_error_records ( f ' { repository [ " path " ] } : Error pinging monitor ' , error )
2023-03-05 19:27:32 +05:30
2019-12-12 22:54:45 -08:00
if not encountered_error :
2019-09-30 22:19:31 -07:00
try :
2023-05-12 19:05:52 +05:30
if monitoring_hooks_are_activated :
2020-06-23 11:01:03 -07:00
dispatch . call_hooks (
' ping_monitor ' ,
2023-07-08 23:14:30 -07:00
config ,
2020-06-23 11:01:03 -07:00
config_filename ,
monitor . MONITOR_HOOK_NAMES ,
monitor . State . FINISH ,
monitoring_log_level ,
global_arguments . dry_run ,
)
dispatch . call_hooks (
' destroy_monitor ' ,
2023-07-08 23:14:30 -07:00
config ,
2020-06-23 11:01:03 -07:00
config_filename ,
monitor . MONITOR_HOOK_NAMES ,
monitoring_log_level ,
global_arguments . dry_run ,
)
2019-09-30 22:19:31 -07:00
except ( OSError , CalledProcessError ) as error :
2020-01-24 20:52:48 -08:00
if command . considered_soft_failure ( config_filename , error ) :
return
2019-10-01 12:23:16 -07:00
encountered_error = error
2023-03-23 23:11:14 -07:00
yield from log_error_records ( f ' { config_filename } : Error pinging monitor ' , error )
2019-09-30 22:19:31 -07:00
2022-02-09 14:33:12 -08:00
if encountered_error and using_primary_action :
2019-09-30 22:19:31 -07:00
try :
2020-06-23 11:01:03 -07:00
command . execute_hook (
2023-07-08 23:14:30 -07:00
config . get ( ' on_error ' ) ,
config . get ( ' umask ' ) ,
2020-06-23 11:01:03 -07:00
config_filename ,
' on-error ' ,
global_arguments . dry_run ,
repository = error_repository ,
error = encountered_error ,
output = getattr ( encountered_error , ' output ' , ' ' ) ,
)
2020-06-02 14:33:41 -07:00
dispatch . call_hooks (
' ping_monitor ' ,
2023-07-08 23:14:30 -07:00
config ,
2020-06-02 14:33:41 -07:00
config_filename ,
monitor . MONITOR_HOOK_NAMES ,
monitor . State . FAIL ,
monitoring_log_level ,
global_arguments . dry_run ,
)
2020-06-23 11:01:03 -07:00
dispatch . call_hooks (
' destroy_monitor ' ,
2023-07-08 23:14:30 -07:00
config ,
2019-09-30 22:19:31 -07:00
config_filename ,
2020-06-23 11:01:03 -07:00
monitor . MONITOR_HOOK_NAMES ,
monitoring_log_level ,
2019-09-30 22:19:31 -07:00
global_arguments . dry_run ,
)
except ( OSError , CalledProcessError ) as error :
2020-01-24 20:52:48 -08:00
if command . considered_soft_failure ( config_filename , error ) :
return
2023-03-23 23:11:14 -07:00
yield from log_error_records ( f ' { config_filename } : Error running on-error hook ' , error )
2017-10-29 16:44:15 -07:00
2019-04-02 22:30:14 -07:00
def run_actions (
2019-06-22 16:10:07 -07:00
* ,
arguments ,
2022-04-21 22:08:25 -07:00
config_filename ,
2023-07-08 23:14:30 -07:00
config ,
2024-01-09 13:47:20 -08:00
config_paths ,
2019-06-22 16:10:07 -07:00
local_path ,
remote_path ,
2022-02-09 14:33:12 -08:00
local_borg_version ,
2023-03-23 01:01:26 +05:30
repository ,
2022-04-21 22:08:25 -07:00
) :
2019-04-02 22:30:14 -07:00
'''
2022-04-21 22:08:25 -07:00
Given parsed command - line arguments as an argparse . ArgumentParser instance , the configuration
2024-01-09 13:47:20 -08:00
filename , a configuration dict , a sequence of loaded configuration paths , local and remote paths
to Borg , a local Borg version string , and a repository name , run all actions from the
command - line arguments on the given repository .
2019-04-02 22:30:14 -07:00
Yield JSON output strings from executing any actions that produce JSON .
2019-10-31 12:57:36 -07:00
Raise OSError or subprocess . CalledProcessError if an error occurs running a command for an
2022-04-21 22:08:25 -07:00
action or a hook . Raise ValueError if the arguments or configuration passed to action are
invalid .
2019-04-02 22:30:14 -07:00
'''
2022-12-02 12:12:10 -08:00
add_custom_log_levels ( )
2023-03-23 01:01:26 +05:30
repository_path = os . path . expanduser ( repository [ ' path ' ] )
2019-06-22 16:10:07 -07:00
global_arguments = arguments [ ' global ' ]
dry_run_label = ' (dry run; not making any changes) ' if global_arguments . dry_run else ' '
2022-04-21 22:08:25 -07:00
hook_context = {
2024-06-05 14:56:21 -07:00
' repository_label ' : repository . get ( ' label ' , ' ' ) ,
2024-06-05 14:47:37 -07:00
' log_file ' : global_arguments . log_file if global_arguments . log_file else ' ' ,
2022-04-21 22:08:25 -07:00
# Deprecated: For backwards compatibility with borgmatic < 1.6.0.
2023-07-08 23:14:30 -07:00
' repositories ' : ' , ' . join ( [ repo [ ' path ' ] for repo in config [ ' repositories ' ] ] ) ,
2024-06-05 14:47:37 -07:00
' repository ' : repository_path ,
2022-04-21 22:08:25 -07:00
}
2023-12-28 10:22:48 -08:00
skip_actions = set ( get_skip_actions ( config , arguments ) )
2022-04-21 22:08:25 -07:00
2022-08-21 21:48:37 -07:00
command . execute_hook (
2023-07-08 23:14:30 -07:00
config . get ( ' before_actions ' ) ,
config . get ( ' umask ' ) ,
2022-08-21 21:48:37 -07:00
config_filename ,
' pre-actions ' ,
global_arguments . dry_run ,
* * hook_context ,
)
2023-04-14 19:35:24 -07:00
for action_name , action_arguments in arguments . items ( ) :
2024-09-09 10:05:32 -07:00
if action_name == ' repo-create ' and action_name not in skip_actions :
borgmatic . actions . repo_create . run_repo_create (
2023-03-08 13:19:41 -08:00
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' transfer ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
borgmatic . actions . transfer . run_transfer (
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' create ' and action_name not in skip_actions :
2023-03-08 14:05:06 -08:00
yield from borgmatic . actions . create . run_create (
2023-03-08 13:19:41 -08:00
config_filename ,
repository ,
2023-07-08 23:14:30 -07:00
config ,
2024-01-09 13:47:20 -08:00
config_paths ,
2023-03-08 13:19:41 -08:00
hook_context ,
local_borg_version ,
action_arguments ,
global_arguments ,
dry_run_label ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' prune ' and action_name not in skip_actions :
2023-03-08 14:05:06 -08:00
borgmatic . actions . prune . run_prune (
2023-03-08 13:19:41 -08:00
config_filename ,
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
hook_context ,
local_borg_version ,
action_arguments ,
global_arguments ,
dry_run_label ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' compact ' and action_name not in skip_actions :
2023-03-08 14:05:06 -08:00
borgmatic . actions . compact . run_compact (
2023-03-08 13:19:41 -08:00
config_filename ,
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
hook_context ,
local_borg_version ,
action_arguments ,
global_arguments ,
dry_run_label ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' check ' and action_name not in skip_actions :
2023-07-08 23:14:30 -07:00
if checks . repository_enabled_for_checks ( repository , config ) :
2023-03-08 13:19:41 -08:00
borgmatic . actions . check . run_check (
config_filename ,
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
hook_context ,
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' extract ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
borgmatic . actions . extract . run_extract (
config_filename ,
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
hook_context ,
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' export-tar ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
borgmatic . actions . export_tar . run_export_tar (
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' mount ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
borgmatic . actions . mount . run_mount (
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
local_borg_version ,
2023-05-08 23:00:49 -07:00
action_arguments ,
global_arguments ,
2023-03-08 13:19:41 -08:00
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' restore ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
borgmatic . actions . restore . run_restore (
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2024-09-09 10:05:32 -07:00
elif action_name == ' repo-list ' and action_name not in skip_actions :
yield from borgmatic . actions . repo_list . run_repo_list (
2023-04-14 19:35:24 -07:00
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-04-14 19:35:24 -07:00
local_borg_version ,
action_arguments ,
2023-05-08 23:00:49 -07:00
global_arguments ,
2023-04-14 19:35:24 -07:00
local_path ,
remote_path ,
2023-03-08 13:19:41 -08:00
)
2023-10-31 21:54:41 -07:00
elif action_name == ' list ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
yield from borgmatic . actions . list . run_list (
2023-04-14 19:35:24 -07:00
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-04-14 19:35:24 -07:00
local_borg_version ,
action_arguments ,
2023-05-08 23:00:49 -07:00
global_arguments ,
2023-04-14 19:35:24 -07:00
local_path ,
remote_path ,
2023-03-08 13:19:41 -08:00
)
2024-09-09 10:05:32 -07:00
elif action_name == ' repo-info ' and action_name not in skip_actions :
yield from borgmatic . actions . repo_info . run_repo_info (
2023-04-14 19:35:24 -07:00
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-04-14 19:35:24 -07:00
local_borg_version ,
action_arguments ,
2023-05-08 23:00:49 -07:00
global_arguments ,
2023-04-14 19:35:24 -07:00
local_path ,
remote_path ,
2023-03-08 13:19:41 -08:00
)
2023-10-31 21:54:41 -07:00
elif action_name == ' info ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
yield from borgmatic . actions . info . run_info (
2023-04-14 19:35:24 -07:00
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-04-14 19:35:24 -07:00
local_borg_version ,
action_arguments ,
2023-05-08 23:00:49 -07:00
global_arguments ,
2023-04-14 19:35:24 -07:00
local_path ,
remote_path ,
2023-03-08 13:19:41 -08:00
)
2023-10-31 21:54:41 -07:00
elif action_name == ' break-lock ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
borgmatic . actions . break_lock . run_break_lock (
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-03-08 13:19:41 -08:00
local_borg_version ,
2023-05-08 23:00:49 -07:00
action_arguments ,
global_arguments ,
2023-03-08 13:19:41 -08:00
local_path ,
remote_path ,
2023-08-07 12:28:39 -07:00
)
2023-10-31 21:54:41 -07:00
elif action_name == ' export ' and action_name not in skip_actions :
2023-08-07 12:28:39 -07:00
borgmatic . actions . export_key . run_export_key (
repository ,
config ,
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
2023-03-08 13:19:41 -08:00
)
2024-09-01 11:13:39 -07:00
elif action_name == ' change-passphrase ' and action_name not in skip_actions :
borgmatic . actions . change_passphrase . run_change_passphrase (
repository ,
config ,
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2024-06-28 16:20:10 -07:00
elif action_name == ' delete ' and action_name not in skip_actions :
borgmatic . actions . delete . run_delete (
repository ,
config ,
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2024-09-09 10:05:32 -07:00
elif action_name == ' repo-delete ' and action_name not in skip_actions :
borgmatic . actions . repo_delete . run_repo_delete (
2024-06-28 16:20:10 -07:00
repository ,
config ,
local_borg_version ,
action_arguments ,
global_arguments ,
local_path ,
remote_path ,
)
2023-10-31 21:54:41 -07:00
elif action_name == ' borg ' and action_name not in skip_actions :
2023-03-08 13:19:41 -08:00
borgmatic . actions . borg . run_borg (
2023-04-14 19:35:24 -07:00
repository ,
2023-07-08 23:14:30 -07:00
config ,
2023-04-14 19:35:24 -07:00
local_borg_version ,
action_arguments ,
2023-05-08 23:00:49 -07:00
global_arguments ,
2023-04-14 19:35:24 -07:00
local_path ,
remote_path ,
2023-03-08 13:19:41 -08:00
)
2018-07-28 21:21:38 +00:00
2022-08-21 21:48:37 -07:00
command . execute_hook (
2023-07-08 23:14:30 -07:00
config . get ( ' after_actions ' ) ,
config . get ( ' umask ' ) ,
2022-08-21 21:48:37 -07:00
config_filename ,
' post-actions ' ,
global_arguments . dry_run ,
* * hook_context ,
)
2018-07-28 21:21:38 +00:00
2022-06-16 18:52:54 +02:00
def load_configurations ( config_filenames , overrides = None , resolve_env = True ) :
2018-12-25 15:23:54 -08:00
'''
2019-06-19 20:48:54 -07:00
Given a sequence of configuration filenames , load and validate each configuration file . Return
the results as a tuple of : dict of configuration filename to corresponding parsed configuration ,
2024-01-09 13:47:20 -08:00
a sequence of paths for all loaded configuration files ( including includes ) , and a sequence of
logging . LogRecord instances containing any parse errors .
2023-06-23 10:11:41 -07:00
Log records are returned here instead of being logged directly because logging isn ' t yet
initialized at this point !
2018-12-25 15:23:54 -08:00
'''
2019-02-18 09:30:34 -08:00
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections . OrderedDict ( )
2024-01-09 13:47:20 -08:00
config_paths = set ( )
2019-06-19 20:48:54 -07:00
logs = [ ]
2019-02-18 09:30:34 -08:00
2019-02-23 23:02:17 -08:00
# Parse and load each configuration file.
2018-12-25 15:23:54 -08:00
for config_filename in config_filenames :
2023-06-23 10:11:41 -07:00
logs . extend (
[
logging . makeLogRecord (
dict (
levelno = logging . DEBUG ,
levelname = ' DEBUG ' ,
msg = f ' { config_filename } : Loading configuration file ' ,
)
) ,
]
)
2018-12-25 15:23:54 -08:00
try :
2024-01-09 13:47:20 -08:00
configs [ config_filename ] , paths , parse_logs = validate . parse_configuration (
2022-06-16 18:52:54 +02:00
config_filename , validate . schema_filename ( ) , overrides , resolve_env
2019-02-18 09:30:34 -08:00
)
2024-01-09 13:47:20 -08:00
config_paths . update ( paths )
2022-08-17 10:13:11 -07:00
logs . extend ( parse_logs )
2022-03-08 10:19:36 -08:00
except PermissionError :
logs . extend (
[
logging . makeLogRecord (
dict (
levelno = logging . WARNING ,
levelname = ' WARNING ' ,
2023-03-23 23:11:14 -07:00
msg = f ' { config_filename } : Insufficient permissions to read configuration file ' ,
2022-03-08 10:19:36 -08:00
)
) ,
]
)
2019-02-18 09:30:34 -08:00
except ( ValueError , OSError , validate . Validation_error ) as error :
2019-06-19 20:48:54 -07:00
logs . extend (
[
logging . makeLogRecord (
dict (
levelno = logging . CRITICAL ,
levelname = ' CRITICAL ' ,
2023-03-23 23:11:14 -07:00
msg = f ' { config_filename } : Error parsing configuration file ' ,
2019-06-19 20:48:54 -07:00
)
) ,
logging . makeLogRecord (
2024-07-14 10:07:15 -07:00
dict ( levelno = logging . CRITICAL , levelname = ' CRITICAL ' , msg = str ( error ) )
2019-06-19 20:48:54 -07:00
) ,
]
2019-05-26 16:34:46 -07:00
)
2019-02-18 09:30:34 -08:00
2024-01-09 13:47:20 -08:00
return ( configs , sorted ( config_paths ) , logs )
2019-06-19 20:48:54 -07:00
2023-07-10 11:16:18 -07:00
def log_record ( suppress_log = False , * * kwargs ) :
2019-11-17 16:54:27 -08:00
'''
Create a log record based on the given makeLogRecord ( ) arguments , one of which must be
2023-07-10 11:16:18 -07:00
named " levelno " . Log the record ( unless suppress log is set ) and return it .
2019-11-17 16:54:27 -08:00
'''
record = logging . makeLogRecord ( kwargs )
2023-07-10 11:16:18 -07:00
if suppress_log :
return record
2023-07-10 10:20:51 -07:00
2023-07-10 11:16:18 -07:00
logger . handle ( record )
2019-11-17 16:54:27 -08:00
return record
2024-10-03 16:48:34 -07:00
BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE = 62
2023-07-10 11:16:18 -07:00
def log_error_records (
message , error = None , levelno = logging . CRITICAL , log_command_error_output = False
) :
2019-09-28 16:18:10 -07:00
'''
2023-07-10 11:16:18 -07:00
Given error message text , an optional exception object , an optional log level , and whether to
log the error output of a CalledProcessError ( if any ) , log error summary information and also
yield it as a series of logging . LogRecord instances .
2022-04-02 22:28:41 -07:00
Note that because the logs are yielded as a generator , logs won ' t get logged unless you consume
the generator output .
2019-09-28 16:18:10 -07:00
'''
2022-04-02 22:28:41 -07:00
level_name = logging . _levelToName [ levelno ]
2019-09-30 22:19:31 -07:00
if not error :
2024-07-15 14:45:21 -07:00
yield log_record ( levelno = levelno , levelname = level_name , msg = str ( message ) )
2019-09-30 22:19:31 -07:00
return
2019-09-28 16:18:10 -07:00
try :
raise error
except CalledProcessError as error :
2024-07-15 14:45:21 -07:00
yield log_record ( levelno = levelno , levelname = level_name , msg = str ( message ) )
2023-09-14 21:10:52 -07:00
2019-10-31 12:57:36 -07:00
if error . output :
2023-07-10 11:16:18 -07:00
try :
output = error . output . decode ( ' utf-8 ' )
except ( UnicodeDecodeError , AttributeError ) :
output = error . output
2023-09-14 21:10:52 -07:00
# Suppress these logs for now and save the error output for the log summary at the end.
# Log a separate record per line, as some errors can be really verbose and overflow the
# per-record size limits imposed by some logging backends.
for output_line in output . splitlines ( ) :
yield log_record (
levelno = levelno ,
levelname = level_name ,
msg = output_line ,
suppress_log = True ,
)
2024-07-15 14:45:21 -07:00
yield log_record ( levelno = levelno , levelname = level_name , msg = str ( error ) )
2024-10-03 16:48:34 -07:00
if error . returncode == BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE :
yield log_record (
levelno = levelno ,
levelname = level_name ,
msg = ' \n To work around this, set either the " relocated_repo_access_is_ok " or " unknown_unencrypted_repo_access_is_ok " option to " true " , as appropriate. ' ,
)
2019-09-28 16:18:10 -07:00
except ( ValueError , OSError ) as error :
2024-07-15 14:45:21 -07:00
yield log_record ( levelno = levelno , levelname = level_name , msg = str ( message ) )
yield log_record ( levelno = levelno , levelname = level_name , msg = str ( error ) )
2019-09-28 16:18:10 -07:00
except : # noqa: E722
# Raising above only as a means of determining the error type. Swallow the exception here
# because we don't want the exception to propagate out of this function.
pass
2019-12-07 21:36:51 -08:00
def get_local_path ( configs ) :
'''
Arbitrarily return the local path from the first configuration dict . Default to " borg " if not
set .
'''
2023-07-08 23:14:30 -07:00
return next ( iter ( configs . values ( ) ) ) . get ( ' local_path ' , ' borg ' )
2019-12-07 21:36:51 -08:00
2023-06-23 10:11:41 -07:00
def collect_highlander_action_summary_logs ( configs , arguments , configuration_parse_errors ) :
2019-06-19 20:48:54 -07:00
'''
2023-06-23 10:11:41 -07:00
Given a dict of configuration filename to corresponding parsed configuration , parsed
command - line arguments as a dict from subparser name to a parsed namespace of arguments , and
whether any configuration files encountered errors during parsing , run a highlander action
specified in the arguments , if any , and yield a series of logging . LogRecord instances containing
summary information .
2019-06-19 20:48:54 -07:00
2023-06-21 12:19:49 -07:00
A highlander action is an action that cannot coexist with other actions on the borgmatic
command - line , and borgmatic exits after processing such an action .
2019-06-19 20:48:54 -07:00
'''
2023-07-08 23:14:30 -07:00
add_custom_log_levels ( )
2023-05-28 01:36:32 +05:30
if ' bootstrap ' in arguments :
2023-06-21 12:19:49 -07:00
try :
# No configuration file is needed for bootstrap.
local_borg_version = borg_version . local_borg_version ( { } , ' borg ' )
except ( OSError , CalledProcessError , ValueError ) as error :
yield from log_error_records ( ' Error getting local Borg version ' , error )
return
2023-06-02 02:04:35 +05:30
try :
2023-06-06 23:37:09 +05:30
borgmatic . actions . config . bootstrap . run_bootstrap (
arguments [ ' bootstrap ' ] , arguments [ ' global ' ] , local_borg_version
)
2023-06-02 02:04:35 +05:30
yield logging . makeLogRecord (
dict (
2023-06-19 16:18:47 -07:00
levelno = logging . ANSWER ,
2023-06-21 12:19:49 -07:00
levelname = ' ANSWER ' ,
2023-06-02 02:04:35 +05:30
msg = ' Bootstrap successful ' ,
)
2023-05-28 01:36:32 +05:30
)
2023-06-07 01:47:16 +05:30
except (
CalledProcessError ,
ValueError ,
OSError ,
2023-06-08 00:08:39 +05:30
) as error :
2023-06-10 15:50:11 -07:00
yield from log_error_records ( error )
2023-06-09 17:31:57 +05:30
2023-05-28 01:36:32 +05:30
return
2019-02-18 09:30:34 -08:00
2023-06-21 12:19:49 -07:00
if ' generate ' in arguments :
try :
borgmatic . actions . config . generate . run_generate (
arguments [ ' generate ' ] , arguments [ ' global ' ]
)
yield logging . makeLogRecord (
dict (
levelno = logging . ANSWER ,
levelname = ' ANSWER ' ,
msg = ' Generate successful ' ,
)
)
except (
CalledProcessError ,
ValueError ,
OSError ,
) as error :
yield from log_error_records ( error )
return
2023-06-23 10:11:41 -07:00
if ' validate ' in arguments :
if configuration_parse_errors :
yield logging . makeLogRecord (
dict (
levelno = logging . CRITICAL ,
levelname = ' CRITICAL ' ,
msg = ' Configuration validation failed ' ,
)
)
return
try :
borgmatic . actions . config . validate . run_validate ( arguments [ ' validate ' ] , configs )
yield logging . makeLogRecord (
dict (
levelno = logging . ANSWER ,
levelname = ' ANSWER ' ,
msg = ' All configuration files are valid ' ,
)
)
except (
CalledProcessError ,
ValueError ,
OSError ,
) as error :
yield from log_error_records ( error )
return
2023-06-21 12:19:49 -07:00
2024-01-09 13:47:20 -08:00
def collect_configuration_run_summary_logs ( configs , config_paths , arguments ) :
2023-06-21 12:19:49 -07:00
'''
2024-01-09 13:47:20 -08:00
Given a dict of configuration filename to corresponding parsed configuration , a sequence of
loaded configuration paths , and parsed command - line arguments as a dict from subparser name to a
parsed namespace of arguments , run each configuration file and yield a series of
logging . LogRecord instances containing summary information about each run .
2023-06-21 12:19:49 -07:00
As a side effect of running through these configuration files , output their JSON results , if
any , to stdout .
'''
2023-06-15 21:45:43 -07:00
# Run cross-file validation checks.
repository = None
for action_name , action_arguments in arguments . items ( ) :
if hasattr ( action_arguments , ' repository ' ) :
repository = getattr ( action_arguments , ' repository ' )
break
try :
if ' extract ' in arguments or ' mount ' in arguments :
validate . guard_single_repository_selected ( repository , configs )
validate . guard_configuration_contains_repository ( repository , configs )
except ValueError as error :
yield from log_error_records ( str ( error ) )
return
2019-09-28 16:18:10 -07:00
if not configs :
2022-04-02 22:28:41 -07:00
yield from log_error_records (
2023-03-24 23:47:05 -07:00
f " { ' ' . join ( arguments [ ' global ' ] . config_paths ) } : No valid configuration files found " ,
2019-09-28 16:18:10 -07:00
)
return
2019-09-30 22:19:31 -07:00
if ' create ' in arguments :
try :
2019-09-28 16:18:10 -07:00
for config_filename , config in configs . items ( ) :
2019-10-21 15:52:14 -07:00
command . execute_hook (
2023-07-08 23:14:30 -07:00
config . get ( ' before_everything ' ) ,
config . get ( ' umask ' ) ,
2019-09-28 16:18:10 -07:00
config_filename ,
' pre-everything ' ,
arguments [ ' global ' ] . dry_run ,
)
2019-09-30 22:19:31 -07:00
except ( CalledProcessError , ValueError , OSError ) as error :
2022-04-02 22:28:41 -07:00
yield from log_error_records ( ' Error running pre-everything hook ' , error )
2019-09-30 22:19:31 -07:00
return
2019-09-28 16:18:10 -07:00
2019-02-23 23:02:17 -08:00
# Execute the actions corresponding to each configuration file.
2019-04-02 22:30:14 -07:00
json_results = [ ]
2019-02-18 09:30:34 -08:00
for config_filename , config in configs . items ( ) :
2024-01-09 13:47:20 -08:00
results = list ( run_configuration ( config_filename , config , config_paths , arguments ) )
2019-09-30 22:19:31 -07:00
error_logs = tuple ( result for result in results if isinstance ( result , logging . LogRecord ) )
if error_logs :
2023-03-23 23:11:14 -07:00
yield from log_error_records ( f ' { config_filename } : An error occurred ' )
2019-09-30 22:19:31 -07:00
yield from error_logs
else :
2018-12-25 15:23:54 -08:00
yield logging . makeLogRecord (
dict (
levelno = logging . INFO ,
2019-05-26 16:34:46 -07:00
levelname = ' INFO ' ,
2023-03-23 23:11:14 -07:00
msg = f ' { config_filename } : Successfully ran configuration file ' ,
2018-12-25 15:23:54 -08:00
)
)
2019-09-30 22:19:31 -07:00
if results :
json_results . extend ( results )
2018-12-25 15:23:54 -08:00
2019-12-07 21:36:51 -08:00
if ' umount ' in arguments :
2023-03-23 23:11:14 -07:00
logger . info ( f " Unmounting mount point { arguments [ ' umount ' ] . mount_point } " )
2019-12-07 21:36:51 -08:00
try :
borg_umount . unmount_archive (
2024-01-21 11:34:40 -08:00
config ,
2023-04-14 19:35:24 -07:00
mount_point = arguments [ ' umount ' ] . mount_point ,
local_path = get_local_path ( configs ) ,
2019-12-07 21:36:51 -08:00
)
except ( CalledProcessError , OSError ) as error :
2022-04-02 22:28:41 -07:00
yield from log_error_records ( ' Error unmounting mount point ' , error )
2019-12-07 21:36:51 -08:00
2019-04-02 22:30:14 -07:00
if json_results :
sys . stdout . write ( json . dumps ( json_results ) )
2019-09-30 22:19:31 -07:00
if ' create ' in arguments :
try :
2019-09-28 16:18:10 -07:00
for config_filename , config in configs . items ( ) :
2019-10-21 15:52:14 -07:00
command . execute_hook (
2023-07-08 23:14:30 -07:00
config . get ( ' after_everything ' ) ,
config . get ( ' umask ' ) ,
2019-09-28 16:18:10 -07:00
config_filename ,
' post-everything ' ,
arguments [ ' global ' ] . dry_run ,
)
2019-09-30 22:19:31 -07:00
except ( CalledProcessError , ValueError , OSError ) as error :
2022-04-02 22:28:41 -07:00
yield from log_error_records ( ' Error running post-everything hook ' , error )
2018-12-25 15:23:54 -08:00
2019-01-27 11:58:04 -08:00
def exit_with_help_link ( ) : # pragma: no cover
'''
Display a link to get help and exit with an error code .
'''
2019-05-27 15:05:26 -07:00
logger . critical ( ' ' )
logger . critical ( ' Need some help? https://torsion.org/borgmatic/#issues ' )
2019-01-27 11:58:04 -08:00
sys . exit ( 1 )
2023-06-21 12:19:49 -07:00
def main ( extra_summary_logs = [ ] ) : # pragma: no cover
2018-12-25 15:23:54 -08:00
configure_signals ( )
2019-01-27 11:58:04 -08:00
try :
2019-06-22 16:10:07 -07:00
arguments = parse_arguments ( * sys . argv [ 1 : ] )
2019-01-27 11:58:04 -08:00
except ValueError as error :
2019-05-27 15:05:26 -07:00
configure_logging ( logging . CRITICAL )
2019-01-27 11:58:04 -08:00
logger . critical ( error )
exit_with_help_link ( )
2019-05-27 15:44:48 -07:00
except SystemExit as error :
if error . code == 0 :
raise error
2019-05-27 15:05:26 -07:00
configure_logging ( logging . CRITICAL )
2023-03-23 23:11:14 -07:00
logger . critical ( f " Error parsing arguments: { ' ' . join ( sys . argv ) } " )
2019-05-27 15:05:26 -07:00
exit_with_help_link ( )
2019-01-27 11:58:04 -08:00
2019-06-22 16:10:07 -07:00
global_arguments = arguments [ ' global ' ]
if global_arguments . version :
2023-11-07 10:17:16 -08:00
print ( importlib . metadata . version ( ' borgmatic ' ) )
2018-12-25 21:01:08 -08:00
sys . exit ( 0 )
2022-05-26 10:27:53 -07:00
if global_arguments . bash_completion :
2023-06-15 10:55:31 -07:00
print ( borgmatic . commands . completion . bash . bash_completion ( ) )
2022-05-26 10:27:53 -07:00
sys . exit ( 0 )
2023-04-27 18:46:13 -07:00
if global_arguments . fish_completion :
2023-06-15 10:55:31 -07:00
print ( borgmatic . commands . completion . fish . fish_completion ( ) )
2023-04-27 18:46:13 -07:00
sys . exit ( 0 )
2018-12-25 21:01:08 -08:00
2019-06-22 16:10:07 -07:00
config_filenames = tuple ( collect . collect_config_filenames ( global_arguments . config_paths ) )
2024-01-09 13:47:20 -08:00
configs , config_paths , parse_logs = load_configurations (
2022-06-16 18:52:54 +02:00
config_filenames , global_arguments . overrides , global_arguments . resolve_env
)
2023-06-23 10:11:41 -07:00
configuration_parse_errors = (
( max ( log . levelno for log in parse_logs ) > = logging . CRITICAL ) if parse_logs else False
)
2019-06-19 20:48:54 -07:00
2020-01-04 15:50:41 -08:00
any_json_flags = any (
getattr ( sub_arguments , ' json ' , False ) for sub_arguments in arguments . values ( )
)
2024-02-11 17:44:43 -08:00
color_enabled = should_do_markup ( global_arguments . no_color or any_json_flags , configs )
colorama . init ( autoreset = color_enabled , strip = not color_enabled )
2019-11-02 11:23:18 -07:00
try :
configure_logging (
verbosity_to_log_level ( global_arguments . verbosity ) ,
verbosity_to_log_level ( global_arguments . syslog_verbosity ) ,
verbosity_to_log_level ( global_arguments . log_file_verbosity ) ,
2020-01-22 15:10:47 -08:00
verbosity_to_log_level ( global_arguments . monitoring_verbosity ) ,
2019-11-02 11:23:18 -07:00
global_arguments . log_file ,
2023-04-02 23:06:36 -07:00
global_arguments . log_file_format ,
2024-02-11 17:44:43 -08:00
color_enabled = color_enabled ,
2019-11-02 11:23:18 -07:00
)
except ( FileNotFoundError , PermissionError ) as error :
configure_logging ( logging . CRITICAL )
2023-03-23 23:11:14 -07:00
logger . critical ( f ' Error configuring logging: { error } ' )
2019-11-02 11:23:18 -07:00
exit_with_help_link ( )
2019-06-19 20:48:54 -07:00
2023-06-21 12:19:49 -07:00
summary_logs = (
2023-06-23 10:11:41 -07:00
extra_summary_logs
+ parse_logs
2023-06-21 12:19:49 -07:00
+ (
2023-06-23 10:11:41 -07:00
list (
collect_highlander_action_summary_logs (
configs , arguments , configuration_parse_errors
)
)
2024-01-09 13:47:20 -08:00
or list ( collect_configuration_run_summary_logs ( configs , config_paths , arguments ) )
2023-06-21 12:19:49 -07:00
)
)
2019-11-25 10:31:09 -08:00
summary_logs_max_level = max ( log . levelno for log in summary_logs )
2017-07-22 22:56:46 -07:00
2019-11-25 10:31:09 -08:00
for message in ( ' ' , ' summary: ' ) :
log_record (
levelno = summary_logs_max_level ,
levelname = logging . getLevelName ( summary_logs_max_level ) ,
msg = message ,
)
for log in summary_logs :
logger . handle ( log )
2018-12-25 15:23:54 -08:00
2019-11-25 10:31:09 -08:00
if summary_logs_max_level > = logging . CRITICAL :
2019-01-27 11:58:04 -08:00
exit_with_help_link ( )