Browse Source

Basic YAML configuration file parsing.

tags/1.1.0
Dan Helfman 2 years ago
parent
commit
4d7556f68b

+ 1
- 0
.hgignore View File

@@ -2,6 +2,7 @@ syntax: glob
*.egg-info
*.pyc
*.swp
.cache
.tox
build
dist

+ 1
- 0
MANIFEST.in View File

@@ -0,0 +1 @@
include borgmatic/config/schema.yaml

+ 1
- 1
NEWS View File

@@ -1,4 +1,4 @@
1.0.3-dev
1.1.0

* #18: Fix for README mention of sample files not included in package.
* #22: Sample files for triggering borgmatic from a systemd timer.

+ 1
- 1
borgmatic/command.py View File

@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
import sys

from borgmatic import borg
from borgmatic.config import parse_configuration, CONFIG_FORMAT
from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT


DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'

+ 0
- 0
borgmatic/config/__init__.py View File


borgmatic/config.py → borgmatic/config/legacy.py View File


+ 48
- 0
borgmatic/config/schema.yaml View File

@@ -0,0 +1,48 @@
map:
location:
required: True
map:
source_directories:
required: True
seq:
- type: scalar
one_file_system:
type: bool
remote_path:
type: scalar
repository:
required: True
type: scalar
storage:
map:
encryption_passphrase:
type: scalar
compression:
type: scalar
umask:
type: scalar
retention:
map:
keep_within:
type: scalar
keep_hourly:
type: int
keep_daily:
type: int
keep_weekly:
type: int
keep_monthly:
type: int
keep_yearly:
type: int
prefix:
type: scalar
consistency:
map:
checks:
seq:
- type: str
enum: ['repository', 'archives', 'disabled']
unique: True
check_last:
type: int

+ 84
- 0
borgmatic/config/yaml.py View File

@@ -0,0 +1,84 @@
import logging
import sys
import warnings

import pkg_resources
import pykwalify.core
import pykwalify.errors
import ruamel.yaml.error


def schema_filename():
'''
Path to the installed YAML configuration schema file, used to validate and parse the
configuration.
'''
return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')


class Validation_error(ValueError):
'''
A collection of error message strings generated when attempting to validate a particular
configurartion file.
'''
def __init__(self, config_filename, error_messages):
self.config_filename = config_filename
self.error_messages = error_messages


def parse_configuration(config_filename, schema_filename):
'''
Given the path to a config filename in YAML format and the path to a schema filename in
pykwalify YAML schema format, return the parsed configuration as a data structure of nested
dicts and lists corresponding to the schema. Example return value:

{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}

Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
have permissions to read the file, or Validation_error if the config does not match the schema.
'''
warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
logging.getLogger('pykwalify').setLevel(logging.CRITICAL)

try:
validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
except pykwalify.errors.CoreError as error:
if 'do not exists on disk' in str(error):
raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
if 'Unable to load any data' in str(error):
# If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
# So reach back to the originating exception from ruamel.yaml for something more useful.
raise Validation_error(config_filename, (error.__context__,))
raise

parsed_result = validator.validate(raise_exception=False)

if validator.validation_errors:
raise Validation_error(config_filename, validator.validation_errors)

return parsed_result


def display_validation_error(validation_error):
'''
Given a Validation_error, display its error messages to stderr.
'''
print(
'An error occurred while parsing a configuration file at {}:'.format(
validation_error.config_filename
),
file=sys.stderr,
)

for error in validation_error.error_messages:
print(error, file=sys.stderr)


# FOR TESTING
if __name__ == '__main__':
try:
configuration = parse_configuration('sample/config.yaml', schema_filename())
print(configuration)
except Validation_error as error:
display_validation_error(error)

+ 0
- 6
borgmatic/tests/builtins.py View File

@@ -1,6 +0,0 @@
from flexmock import flexmock
import sys


def builtins_mock():
return flexmock(sys.modules['builtins'])

+ 0
- 0
borgmatic/tests/integration/config/__init__.py View File


borgmatic/tests/integration/test_config.py → borgmatic/tests/integration/config/test_legacy.py View File

@@ -3,7 +3,7 @@ from io import StringIO
from collections import OrderedDict
import string

from borgmatic import config as module
from borgmatic.config import legacy as module


def test_parse_section_options_with_punctuation_should_return_section_options():

+ 0
- 0
borgmatic/tests/unit/config/__init__.py View File


borgmatic/tests/unit/test_config.py → borgmatic/tests/unit/config/test_legacy.py View File

@@ -3,7 +3,7 @@ from collections import OrderedDict
from flexmock import flexmock
import pytest

from borgmatic import config as module
from borgmatic.config import legacy as module


def test_option_should_create_config_option():

+ 3
- 3
borgmatic/tests/unit/test_borg.py View File

@@ -1,11 +1,11 @@
from collections import OrderedDict
from subprocess import STDOUT
import sys
import os

from flexmock import flexmock

from borgmatic import borg as module
from borgmatic.tests.builtins import builtins_mock
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS


@@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters():
)
insert_platform_mock()
insert_datetime_mock()
builtins_mock().should_receive('open').and_return(stdout)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')

module.check_archives(
@@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
)
insert_platform_mock()
insert_datetime_mock()
builtins_mock().should_receive('open').and_return(stdout)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')

module.check_archives(

+ 54
- 0
sample/config.yaml View File

@@ -0,0 +1,54 @@
location:
# List of source directories to backup. Globs are expanded.
source_directories:
- /home
- /etc
- /var/log/syslog*

# Stay in same file system (do not cross mount points).
#one_file_system: yes

# Alternate Borg remote executable (defaults to "borg"):
#remote_path: borg1

# Path to local or remote repository.
repository: user@backupserver:sourcehostname.borg

#storage:
# Passphrase to unlock the encryption key with. Only use on repositories
# that were initialized with passphrase/repokey encryption.
#encryption_passphrase: foo

# Type of compression to use when creating archives. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
# for details. Defaults to no compression.
#compression: lz4

# Umask to be used for borg create.
#umask: 0077

retention:
# Retention policy for how many backups to keep in each category. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
# details.
#keep_within: 3H
#keep_hourly: 24
keep_daily: 7
keep_weekly: 4
keep_monthly: 6
keep_yearly: 1

# When pruning, only consider archive names starting with this prefix.
#prefix: sourcehostname

consistency:
# List of consistency checks to run: "repository", "archives", or both.
# Defaults to both. Set to "disabled" to disable all consistency checks.
# See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
# for details.
checks:
- repository
- archives

# Restrict the number of checked archives to the last n.
#check_last: 3

+ 8
- 2
setup.py View File

@@ -1,7 +1,7 @@
from setuptools import setup, find_packages


VERSION = '1.0.3-dev'
VERSION = '1.1.0'


setup(
@@ -30,8 +30,14 @@ setup(
obsoletes=[
'atticmatic',
],
install_requires=(
'pykwalify',
'ruamel.yaml<=0.15',
'setuptools',
),
tests_require=(
'flexmock',
'pytest',
)
),
include_package_data=True,
)

Loading…
Cancel
Save