Browse Source

Basic YAML generating / validating / converting to.

tags/1.1.0
Dan Helfman 3 years ago
parent
commit
f19a40ef9c
16 changed files with 327 additions and 40 deletions
  1. +3
    -2
      NEWS
  2. +0
    -0
     
  3. +3
    -4
      borgmatic/commands/borgmatic.py
  4. +54
    -0
      borgmatic/commands/convert_config.py
  5. +41
    -0
      borgmatic/config/convert.py
  6. +90
    -0
      borgmatic/config/generate.py
  7. +62
    -0
      borgmatic/config/schema.yaml
  8. +11
    -22
      borgmatic/config/validate.py
  9. +0
    -0
     
  10. +1
    -1
      borgmatic/tests/integration/commands/test_borgmatic.py
  11. +7
    -6
      borgmatic/tests/integration/config/test_validate.py
  12. +44
    -0
      borgmatic/tests/unit/config/test_convert.py
  13. +4
    -2
      sample/config.yaml
  14. +3
    -2
      setup.py
  15. +3
    -0
      test_requirements.txt
  16. +1
    -1
      tox.ini

+ 3
- 2
NEWS View File

@@ -1,8 +1,9 @@
1.1.0
1.1.0.dev0

* Switched config file format to YAML. Run convert-borgmatic-config to upgrade.
* Dropped Python 2 support. Now Python 3 only.
* #18: Fix for README mention of sample files not included in package.
* #22: Sample files for triggering borgmatic from a systemd timer.
* Dropped Python 2 support. Now Python 3 only.
* Added logo.

1.0.3


+ 0
- 0
View File


borgmatic/command.py → borgmatic/commands/borgmatic.py View File

@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
import sys

from borgmatic import borg
from borgmatic.config.yaml import parse_configuration, schema_filename
from borgmatic.config.validate import parse_configuration, schema_filename


DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@@ -14,9 +14,8 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'

def parse_arguments(*arguments):
'''
Given the name of the command with which this script was invoked and command-line arguments,
parse the arguments and return them as an ArgumentParser instance. Use the command name to
determine the default configuration and excludes paths.
Given command-line arguments with which this script was invoked, parse the arguments and return
them as an ArgumentParser instance.
'''
parser = ArgumentParser()
parser.add_argument(

+ 54
- 0
borgmatic/commands/convert_config.py View File

@@ -0,0 +1,54 @@
from __future__ import print_function
from argparse import ArgumentParser
import os
from subprocess import CalledProcessError
import sys

from ruamel import yaml

from borgmatic import borg
from borgmatic.config import convert, generate, legacy, validate


DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'


def parse_arguments(*arguments):
'''
Given command-line arguments with which this script was invoked, parse the arguments and return
them as an ArgumentParser instance.
'''
parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
parser.add_argument(
'-s', '--source',
dest='source_filename',
default=DEFAULT_SOURCE_CONFIG_FILENAME,
help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
)
parser.add_argument(
'-d', '--destination',
dest='destination_filename',
default=DEFAULT_DESTINATION_CONFIG_FILENAME,
help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
)

return parser.parse_args(arguments)


def main():
try:
args = parse_arguments(*sys.argv[1:])
source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
schema = yaml.round_trip_load(open(validate.schema_filename()).read())

destination_config = convert.convert_legacy_parsed_config(source_config, schema)

generate.write_configuration(args.destination_filename, destination_config)

# TODO: As a backstop, check that the written config can actually be read and parsed, and
# that it matches the destination config data structure that was written.
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)

+ 41
- 0
borgmatic/config/convert.py View File

@@ -0,0 +1,41 @@
from ruamel import yaml

from borgmatic.config import generate


def _convert_section(source_section_config, section_schema):
'''
Given a legacy Parsed_config instance for a single section, convert it to its corresponding
yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.

Additionally, use the section schema as a source of helpful comments to include within the
returned CommentedMap.
'''
destination_section_config = yaml.comments.CommentedMap(source_section_config)
generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)

return destination_section_config


def convert_legacy_parsed_config(source_config, schema):
'''
Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
to YAML.

Additionally, use the given schema as a source of helpful comments to include within the
returned CommentedMap.
'''
destination_config = yaml.comments.CommentedMap([
(section_name, _convert_section(section_config, schema['map'][section_name]))
for section_name, section_config in source_config._asdict().items()
])

destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')

if source_config.consistency['checks']:
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')

generate.add_comments_to_configuration(destination_config, schema)

return destination_config

+ 90
- 0
borgmatic/config/generate.py View File

@@ -0,0 +1,90 @@
from collections import OrderedDict

from ruamel import yaml


INDENT = 4


def write_configuration(config_filename, config):
'''
Given a target config filename and a config data structure of nested OrderedDicts, write out the
config to file as YAML.
'''
with open(config_filename, 'w') as config_file:
config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))


def _insert_newline_before_comment(config, field_name):
'''
Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
field and its comments.
'''
config.ca.items[field_name][1].insert(
0,
yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
)


def add_comments_to_configuration(config, schema, indent=0):
'''
Using descriptions from a schema as a source, add those descriptions as comments to the given
config before each field. This function only adds comments for the top-most config map level.
Indent the comment the given number of characters.
'''
for index, field_name in enumerate(config.keys()):
field_schema = schema['map'].get(field_name, {})
description = field_schema.get('desc')

# No description to use? Skip it.
if not schema or not description:
continue

config.yaml_set_comment_before_after_key(
key=field_name,
before=description,
indent=indent,
)
if index > 0:
_insert_newline_before_comment(config, field_name)


def _section_schema_to_sample_configuration(section_schema):
'''
Given the schema for a particular config section, generate and return sample config for that
section. Include comments for each field based on the schema "desc" description.
'''
section_config = yaml.comments.CommentedMap([
(field_name, field_schema['example'])
for field_name, field_schema in section_schema['map'].items()
])

add_comments_to_configuration(section_config, section_schema, indent=INDENT)

return section_config


def _schema_to_sample_configuration(schema):
'''
Given a loaded configuration schema, generate and return sample config for it. Include comments
for each section based on the schema "desc" description.
'''
config = yaml.comments.CommentedMap([
(section_name, _section_schema_to_sample_configuration(section_schema))
for section_name, section_schema in schema['map'].items()
])

add_comments_to_configuration(config, schema)

return config


def generate_sample_configuration(config_filename, schema_filename):
'''
Given a target config filename and the path to a schema filename in pykwalify YAML schema
format, write out a sample configuration file based on that schema.
'''
schema = yaml.round_trip_load(open(schema_filename))
config = _schema_to_sample_configuration(schema)

write_configuration(config_filename, config)

+ 62
- 0
borgmatic/config/schema.yaml View File

@@ -1,48 +1,110 @@
name: Borgmatic configuration file schema
map:
location:
desc: |
Where to look for files to backup, and where to store those backups. See
https://borgbackup.readthedocs.io/en/stable/quickstart.html and
https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
required: True
map:
source_directories:
required: True
seq:
- type: scalar
desc: List of source directories to backup. Globs are expanded.
example:
- /home
- /etc
- /var/log/syslog*
one_file_system:
type: bool
desc: Stay in same file system (do not cross mount points).
example: yes
remote_path:
type: scalar
desc: Alternate Borg remote executable. Defaults to "borg".
example: borg1
repository:
required: True
type: scalar
desc: Path to local or remote repository.
example: user@backupserver:sourcehostname.borg
storage:
desc: |
Repository storage options. See
https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details.
map:
encryption_passphrase:
type: scalar
desc: |
Passphrase to unlock the encryption key with. Only use on repositories that were
initialized with passphrase/repokey encryption. Quote the value if it contains
punctuation, so it parses correctly. And backslash any quote or backslash
literals as well.
example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
compression:
type: scalar
desc: |
Type of compression to use when creating archives. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
Defaults to no compression.
example: lz4
umask:
type: scalar
desc: Umask to be used for borg create.
example: 0077
retention:
desc: |
Retention policy for how many backups to keep in each category. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
map:
keep_within:
type: scalar
desc: Keep all archives within this time interval.
example: 3H
keep_hourly:
type: int
desc: Number of hourly archives to keep.
example: 24
keep_daily:
type: int
desc: Number of daily archives to keep.
example: 7
keep_weekly:
type: int
desc: Number of weekly archives to keep.
example: 4
keep_monthly:
type: int
desc: Number of monthly archives to keep.
example: 6
keep_yearly:
type: int
desc: Number of yearly archives to keep.
example: 1
prefix:
type: scalar
desc: When pruning, only consider archive names starting with this prefix.
example: sourcehostname
consistency:
desc: |
Consistency checks to run after backups. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
map:
checks:
seq:
- type: str
enum: ['repository', 'archives', 'disabled']
unique: True
desc: |
List of consistency checks to run: "repository", "archives", or both. Defaults
to both. Set to "disabled" to disable all consistency checks. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
example:
- repository
- archives
check_last:
type: int
desc: Restrict the number of checked archives to the last n.
example: 3

borgmatic/config/yaml.py → borgmatic/config/validate.py View File

@@ -5,7 +5,7 @@ import warnings
import pkg_resources
import pykwalify.core
import pykwalify.errors
import ruamel.yaml.error
from ruamel import yaml


def schema_filename():
@@ -38,20 +38,18 @@ def parse_configuration(config_filename, schema_filename):
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
have permissions to read the file, or Validation_error if the config does not match the schema.
'''
warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
logging.getLogger('pykwalify').setLevel(logging.CRITICAL)

try:
validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
except pykwalify.errors.CoreError as error:
if 'do not exists on disk' in str(error):
raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
if 'Unable to load any data' in str(error):
# If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
# So reach back to the originating exception from ruamel.yaml for something more useful.
raise Validation_error(config_filename, (error.__context__,))
raise
schema = yaml.round_trip_load(open(schema_filename))
except yaml.error.YAMLError as error:
raise Validation_error(config_filename, (str(error),))
# pykwalify gets angry if the example field is not a string. So rather than bend to its will,
# simply remove all examples before passing the schema to pykwalify.
for section_name, section_schema in schema['map'].items():
for field_name, field_schema in section_schema['map'].items():
field_schema.pop('example')

validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema)
parsed_result = validator.validate(raise_exception=False)

if validator.validation_errors:
@@ -73,12 +71,3 @@ def display_validation_error(validation_error):

for error in validation_error.error_messages:
print(error, file=sys.stderr)


# FOR TESTING
if __name__ == '__main__':
try:
configuration = parse_configuration('sample/config.yaml', schema_filename())
print(configuration)
except Validation_error as error:
display_validation_error(error)

+ 0
- 0
View File


borgmatic/tests/integration/test_command.py → borgmatic/tests/integration/commands/test_borgmatic.py View File

@@ -4,7 +4,7 @@ import sys
from flexmock import flexmock
import pytest

from borgmatic import command as module
from borgmatic.commands import borgmatic as module


def test_parse_arguments_with_no_arguments_uses_defaults():

borgmatic/tests/integration/config/test_yaml.py → borgmatic/tests/integration/config/test_validate.py View File

@@ -6,7 +6,7 @@ import os
from flexmock import flexmock
import pytest

from borgmatic.config import yaml as module
from borgmatic.config import validate as module


def test_schema_filename_returns_plausable_path():
@@ -18,13 +18,13 @@ def test_schema_filename_returns_plausable_path():
def mock_config_and_schema(config_yaml):
'''
Set up mocks for the config config YAML string and the default schema so that pykwalify consumes
them when parsing the configuration. This is a little brittle in that it's relying on pykwalify
to open() the respective files in a particular order.
them when parsing the configuration. This is a little brittle in that it's relying on the code
under test to open() the respective files in a particular order.
'''
config_stream = io.StringIO(config_yaml)
schema_stream = open(module.schema_filename())
config_stream = io.StringIO(config_yaml)
builtins = flexmock(sys.modules['builtins']).should_call('open').mock
builtins.should_receive('open').and_return(config_stream).and_return(schema_stream)
builtins.should_receive('open').and_return(schema_stream).and_return(config_stream)
flexmock(os.path).should_receive('exists').and_return(True)


@@ -87,7 +87,8 @@ def test_parse_configuration_raises_for_missing_config_file():

def test_parse_configuration_raises_for_missing_schema_file():
mock_config_and_schema('')
flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False)
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)

with pytest.raises(FileNotFoundError):
module.parse_configuration('config.yaml', 'schema.yaml')

+ 44
- 0
borgmatic/tests/unit/config/test_convert.py View File

@@ -0,0 +1,44 @@
from collections import defaultdict, OrderedDict, namedtuple

from borgmatic.config import convert as module


Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))


def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
source_config = Parsed_config(
location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
retention=OrderedDict([('keep_daily', 7)]),
consistency=OrderedDict([('checks', 'repository')]),
)
schema = {'map': defaultdict(lambda: {'map': {}})}

destination_config = module.convert_legacy_parsed_config(source_config, schema)

assert destination_config == OrderedDict([
('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])),
('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
('retention', OrderedDict([('keep_daily', 7)])),
('consistency', OrderedDict([('checks', ['repository'])])),
])


def test_convert_legacy_parsed_config_splits_space_separated_values():
source_config = Parsed_config(
location=OrderedDict([('source_directories', '/home /etc')]),
storage=OrderedDict(),
retention=OrderedDict(),
consistency=OrderedDict([('checks', 'repository archives')]),
)
schema = {'map': defaultdict(lambda: {'map': {}})}

destination_config = module.convert_legacy_parsed_config(source_config, schema)

assert destination_config == OrderedDict([
('location', OrderedDict([('source_directories', ['/home', '/etc'])])),
('storage', OrderedDict()),
('retention', OrderedDict()),
('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
])

+ 4
- 2
sample/config.yaml View File

@@ -16,8 +16,10 @@ location:

#storage:
# Passphrase to unlock the encryption key with. Only use on repositories
# that were initialized with passphrase/repokey encryption.
#encryption_passphrase: foo
# that were initialized with passphrase/repokey encryption. Quote the value
# if it contains punctuation so it parses correctly. And backslash any
# quote or backslash literals as well.
#encryption_passphrase: "foo"

# Type of compression to use when creating archives. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create


+ 3
- 2
setup.py View File

@@ -1,7 +1,7 @@
from setuptools import setup, find_packages


VERSION = '1.1.0'
VERSION = '1.1.0.dev0'


setup(
@@ -24,7 +24,8 @@ setup(
packages=find_packages(),
entry_points={
'console_scripts': [
'borgmatic = borgmatic.command:main',
'borgmatic = borgmatic.commands.borgmatic:main',
'convert-borgmatic-config = borgmatic.commands.convert_config:main',
]
},
obsoletes=[


+ 3
- 0
test_requirements.txt View File

@@ -1,2 +1,5 @@
flexmock==0.10.2
pykwalify==1.6.0
pytest==2.9.1
pytest-cov==2.5.1
ruamel.yaml==0.15.18

+ 1
- 1
tox.ini View File

@@ -5,4 +5,4 @@ skipsdist=True
[testenv]
usedevelop=True
deps=-rtest_requirements.txt
commands = py.test borgmatic []
commands = py.test --cov=borgmatic borgmatic []

Loading…
Cancel
Save