From 27d37b606b912e91e26ccf56d1e28a4210f38182 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 22 Jun 2021 13:27:59 -0700 Subject: [PATCH] Better error messages! Switch the library used for validating configuration files (from pykwalify to jsonschema). --- NEWS | 2 + borgmatic/config/convert.py | 10 +- borgmatic/config/generate.py | 42 +- borgmatic/config/schema.yaml | 517 ++++++++++++---------- borgmatic/config/validate.py | 78 ++-- borgmatic/logger.py | 1 + setup.py | 2 +- test_requirements.txt | 9 +- tests/integration/config/test_generate.py | 24 +- tests/unit/config/test_convert.py | 16 +- tests/unit/config/test_generate.py | 38 +- tests/unit/config/test_validate.py | 56 ++- 12 files changed, 434 insertions(+), 361 deletions(-) diff --git a/NEWS b/NEWS index adfc2f48a..1f58c70c4 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.5.16.dev0 + * Better error messages! Switch the library used for validating configuration files (from pykwalify + to jsonschema). * Link borgmatic Ansible role from installation documentation: https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 83384ea5a..e4e55e43b 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -17,7 +17,7 @@ def _convert_section(source_section_config, section_schema): ( option_name, int(option_value) - if section_schema['map'].get(option_name, {}).get('type') == 'int' + if section_schema['properties'].get(option_name, {}).get('type') == 'integer' else option_value, ) for option_name, option_value in source_section_config.items() @@ -38,7 +38,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): ''' destination_config = yaml.comments.CommentedMap( [ - (section_name, _convert_section(section_config, schema['map'][section_name])) + (section_name, _convert_section(section_config, schema['properties'][section_name])) for section_name, section_config in source_config._asdict().items() ] ) @@ -54,11 +54,11 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') # Add comments to each section, and then add comments to the fields in each section. - generate.add_comments_to_configuration_map(destination_config, schema) + generate.add_comments_to_configuration_object(destination_config, schema) for section_name, section_config in destination_config.items(): - generate.add_comments_to_configuration_map( - section_config, schema['map'][section_name], indent=generate.INDENT + generate.add_comments_to_configuration_object( + section_config, schema['properties'][section_name], indent=generate.INDENT ) return destination_config diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 82dadeaa7..375b61a89 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -24,29 +24,27 @@ def _insert_newline_before_comment(config, field_name): def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema, generate and return sample config for it. Include comments - for each section based on the schema "desc" description. + for each section based on the schema "description". ''' + schema_type = schema.get('type') example = schema.get('example') if example is not None: return example - if 'seq' in schema: + if schema_type == 'array': config = yaml.comments.CommentedSeq( - [ - _schema_to_sample_configuration(item_schema, level, parent_is_sequence=True) - for item_schema in schema['seq'] - ] + [_schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) - elif 'map' in schema: + elif schema_type == 'object': config = yaml.comments.CommentedMap( [ (field_name, _schema_to_sample_configuration(sub_schema, level + 1)) - for field_name, sub_schema in schema['map'].items() + for field_name, sub_schema in schema['properties'].items() ] ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) - add_comments_to_configuration_map( + add_comments_to_configuration_object( config, schema, indent=indent, skip_first=parent_is_sequence ) else: @@ -132,8 +130,8 @@ def write_configuration(config_filename, rendered_config, mode=0o600): def add_comments_to_configuration_sequence(config, schema, indent=0): ''' - If the given config sequence's items are maps, then mine the schema for the description of the - map's first item, and slap that atop the sequence. Indent the comment the given number of + If the given config sequence's items are object, then mine the schema for the description of the + object's first item, and slap that atop the sequence. Indent the comment the given number of characters. Doing this for sequences of maps results in nice comments that look like: @@ -142,16 +140,16 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): things: # First key description. Added by this function. - key: foo - # Second key description. Added by add_comments_to_configuration_map(). + # Second key description. Added by add_comments_to_configuration_object(). other: bar ``` ''' - if 'map' not in schema['seq'][0]: + if schema['items'].get('type') != 'object': return for field_name in config[0].keys(): - field_schema = schema['seq'][0]['map'].get(field_name, {}) - description = field_schema.get('desc') + field_schema = schema['items']['properties'].get(field_name, {}) + description = field_schema.get('description') # No description to use? Skip it. if not field_schema or not description: @@ -160,7 +158,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): config[0].yaml_set_start_comment(description, indent=indent) # We only want the first key's description here, as the rest of the keys get commented by - # add_comments_to_configuration_map(). + # add_comments_to_configuration_object(). return @@ -169,7 +167,7 @@ REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'} COMMENTED_OUT_SENTINEL = 'COMMENT_OUT' -def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False): +def add_comments_to_configuration_object(config, schema, indent=0, skip_first=False): ''' Using descriptions from a schema as a source, add those descriptions as comments to the given config mapping, before each field. Indent the comment the given number of characters. @@ -178,8 +176,8 @@ def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False if skip_first and index == 0: continue - field_schema = schema['map'].get(field_name, {}) - description = field_schema.get('desc', '').strip() + field_schema = schema['properties'].get(field_name, {}) + description = field_schema.get('description', '').strip() # If this is an optional key, add an indicator to the comment flagging it to be commented # out from the sample configuration. This sentinel is consumed by downstream processing that @@ -268,9 +266,9 @@ def merge_source_configuration_into_destination(destination_config, source_confi def generate_sample_configuration(source_filename, destination_filename, schema_filename): ''' Given an optional source configuration filename, and a required destination configuration - filename, and the path to a schema filename in pykwalify YAML schema format, write out a - sample configuration file based on that schema. If a source filename is provided, merge the - parsed contents of that configuration into the generated configuration. + filename, and the path to a schema filename in a YAML rendition of the JSON Schema format, + write out a sample configuration file based on that schema. If a source filename is provided, + merge the parsed contents of that configuration into the generated configuration. ''' schema = yaml.round_trip_load(open(schema_filename)) source_config = None diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index dcc470682..00df52e33 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1,19 +1,25 @@ -name: Borgmatic configuration file schema -version: 1 -map: +type: object +required: + - location +additionalProperties: false +properties: location: - desc: | + type: object + description: | Where to look for files to backup, and where to store those backups. See https://borgbackup.readthedocs.io/en/stable/quickstart.html and https://borgbackup.readthedocs.io/en/stable/usage/create.html for details. - required: true - map: + required: + - source_directories + - repositories + additionalProperties: false + properties: source_directories: - required: true - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of source directories to backup (required). Globs and tildes are expanded. Do not backslash spaces in path names. example: @@ -22,10 +28,10 @@ map: - /var/log/syslog* - /home/user/path with spaces repositories: - required: true - seq: - - type: str - desc: | + type: array + items: + type: string + description: | Paths to local or remote repositories (required). Tildes are expanded. Multiple repositories are backed up to in sequence. Borg placeholders can be used. See the output of @@ -37,36 +43,36 @@ map: - user@backupserver:sourcehostname.borg - "user@backupserver:{fqdn}" one_file_system: - type: bool - desc: | + type: boolean + description: | Stay in same file system (do not cross mount points). Defaults to false. But when a database hook is used, the setting here is ignored and one_file_system is considered true. example: true numeric_owner: - type: bool - desc: | + type: boolean + description: | Only store/extract numeric user and group identifiers. Defaults to false. example: true atime: - type: bool - desc: Store atime into archive. Defaults to true. + type: boolean + description: Store atime into archive. Defaults to true. example: false ctime: - type: bool - desc: Store ctime into archive. Defaults to true. + type: boolean + description: Store ctime into archive. Defaults to true. example: false birthtime: - type: bool - desc: | + type: boolean + description: | Store birthtime (creation date) into archive. Defaults to true. example: false read_special: - type: bool - desc: | + type: boolean + description: | Use Borg's --read-special flag to allow backup of block and other special devices. Use with caution, as it will lead to problems if used when backing up special devices such as @@ -75,30 +81,33 @@ map: considered true. example: false bsd_flags: - type: bool - desc: | + type: boolean + description: | Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true. example: true files_cache: - type: str - desc: | + type: string + description: | Mode in which to operate the files cache. See http://borgbackup.readthedocs.io/en/stable/usage/create.html for details. Defaults to "ctime,size,inode". example: ctime,size,inode local_path: - type: str - desc: Alternate Borg local executable. Defaults to "borg". + type: string + description: | + Alternate Borg local executable. Defaults to "borg". example: borg1 remote_path: - type: str - desc: Alternate Borg remote executable. Defaults to "borg". + type: string + description: | + Alternate Borg remote executable. Defaults to "borg". example: borg1 patterns: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | Any paths matching these patterns are included/excluded from backups. Globs are expanded. (Tildes are not.) Note that Borg considers this option experimental. See the output of @@ -110,9 +119,10 @@ map: - '+ /home/susan' - '- /home/*' patterns_from: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | Read include/exclude patterns from one or more separate named files, one pattern per line. Note that Borg considers this option experimental. See the output of "borg help @@ -120,9 +130,10 @@ map: example: - /etc/borgmatic/patterns exclude_patterns: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | Any paths matching these patterns are excluded from backups. Globs and tildes are expanded. Do not backslash spaces in path names. See the output of "borg help patterns" for more @@ -133,59 +144,63 @@ map: - /etc/ssl - /home/user/path with spaces exclude_from: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | Read exclude patterns from one or more separate named files, one pattern per line. See the output of "borg help patterns" for more details. example: - /etc/borgmatic/excludes exclude_caches: - type: bool - desc: | + type: boolean + description: | Exclude directories that contain a CACHEDIR.TAG file. See http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false. example: true exclude_if_present: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | Exclude directories that contain a file with the given filenames. Defaults to not set. example: - .nobackup keep_exclude_tags: - type: bool - desc: | + type: boolean + description: | If true, the exclude_if_present filename is included in backups. Defaults to false, meaning that the exclude_if_present filename is omitted from backups. example: true exclude_nodump: - type: bool - desc: | + type: boolean + description: | Exclude files with the NODUMP flag. Defaults to false. example: true borgmatic_source_directory: - type: str - desc: | + type: string + description: | Path for additional source files used for temporary internal state like borgmatic database dumps. Note that changing this path prevents "borgmatic restore" from finding any database dumps created before the change. Defaults to ~/.borgmatic example: /tmp/borgmatic storage: - desc: | + type: object + description: | Repository storage options. See https://borgbackup.readthedocs.io/en/stable/usage/create.html and https://borgbackup.readthedocs.io/en/stable/usage/general.html for details. - map: + additionalProperties: false + properties: encryption_passcommand: - type: str - desc: | + type: string + description: | The standard output of this command is used to unlock the encryption key. Only use on repositories that were initialized with passcommand/repokey/keyfile encryption. @@ -194,8 +209,8 @@ map: takes precedence. Defaults to not set. example: "secret-tool lookup borg-repository repo-name" encryption_passphrase: - type: str - desc: | + type: string + description: | Passphrase to unlock the encryption key with. Only use on repositories that were initialized with passphrase/repokey/keyfile encryption. Quote the value if it @@ -204,8 +219,8 @@ map: set. example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" checkpoint_interval: - type: int - desc: | + type: integer + description: | Number of seconds between each checkpoint during a long-running backup. See https://borgbackup.readthedocs.io/en/stable/faq.html @@ -213,8 +228,8 @@ map: minutes). example: 1800 chunker_params: - type: str - desc: | + type: string + description: | Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). See @@ -222,73 +237,73 @@ map: for details. Defaults to "19,23,21,4095". example: 19,23,21,4095 compression: - type: str - desc: | + type: string + description: | Type of compression to use when creating archives. See http://borgbackup.readthedocs.io/en/stable/usage/create.html for details. Defaults to "lz4". example: lz4 remote_rate_limit: - type: int - desc: | + type: integer + description: | Remote network upload rate limit in kiBytes/second. Defaults to unlimited. example: 100 temporary_directory: - type: str - desc: | + type: string + description: | Directory where temporary files are stored. Defaults to $TMPDIR example: /path/to/tmpdir ssh_command: - type: str - desc: | + type: string + description: | Command to use instead of "ssh". This can be used to specify ssh options. Defaults to not set. example: ssh -i /path/to/private/key borg_base_directory: - type: str - desc: | + type: string + description: | Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~. example: /path/to/base borg_config_directory: - type: str - desc: | + type: string + description: | Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg example: /path/to/base/config borg_cache_directory: - type: str - desc: | + type: string + description: | Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg example: /path/to/base/cache borg_security_directory: - type: str - desc: | + type: string + description: | Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security example: /path/to/base/config/security borg_keys_directory: - type: str - desc: | + type: string + description: | Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys example: /path/to/base/config/keys umask: - type: scalar - desc: Umask to be used for borg create. Defaults to 0077. + type: string + description: Umask to be used for borg create. Defaults to 0077. example: 0077 lock_wait: - type: int - desc: | + type: integer + description: | Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1. example: 5 archive_name_format: - type: str - desc: | + type: string + description: | Name of the archive. Borg placeholders can be used. See the output of "borg help placeholders" for details. Defaults to "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this @@ -298,40 +313,42 @@ map: prefix in the consistency section as well. example: "{hostname}-documents-{now}" relocated_repo_access_is_ok: - type: bool - desc: | + type: boolean + description: | Bypass Borg error about a repository that has been moved. Defaults to false. example: true unknown_unencrypted_repo_access_is_ok: - type: bool - desc: | + type: boolean + description: | Bypass Borg error about a previously unknown unencrypted repository. Defaults to false. example: true extra_borg_options: - map: + type: object + additionalProperties: false + properties: init: - type: str - desc: | + type: string + description: | Extra command-line options to pass to "borg init". example: "--make-parent-dirs" prune: - type: str - desc: | + type: string + description: | Extra command-line options to pass to "borg prune". example: "--save-space" create: - type: str - desc: | + type: string + description: | Extra command-line options to pass to "borg create". example: "--no-files-cache" check: - type: str - desc: | + type: string + description: | Extra command-line options to pass to "borg check". example: "--save-space" - desc: | + description: | Additional options to pass directly to particular Borg commands, handy for Borg options that borgmatic does not yet support natively. Note that borgmatic does not perform any @@ -339,72 +356,76 @@ map: "--verbosity 2" shows the exact Borg command-line invocation. retention: - desc: | + type: object + description: | Retention policy for how many backups to keep in each category. See https://borgbackup.readthedocs.io/en/stable/usage/prune.html for details. At least one of the "keep" options is required for pruning to work. To skip pruning entirely, run "borgmatic create" or "check" without the "prune" action. See borgmatic documentation for details. - map: + additionalProperties: false + properties: keep_within: - type: str - desc: Keep all archives within this time interval. + type: string + description: Keep all archives within this time interval. example: 3H keep_secondly: - type: int - desc: Number of secondly archives to keep. + type: integer + description: Number of secondly archives to keep. example: 60 keep_minutely: - type: int - desc: Number of minutely archives to keep. + type: integer + description: Number of minutely archives to keep. example: 60 keep_hourly: - type: int - desc: Number of hourly archives to keep. + type: integer + description: Number of hourly archives to keep. example: 24 keep_daily: - type: int - desc: Number of daily archives to keep. + type: integer + description: Number of daily archives to keep. example: 7 keep_weekly: - type: int - desc: Number of weekly archives to keep. + type: integer + description: Number of weekly archives to keep. example: 4 keep_monthly: - type: int - desc: Number of monthly archives to keep. + type: integer + description: Number of monthly archives to keep. example: 6 keep_yearly: - type: int - desc: Number of yearly archives to keep. + type: integer + description: Number of yearly archives to keep. example: 1 prefix: - type: str - desc: | + type: string + description: | When pruning, only consider archive names starting with this prefix. Borg placeholders can be used. See the output of "borg help placeholders" for details. Defaults to "{hostname}-". Use an empty value to disable the default. example: sourcehostname consistency: - desc: | + type: object + description: | Consistency checks to run after backups. See https://borgbackup.readthedocs.io/en/stable/usage/check.html and https://borgbackup.readthedocs.io/en/stable/usage/extract.html for details. - map: + additionalProperties: false + properties: checks: - seq: - - type: str - enum: [ - 'repository', - 'archives', - 'data', - 'extract', - 'disabled' - ] - unique: true - desc: | + type: array + items: + type: string + enum: + - repository + - archives + - data + - extract + - disabled + uniqueItems: true + description: | List of one or more consistency checks to run: "repository", "archives", "data", and/or "extract". Defaults to "repository" and "archives". Set to "disabled" to disable @@ -417,9 +438,10 @@ map: - repository - archives check_repositories: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | Paths to a subset of the repositories in the location section on which to run consistency checks. Handy in case some of your repositories are very large, and so running @@ -429,15 +451,15 @@ map: example: - user@backupserver:sourcehostname.borg check_last: - type: int - desc: | + type: integer + description: | Restrict the number of checked archives to the last n. Applies only to the "archives" check. Defaults to checking all archives. example: 3 prefix: - type: str - desc: | + type: string + description: | When performing the "archives" check, only consider archive names starting with this prefix. Borg placeholders can be used. See the output of "borg help placeholders" for @@ -445,101 +467,115 @@ map: disable the default. example: sourcehostname output: - desc: | + type: object + description: | Options for customizing borgmatic's own output and logging. - map: + additionalProperties: false + properties: color: - type: bool - desc: | + type: boolean + description: | Apply color to console output. Can be overridden with --no-color command-line flag. Defaults to true. example: false hooks: - desc: | + type: object + description: | Shell commands, scripts, or integrations to execute at various points during a borgmatic run. IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic. Do not forget to set secure permissions on this configuration file (chmod 0600) as well as on any script called from a hook (chmod 0700) to prevent potential shell injection or privilege escalation. - map: + additionalProperties: false + properties: before_backup: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute before creating a backup, run once per configuration file. example: - echo "Starting a backup." before_prune: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute before pruning, run once per configuration file. example: - echo "Starting pruning." before_check: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute before consistency checks, run once per configuration file. example: - echo "Starting checks." before_extract: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute before extracting a backup, run once per configuration file. example: - echo "Starting extracting." after_backup: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute after creating a backup, run once per configuration file. example: - echo "Finished a backup." after_prune: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute after pruning, run once per configuration file. example: - echo "Finished pruning." after_check: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute after consistency checks, run once per configuration file. example: - echo "Finished checks." after_extract: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute after extracting a backup, run once per configuration file. example: - echo "Finished extracting." on_error: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute when an exception occurs during a "prune", "create", or "check" action or an associated before/after hook. example: - echo "Error during prune/create/check." before_everything: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute before running all actions (if one of them is "create"). These are collected from all configuration files and then @@ -547,9 +583,10 @@ map: example: - echo "Starting actions." after_everything: - seq: - - type: str - desc: | + type: array + items: + type: string + description: | List of one or more shell commands or scripts to execute after running all actions (if one of them is "create"). These are collected from all configuration files and then @@ -557,12 +594,15 @@ map: example: - echo "Completed actions." postgresql_databases: - seq: - - map: + type: array + items: + type: object + required: ['name'] + additionalProperties: false + properties: name: - required: true - type: str - desc: | + type: string + description: | Database name (required if using this hook). Or "all" to dump all databases on the host. Note that using this database hook implicitly enables @@ -570,26 +610,26 @@ map: above) to support dump and restore streaming. example: users hostname: - type: str - desc: | + type: string + description: | Database hostname to connect to. Defaults to connecting via local Unix socket. example: database.example.org port: - type: int - desc: Port to connect to. Defaults to 5432. + type: integer + description: Port to connect to. Defaults to 5432. example: 5433 username: - type: str - desc: | + type: string + description: | Username with which to connect to the database. Defaults to the username of the current user. You probably want to specify the "postgres" superuser here when the database name is "all". example: dbuser password: - type: str - desc: | + type: string + description: | Password with which to connect to the database. Omitting a password will only work if PostgreSQL is configured to trust the configured username @@ -597,9 +637,9 @@ map: file. example: trustsome1 format: - type: str + type: string enum: ['plain', 'custom', 'directory', 'tar'] - desc: | + description: | Database dump output format. One of "plain", "custom", "directory", or "tar". Defaults to "custom" (unlike raw pg_dump). See pg_dump @@ -607,45 +647,45 @@ map: ignored when the database name is "all". example: directory ssl_mode: - type: str + type: string enum: ['disable', 'allow', 'prefer', 'require', 'verify-ca', 'verify-full'] - desc: | + description: | SSL mode to use to connect to the database server. One of "disable", "allow", "prefer", "require", "verify-ca" or "verify-full". Defaults to "disable". example: require ssl_cert: - type: str - desc: | + type: string + description: | Path to a client certificate. example: "/root/.postgresql/postgresql.crt" ssl_key: - type: str - desc: | + type: string + description: | Path to a private client key. example: "/root/.postgresql/postgresql.key" ssl_root_cert: - type: str - desc: | + type: string + description: | Path to a root certificate containing a list of trusted certificate authorities. example: "/root/.postgresql/root.crt" ssl_crl: - type: str - desc: | + type: string + description: | Path to a certificate revocation list. example: "/root/.postgresql/root.crl" options: - type: str - desc: | + type: string + description: | Additional pg_dump/pg_dumpall options to pass directly to the dump command, without performing any validation on them. See pg_dump documentation for details. example: --role=someone - desc: | + description: | List of one or more PostgreSQL databases to dump before creating a backup, run once per configuration file. The database dumps are added to your source directories at @@ -655,12 +695,15 @@ map: https://www.postgresql.org/docs/current/libpq-ssl.html for details. mysql_databases: - seq: - - map: + type: array + items: + type: object + required: ['name'] + additionalProperties: false + properties: name: - required: true - type: str - desc: | + type: string + description: | Database name (required if using this hook). Or "all" to dump all databases on the host. Note that using this database hook implicitly enables @@ -668,38 +711,38 @@ map: above) to support dump and restore streaming. example: users hostname: - type: str - desc: | + type: string + description: | Database hostname to connect to. Defaults to connecting via local Unix socket. example: database.example.org port: - type: int - desc: Port to connect to. Defaults to 3306. + type: integer + description: Port to connect to. Defaults to 3306. example: 3307 username: - type: str - desc: | + type: string + description: | Username with which to connect to the database. Defaults to the username of the current user. example: dbuser password: - type: str - desc: | + type: string + description: | Password with which to connect to the database. Omitting a password will only work if MySQL is configured to trust the configured username without a password. example: trustsome1 options: - type: str - desc: | + type: string + description: | Additional mysqldump options to pass directly to the dump command, without performing any validation on them. See mysqldump documentation for details. example: --skip-comments - desc: | + description: | List of one or more MySQL/MariaDB databases to dump before creating a backup, run once per configuration file. The database dumps are added to your source directories at @@ -708,8 +751,8 @@ map: https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or https://mariadb.com/kb/en/library/mysqldump/ for details. healthchecks: - type: str - desc: | + type: string + description: | Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors. Create an account at https://healthchecks.io if you'd like to use this service. @@ -717,8 +760,8 @@ map: example: https://hc-ping.com/your-uuid-here cronitor: - type: str - desc: | + type: string + description: | Cronitor ping URL to notify when a backup begins, ends, or errors. Create an account at https://cronitor.io if you'd like to use this service. See borgmatic monitoring @@ -726,8 +769,8 @@ map: example: https://cronitor.link/d3x0c1 pagerduty: - type: str - desc: | + type: string + description: | PagerDuty integration key used to notify PagerDuty when a backup errors. Create an account at https://www.pagerduty.com/ if you'd like to use this @@ -735,8 +778,8 @@ map: example: a177cad45bd374409f78906a810a3074 cronhub: - type: str - desc: | + type: string + description: | Cronhub ping URL to notify when a backup begins, ends, or errors. Create an account at https://cronhub.io if you'd like to use this service. See borgmatic monitoring @@ -745,7 +788,7 @@ map: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01 umask: type: scalar - desc: | + description: | Umask used when executing hooks. Defaults to the umask that borgmatic is run with. example: 0077 diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 504e18185..5658c1b16 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -1,9 +1,7 @@ -import logging import os +import jsonschema import pkg_resources -import pykwalify.core -import pykwalify.errors import ruamel.yaml from borgmatic.config import load, normalize, override @@ -17,15 +15,40 @@ def schema_filename(): return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml') +def format_error_path_element(path_element): + ''' + Given a path element into a JSON data structure, format it for display as a string. + ''' + if isinstance(path_element, int): + return str('[{}]'.format(path_element)) + + return str('.{}'.format(path_element)) + + +def format_error(error): + ''' + Given an instance of jsonschema.exceptions.ValidationError, format it for display as a string. + ''' + if not error.path: + return 'At the top level: {}'.format(error.message) + + formatted_path = ''.join(format_error_path_element(element) for element in error.path) + return "At '{}': {}".format(formatted_path.lstrip('.'), error.message) + + class Validation_error(ValueError): ''' - A collection of error message strings generated when attempting to validate a particular - configurartion file. + A collection of error messages generated when attempting to validate a particular + configuration file. ''' - def __init__(self, config_filename, error_messages): + def __init__(self, config_filename, errors): + ''' + Given a configuration filename path and a sequence of + jsonschema.exceptions.ValidationError instances, create a Validation_error. + ''' self.config_filename = config_filename - self.error_messages = error_messages + self.errors = errors def __str__(self): ''' @@ -33,7 +56,7 @@ class Validation_error(ValueError): ''' return 'An error occurred while parsing a configuration file at {}:\n'.format( self.config_filename - ) + '\n'.join(self.error_messages) + ) + '\n'.join(format_error(error) for error in self.errors) def apply_logical_validation(config_filename, parsed_configuration): @@ -65,29 +88,12 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def remove_examples(schema): - ''' - pykwalify gets angry if the example field is not a string. So rather than bend to its will, - remove all examples from the given schema before passing the schema to pykwalify. - ''' - if 'map' in schema: - for item_name, item_schema in schema['map'].items(): - item_schema.pop('example', None) - remove_examples(item_schema) - elif 'seq' in schema: - for item_schema in schema['seq']: - item_schema.pop('example', None) - remove_examples(item_schema) - - return schema - - def parse_configuration(config_filename, schema_filename, overrides=None): ''' - Given the path to a config filename in YAML format, the path to a schema filename in pykwalify - YAML schema format, a sequence of configuration file override strings in the form of - "section.option=value", return the parsed configuration as a data structure of nested dicts and - lists corresponding to the schema. Example return value: + Given the path to a config filename in YAML format, the path to a schema filename in a YAML + rendition of JSON Schema format, a sequence of configuration file override strings in the form + of "section.option=value", return the parsed configuration as a data structure of nested dicts + and lists corresponding to the schema. Example return value: {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}} @@ -95,8 +101,6 @@ def parse_configuration(config_filename, schema_filename, overrides=None): Raise FileNotFoundError if the file does not exist, PermissionError if the user does not have permissions to read the file, or Validation_error if the config does not match the schema. ''' - logging.getLogger('pykwalify').setLevel(logging.ERROR) - try: config = load.load_configuration(config_filename) schema = load.load_configuration(schema_filename) @@ -106,15 +110,15 @@ def parse_configuration(config_filename, schema_filename, overrides=None): override.apply_overrides(config, overrides) normalize.normalize(config) - validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema)) - parsed_result = validator.validate(raise_exception=False) + validator = jsonschema.Draft7Validator(schema) + validation_errors = tuple(validator.iter_errors(config)) - if validator.validation_errors: - raise Validation_error(config_filename, validator.validation_errors) + if validation_errors: + raise Validation_error(config_filename, validation_errors) - apply_logical_validation(config_filename, parsed_result) + apply_logical_validation(config_filename, config) - return parsed_result + return config def normalize_repository_path(repository): diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 709923050..84a31660b 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -1,4 +1,5 @@ import logging +import logging.handlers import os import sys diff --git a/setup.py b/setup.py index 31b47dd2e..ebc47fd2e 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setup( }, obsoletes=['atticmatic'], install_requires=( - 'pykwalify>=1.6.0,<14.06', + 'jsonschema', 'requests', 'ruamel.yaml>0.15.0,<0.18.0', 'setuptools', diff --git a/test_requirements.txt b/test_requirements.txt index 0fa3e6ec8..56482a7c2 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,26 +1,21 @@ appdirs==1.4.4; python_version >= '3.8' -atomicwrites==1.4.0 attrs==20.3.0; python_version >= '3.8' black==19.10b0; python_version >= '3.8' click==7.1.2; python_version >= '3.8' colorama==0.4.4 coverage==5.3 -docopt==0.6.2 flake8==3.8.4 flexmock==0.10.4 -isort==5.6.4 +isort==5.9.1 mccabe==0.6.1 -more-itertools==8.6.0 pluggy==0.13.1 pathspec==0.8.1; python_version >= '3.8' py==1.10.0 pycodestyle==2.6.0 pyflakes==2.2.0 -pykwalify==1.7.0 +jsonschema==3.2.0 pytest==6.1.2 pytest-cov==2.10.1 -python-dateutil==2.8.1 -PyYAML==5.4.1 regex; python_version >= '3.8' requests==2.25.0 ruamel.yaml>0.15.0,<0.18.0 diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 405beb33b..9534a5fb0 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -122,38 +122,44 @@ def test_write_configuration_with_already_existing_directory_does_not_raise(): def test_add_comments_to_configuration_sequence_of_strings_does_not_raise(): config = module.yaml.comments.CommentedSeq(['foo', 'bar']) - schema = {'seq': [{'type': 'str'}]} + schema = {'type': 'array', 'items': {'type': 'string'}} module.add_comments_to_configuration_sequence(config, schema) def test_add_comments_to_configuration_sequence_of_maps_does_not_raise(): config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])]) - schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]} + schema = { + 'type': 'array', + 'items': {'type': 'object', 'properties': {'foo': {'description': 'yo'}}}, + } module.add_comments_to_configuration_sequence(config, schema) def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise(): config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])]) - schema = {'seq': [{'map': {'foo': {}}}]} + schema = {'type': 'array', 'items': {'type': 'object', 'properties': {'foo': {}}}} module.add_comments_to_configuration_sequence(config, schema) -def test_add_comments_to_configuration_map_does_not_raise(): +def test_add_comments_to_configuration_object_does_not_raise(): # Ensure that it can deal with fields both in the schema and missing from the schema. config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) - schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}} + schema = { + 'type': 'object', + 'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}}, + } - module.add_comments_to_configuration_map(config, schema) + module.add_comments_to_configuration_object(config, schema) -def test_add_comments_to_configuration_map_with_skip_first_does_not_raise(): +def test_add_comments_to_configuration_object_with_skip_first_does_not_raise(): config = module.yaml.comments.CommentedMap([('foo', 33)]) - schema = {'map': {'foo': {'desc': 'Foo'}}} + schema = {'type': 'object', 'properties': {'foo': {'description': 'Foo'}}} - module.add_comments_to_configuration_map(config, schema, skip_first=True) + module.add_comments_to_configuration_object(config, schema, skip_first=True) def test_remove_commented_out_sentinel_keeps_other_comments(): diff --git a/tests/unit/config/test_convert.py b/tests/unit/config/test_convert.py index 8ae5dc678..8ce1cdc6c 100644 --- a/tests/unit/config/test_convert.py +++ b/tests/unit/config/test_convert.py @@ -12,7 +12,7 @@ Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', def test_convert_section_generates_integer_value_for_integer_type_in_schema(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) source_section_config = OrderedDict([('check_last', '3')]) - section_schema = {'map': {'check_last': {'type': 'int'}}} + section_schema = {'type': 'object', 'properties': {'check_last': {'type': 'integer'}}} destination_config = module._convert_section(source_section_config, section_schema) @@ -21,7 +21,7 @@ def test_convert_section_generates_integer_value_for_integer_type_in_schema(): def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) - flexmock(module.generate).should_receive('add_comments_to_configuration_map') + flexmock(module.generate).should_receive('add_comments_to_configuration_object') source_config = Parsed_config( location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]), storage=OrderedDict([('encryption_passphrase', 'supersecret')]), @@ -29,7 +29,10 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): consistency=OrderedDict([('checks', 'repository')]), ) source_excludes = ['/var'] - schema = {'map': defaultdict(lambda: {'map': {}})} + schema = { + 'type': 'object', + 'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}), + } destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) @@ -54,7 +57,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): def test_convert_legacy_parsed_config_splits_space_separated_values(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) - flexmock(module.generate).should_receive('add_comments_to_configuration_map') + flexmock(module.generate).should_receive('add_comments_to_configuration_object') source_config = Parsed_config( location=OrderedDict( [('source_directories', '/home /etc'), ('repository', 'hostname.borg')] @@ -64,7 +67,10 @@ def test_convert_legacy_parsed_config_splits_space_separated_values(): consistency=OrderedDict([('checks', 'repository archives')]), ) source_excludes = ['/var'] - schema = {'map': defaultdict(lambda: {'map': {}})} + schema = { + 'type': 'object', + 'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}), + } destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 8e5a07f12..7855592ca 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -8,24 +8,32 @@ from borgmatic.config import generate as module def test_schema_to_sample_configuration_generates_config_map_with_examples(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) - flexmock(module).should_receive('add_comments_to_configuration_map') + flexmock(module).should_receive('add_comments_to_configuration_object') schema = { - 'map': OrderedDict( + 'type': 'object', + 'properties': OrderedDict( [ - ('section1', {'map': {'field1': OrderedDict([('example', 'Example 1')])}}), + ( + 'section1', + { + 'type': 'object', + 'properties': {'field1': OrderedDict([('example', 'Example 1')])}, + }, + ), ( 'section2', { - 'map': OrderedDict( + 'type': 'object', + 'properties': OrderedDict( [ ('field2', {'example': 'Example 2'}), ('field3', {'example': 'Example 3'}), ] - ) + ), }, ), ] - ) + ), } config = module._schema_to_sample_configuration(schema) @@ -41,7 +49,7 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example(): flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') - schema = {'seq': [{'type': 'str'}], 'example': ['hi']} + schema = {'type': 'array', 'items': {'type': 'string'}, 'example': ['hi']} config = module._schema_to_sample_configuration(schema) @@ -51,15 +59,15 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_strings_wit def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples(): flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') - flexmock(module).should_receive('add_comments_to_configuration_map') + flexmock(module).should_receive('add_comments_to_configuration_object') schema = { - 'seq': [ - { - 'map': OrderedDict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] - ) - } - ] + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': OrderedDict( + [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + ), + }, } config = module._schema_to_sample_configuration(schema) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 4fc4a620d..0ab4c0152 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -4,8 +4,33 @@ from flexmock import flexmock from borgmatic.config import validate as module -def test_validation_error_str_contains_error_messages_and_config_filename(): - error = module.Validation_error('config.yaml', ('oops', 'uh oh')) +def test_format_error_path_element_formats_array_index(): + module.format_error_path_element(3) == '[3]' + + +def test_format_error_path_element_formats_property(): + module.format_error_path_element('foo') == '.foo' + + +def test_format_error_formats_error_including_path(): + flexmock(module).format_error_path_element = lambda element: '.{}'.format(element) + error = flexmock(message='oops', path=['foo', 'bar']) + + assert module.format_error(error) == "At 'foo.bar': oops" + + +def test_format_error_formats_error_without_path(): + flexmock(module).should_receive('format_error_path_element').never() + error = flexmock(message='oops', path=[]) + + assert module.format_error(error) == 'At the top level: oops' + + +def test_validation_error_string_contains_error_messages_and_config_filename(): + flexmock(module).format_error = lambda error: error.message + error = module.Validation_error( + 'config.yaml', (flexmock(message='oops', path=None), flexmock(message='uh oh')) + ) result = str(error) @@ -15,6 +40,8 @@ def test_validation_error_str_contains_error_messages_and_config_filename(): def test_apply_logical_validation_raises_if_archive_name_format_present_without_prefix(): + flexmock(module).format_error = lambda error: error.message + with pytest.raises(module.Validation_error): module.apply_logical_validation( 'config.yaml', @@ -26,6 +53,8 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_ def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix(): + flexmock(module).format_error = lambda error: error.message + with pytest.raises(module.Validation_error): module.apply_logical_validation( 'config.yaml', @@ -38,6 +67,8 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_ def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories(): + flexmock(module).format_error = lambda error: error.message + with pytest.raises(module.Validation_error): module.apply_logical_validation( 'config.yaml', @@ -75,27 +106,6 @@ def test_apply_logical_validation_does_not_raise_otherwise(): module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}}) -def test_remove_examples_strips_examples_from_map(): - schema = { - 'map': { - 'foo': {'desc': 'thing1', 'example': 'bar'}, - 'baz': {'desc': 'thing2', 'example': 'quux'}, - } - } - - module.remove_examples(schema) - - assert schema == {'map': {'foo': {'desc': 'thing1'}, 'baz': {'desc': 'thing2'}}} - - -def test_remove_examples_strips_examples_from_sequence_of_maps(): - schema = {'seq': [{'map': {'foo': {'desc': 'thing', 'example': 'bar'}}, 'example': 'stuff'}]} - - module.remove_examples(schema) - - assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]} - - def test_normalize_repository_path_passes_through_remote_repository(): repository = 'example.org:test.borg'