Selectively shallow merge certain mappings or sequences when including configuration files (#672).
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
This commit is contained in:
parent
4c0e2cab78
commit
1ea4433aa9
4 changed files with 374 additions and 70 deletions
|
|
@ -2,7 +2,6 @@ import io
|
|||
import sys
|
||||
|
||||
import pytest
|
||||
import ruamel.yaml
|
||||
from flexmock import flexmock
|
||||
|
||||
from borgmatic.config import load as module
|
||||
|
|
@ -150,6 +149,99 @@ def test_load_configuration_merges_include():
|
|||
assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
|
||||
|
||||
|
||||
def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values():
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
|
||||
flexmock(module.os.path).should_receive('isabs').and_return(False)
|
||||
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||
include_file = io.StringIO(
|
||||
'''
|
||||
stuff:
|
||||
foo: bar
|
||||
baz: quux
|
||||
|
||||
other:
|
||||
a: b
|
||||
c: d
|
||||
'''
|
||||
)
|
||||
include_file.name = 'include.yaml'
|
||||
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
|
||||
config_file = io.StringIO(
|
||||
'''
|
||||
stuff: !retain
|
||||
foo: override
|
||||
|
||||
other:
|
||||
a: override
|
||||
<<: !include include.yaml
|
||||
'''
|
||||
)
|
||||
config_file.name = 'config.yaml'
|
||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||
|
||||
assert module.load_configuration('config.yaml') == {
|
||||
'stuff': {'foo': 'override'},
|
||||
'other': {'a': 'override', 'c': 'd'},
|
||||
}
|
||||
|
||||
|
||||
def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
|
||||
flexmock(module.os.path).should_receive('isabs').and_return(False)
|
||||
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||
include_file = io.StringIO(
|
||||
'''
|
||||
stuff: !retain
|
||||
foo: bar
|
||||
baz: quux
|
||||
'''
|
||||
)
|
||||
include_file.name = 'include.yaml'
|
||||
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
|
||||
config_file = io.StringIO(
|
||||
'''
|
||||
stuff:
|
||||
foo: override
|
||||
<<: !include include.yaml
|
||||
'''
|
||||
)
|
||||
config_file.name = 'config.yaml'
|
||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
assert module.load_configuration('config.yaml')
|
||||
|
||||
|
||||
def test_load_configuration_with_retain_tag_on_scalar_raises():
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
|
||||
flexmock(module.os.path).should_receive('isabs').and_return(False)
|
||||
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||
include_file = io.StringIO(
|
||||
'''
|
||||
stuff:
|
||||
foo: bar
|
||||
baz: quux
|
||||
'''
|
||||
)
|
||||
include_file.name = 'include.yaml'
|
||||
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
|
||||
config_file = io.StringIO(
|
||||
'''
|
||||
stuff:
|
||||
foo: !retain override
|
||||
<<: !include include.yaml
|
||||
'''
|
||||
)
|
||||
config_file.name = 'config.yaml'
|
||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
assert module.load_configuration('config.yaml')
|
||||
|
||||
|
||||
def test_load_configuration_does_not_merge_include_list():
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
|
||||
|
|
@ -173,42 +265,59 @@ def test_load_configuration_does_not_merge_include_list():
|
|||
config_file.name = 'config.yaml'
|
||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||
|
||||
with pytest.raises(ruamel.yaml.error.YAMLError):
|
||||
with pytest.raises(module.ruamel.yaml.error.YAMLError):
|
||||
assert module.load_configuration('config.yaml')
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'node_class',
|
||||
(
|
||||
module.ruamel.yaml.nodes.MappingNode,
|
||||
module.ruamel.yaml.nodes.SequenceNode,
|
||||
module.ruamel.yaml.nodes.ScalarNode,
|
||||
),
|
||||
)
|
||||
def test_retain_node_error_raises(node_class):
|
||||
with pytest.raises(ValueError):
|
||||
module.retain_node_error(
|
||||
loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock())
|
||||
)
|
||||
|
||||
|
||||
def test_deep_merge_nodes_replaces_colliding_scalar_values():
|
||||
node_values = [
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_hourly'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:int', value='24'
|
||||
),
|
||||
),
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_daily'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_daily'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
|
||||
),
|
||||
],
|
||||
),
|
||||
|
|
@ -230,35 +339,39 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values():
|
|||
def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
|
||||
node_values = [
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_hourly'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:int', value='24'
|
||||
),
|
||||
),
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_daily'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_minutely'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='10'),
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:int', value='10'
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
|
|
@ -282,28 +395,28 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
|
|||
def test_deep_merge_nodes_keeps_deeply_nested_values():
|
||||
node_values = [
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='lock_wait'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
|
||||
),
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='extra_borg_options'
|
||||
),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='init'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='--init-option'
|
||||
),
|
||||
),
|
||||
|
|
@ -314,22 +427,22 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
|
|||
),
|
||||
),
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='extra_borg_options'
|
||||
),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='prune'
|
||||
),
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='--prune-option'
|
||||
),
|
||||
),
|
||||
|
|
@ -361,32 +474,32 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
|
|||
def test_deep_merge_nodes_appends_colliding_sequence_values():
|
||||
node_values = [
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='before_backup'
|
||||
),
|
||||
ruamel.yaml.nodes.SequenceNode(
|
||||
tag='tag:yaml.org,2002:int', value=['echo 1', 'echo 2']
|
||||
module.ruamel.yaml.nodes.SequenceNode(
|
||||
tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2']
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
|
||||
ruamel.yaml.nodes.MappingNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
ruamel.yaml.nodes.ScalarNode(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='before_backup'
|
||||
),
|
||||
ruamel.yaml.nodes.SequenceNode(
|
||||
tag='tag:yaml.org,2002:int', value=['echo 3', 'echo 4']
|
||||
module.ruamel.yaml.nodes.SequenceNode(
|
||||
tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4']
|
||||
),
|
||||
),
|
||||
],
|
||||
|
|
@ -402,3 +515,101 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
|
|||
assert len(options) == 1
|
||||
assert options[0][0].value == 'before_backup'
|
||||
assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
|
||||
|
||||
|
||||
def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain():
|
||||
node_values = [
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_hourly'
|
||||
),
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:int', value='24'
|
||||
),
|
||||
),
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_daily'
|
||||
),
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='!retain',
|
||||
value=[
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='keep_daily'
|
||||
),
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
result = module.deep_merge_nodes(node_values)
|
||||
assert len(result) == 1
|
||||
(section_key, section_value) = result[0]
|
||||
assert section_key.value == 'retention'
|
||||
assert section_value.tag == 'tag:yaml.org,2002:map'
|
||||
options = section_value.value
|
||||
assert len(options) == 1
|
||||
assert options[0][0].value == 'keep_daily'
|
||||
assert options[0][1].value == '5'
|
||||
|
||||
|
||||
def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
|
||||
node_values = [
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='before_backup'
|
||||
),
|
||||
module.ruamel.yaml.nodes.SequenceNode(
|
||||
tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2']
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
|
||||
module.ruamel.yaml.nodes.MappingNode(
|
||||
tag='tag:yaml.org,2002:map',
|
||||
value=[
|
||||
(
|
||||
module.ruamel.yaml.nodes.ScalarNode(
|
||||
tag='tag:yaml.org,2002:str', value='before_backup'
|
||||
),
|
||||
module.ruamel.yaml.nodes.SequenceNode(
|
||||
tag='!retain', value=['echo 3', 'echo 4']
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
result = module.deep_merge_nodes(node_values)
|
||||
assert len(result) == 1
|
||||
(section_key, section_value) = result[0]
|
||||
assert section_key.value == 'hooks'
|
||||
options = section_value.value
|
||||
assert len(options) == 1
|
||||
assert options[0][0].value == 'before_backup'
|
||||
assert options[0][1].tag == 'tag:yaml.org,2002:seq'
|
||||
assert options[0][1].value == ['echo 3', 'echo 4']
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue