Selectively omit list values when including configuration files (#672).

tests/integration/config/test_load.py

@@ -211,7 +211,7 @@ def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
     builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
 
     with pytest.raises(ValueError):
-        assert module.load_configuration('config.yaml')
+        module.load_configuration('config.yaml')
 
 
 def test_load_configuration_with_retain_tag_on_scalar_raises():
@@ -239,7 +239,156 @@ def test_load_configuration_with_retain_tag_on_scalar_raises():
     builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
 
     with pytest.raises(ValueError):
-        assert module.load_configuration('config.yaml')
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - !omit b
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']}
+
+
+def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - !omit q
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']}
+
+
+def test_load_configuration_with_omit_tag_on_non_list_item_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff: !omit
+          - x
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - foo: bar
+            baz: quux
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - !omit foo: bar
+            baz: quux
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_but_without_merge_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - !omit b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
 
 
 def test_load_configuration_does_not_merge_include_list():
@@ -277,13 +426,33 @@ def test_load_configuration_does_not_merge_include_list():
         module.ruamel.yaml.nodes.ScalarNode,
     ),
 )
-def test_retain_node_error_raises(node_class):
+def test_raise_retain_node_error_raises(node_class):
     with pytest.raises(ValueError):
-        module.retain_node_error(
+        module.raise_retain_node_error(
             loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock())
         )
 
 
+def test_raise_omit_node_error_raises():
+    with pytest.raises(ValueError):
+        module.raise_omit_node_error(loader=flexmock(), node=flexmock())
+
+
+def test_filter_omitted_nodes():
+    nodes = [
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+    ]
+
+    result = module.filter_omitted_nodes(nodes)
+
+    assert [item.value for item in result] == ['a', 'c', 'a', 'c']
+
+
 def test_deep_merge_nodes_replaces_colliding_scalar_values():
     node_values = [
         (
@@ -483,7 +652,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
                         module.ruamel.yaml.nodes.SequenceNode(
-                            tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2']
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
                         ),
                     ),
                 ],
@@ -499,7 +676,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
                         module.ruamel.yaml.nodes.SequenceNode(
-                            tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4']
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 4'
+                                ),
+                            ],
                         ),
                     ),
                 ],
@@ -514,10 +699,10 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
     options = section_value.value
     assert len(options) == 1
     assert options[0][0].value == 'before_backup'
-    assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
+    assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
 
 
-def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain():
+def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain():
     node_values = [
         (
             module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
@@ -568,7 +753,7 @@ def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain():
     assert options[0][1].value == '5'
 
 
-def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
+def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain():
     node_values = [
         (
             module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
@@ -580,7 +765,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
                         module.ruamel.yaml.nodes.SequenceNode(
-                            tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2']
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
                         ),
                     ),
                 ],
@@ -596,7 +789,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
                         module.ruamel.yaml.nodes.SequenceNode(
-                            tag='!retain', value=['echo 3', 'echo 4']
+                            tag='!retain',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 4'
+                                ),
+                            ],
                         ),
                     ),
                 ],
@@ -612,4 +813,64 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
     assert len(options) == 1
     assert options[0][0].value == 'before_backup'
     assert options[0][1].tag == 'tag:yaml.org,2002:seq'
-    assert options[0][1].value == ['echo 3', 'echo 4']
+    assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4']
+
+
+def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit():
+    node_values = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'hooks'
+    options = section_value.value
+    assert len(options) == 1
+    assert options[0][0].value == 'before_backup'
+    assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3']