diff --git a/changelogs/fragments/vyos_config_improved_diffing.yml b/changelogs/fragments/vyos_config_improved_diffing.yml new file mode 100644 index 00000000..94b8a9a5 --- /dev/null +++ b/changelogs/fragments/vyos_config_improved_diffing.yml @@ -0,0 +1,3 @@ +--- +minor_changes: + - vyos_config - Added new `match`-value `smart` with updated configuration diffing for more reproducible provisioning of devices. diff --git a/plugins/cliconf_utils/vyosconf.py b/plugins/cliconf_utils/vyosconf.py index 404948ed..d0849852 100644 --- a/plugins/cliconf_utils/vyosconf.py +++ b/plugins/cliconf_utils/vyosconf.py @@ -1,220 +1,222 @@ # # This file is part of Ansible # # Ansible is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Ansible is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . # from __future__ import absolute_import, division, print_function __metaclass__ = type import re +KEEP_EXISTING_VALUES = "..." + class VyosConf: def __init__(self, commands=None): self.config = {} if type(commands) is list: self.run_commands(commands) def set_entry(self, path, leaf): """ This function sets a value in the configuration given a path. :param path: list of strings to traveser in the config :param leaf: value to set at the destination :return: dict """ target = self.config path = path + [leaf] for key in path: if key not in target or type(target[key]) is not dict: target[key] = {} target = target[key] return self.config def del_entry(self, path, leaf): """ This function deletes a value from the configuration given a path and also removes all the parents that are now empty. :param path: list of strings to traveser in the config :param leaf: value to delete at the destination :return: dict """ target = self.config firstNoSiblingKey = None for key in path: if key not in target: return self.config if len(target[key]) <= 1: if firstNoSiblingKey is None: firstNoSiblingKey = [target, key] else: firstNoSiblingKey = None target = target[key] if firstNoSiblingKey is None: firstNoSiblingKey = [target, leaf] target = firstNoSiblingKey[0] targetKey = firstNoSiblingKey[1] del target[targetKey] return self.config def check_entry(self, path, leaf): """ This function checks if a value exists in the config. :param path: list of strings to traveser in the config :param leaf: value to check for existence :return: bool """ target = self.config path = path + [leaf] existing = [] for key in path: if key not in target or type(target[key]) is not dict: return False existing.append(key) target = target[key] return True def parse_line(self, line): """ This function parses a given command from string. :param line: line to parse :return: [command, path, leaf] """ line = ( re.match(r"^('(.*)'|\"(.*)\"|([^#\"']*))*", line).group(0).strip() ) path = re.findall(r"('.*?'|\".*?\"|\S+)", line) leaf = path[-1] if leaf.startswith('"') and leaf.endswith('"'): leaf = leaf[1:-1] if leaf.startswith("'") and leaf.endswith("'"): leaf = leaf[1:-1] return [path[0], path[1:-1], leaf] def run_command(self, command): """ This function runs a given command string. :param command: command to run :return: dict """ [cmd, path, leaf] = self.parse_line(command) if cmd.startswith("set"): self.set_entry(path, leaf) if cmd.startswith("del"): self.del_entry(path, leaf) return self.config def run_commands(self, commands): """ This function runs a a list of command strings. :param commands: commands to run :return: dict """ for c in commands: self.run_command(c) return self.config def check_command(self, command): """ This function checkes a command for existance in the config. :param command: command to check :return: bool """ [cmd, path, leaf] = self.parse_line(command) if cmd.startswith("set"): return self.check_entry(path, leaf) if cmd.startswith("del"): return not self.check_entry(path, leaf) return True def check_commands(self, commands): """ This function checkes a list of commands for existance in the config. :param commands: list of commands to check :return: [bool] """ return [self.check_command(c) for c in commands] def build_commands(self, structure=None, nested=False): """ This function builds a list of commands to recreate the current configuration. :return: [str] """ if type(structure) is not dict: structure = self.config if len(structure) == 0: return [""] if nested else [] commands = [] for (key, value) in structure.items(): for c in self.build_commands(value, True): if " " in key or '"' in key: key = "'" + key + "'" commands.append((key + " " + c).strip()) if nested: return commands return ["set " + c for c in commands] def diff_to(self, other, structure): if type(other) is not dict: other = {} if len(structure) == 0: return ([], [""]) if type(structure) is not dict: structure = {} if len(other) == 0: return ([""], []) if len(other) == 0 and len(structure) == 0: return ([], []) toset = [] todel = [] for key in structure.keys(): quoted_key = "'" + key + "'" if " " in key or '"' in key else key if key in other: # keys in both configs, pls compare subkeys (subset, subdel) = self.diff_to(other[key], structure[key]) for s in subset: toset.append(quoted_key + " " + s) - if "!" not in other[key]: + if KEEP_EXISTING_VALUES not in other[key]: for d in subdel: todel.append(quoted_key + " " + d) else: # keys only in this, pls del todel.append(quoted_key) continue # del for (key, value) in other.items(): - if key == "!": + if key == KEEP_EXISTING_VALUES: continue quoted_key = "'" + key + "'" if " " in key or '"' in key else key if key not in structure: # keys only in other, pls set all subkeys (subset, subdel) = self.diff_to(other[key], None) for s in subset: toset.append(quoted_key + " " + s) return (toset, todel) def diff_commands_to(self, other): """ This function calculates the required commands to change the current into the given configuration. :param other: VyosConf :return: [str] """ (toset, todel) = self.diff_to(other.config, self.config) return ["delete " + c.strip() for c in todel] + [ "set " + c.strip() for c in toset ] diff --git a/plugins/modules/vyos_config.py b/plugins/modules/vyos_config.py index 6c737f09..136b7b83 100644 --- a/plugins/modules/vyos_config.py +++ b/plugins/modules/vyos_config.py @@ -1,395 +1,397 @@ #!/usr/bin/python # # This file is part of Ansible # # Ansible is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Ansible is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . # from __future__ import absolute_import, division, print_function __metaclass__ = type DOCUMENTATION = """ module: vyos_config author: Nathaniel Case (@Qalthos) short_description: Manage VyOS configuration on remote device description: - This module provides configuration file management of VyOS devices. It provides arguments for managing both the configuration file and state of the active configuration. All configuration statements are based on `set` and `delete` commands in the device configuration. version_added: 1.0.0 extends_documentation_fragment: - vyos.vyos.vyos notes: - Tested against VyOS 1.1.8 (helium). - This module works with connection C(network_cli). See L(the VyOS OS Platform Options,../network/user_guide/platform_vyos.html). - To ensure idempotency and correct diff the configuration lines in the relevant module options should be similar to how they appear if present in the running configuration on device including the indentation. options: lines: description: - The ordered set of commands that should be configured in the section. The commands must be the exact same commands as found in the device running-config as found in the device running-config to ensure idempotency and correct diff. Be sure to note the configuration command syntax as some commands are automatically modified by the device config parser. type: list elements: str src: description: - The C(src) argument specifies the path to the source config file to load. The source config file can either be in bracket format or set format. The source file can include Jinja2 template variables. The configuration lines in the source file should be similar to how it will appear if present in the running-configuration of the device including indentation to ensure idempotency and correct diff. type: path match: description: - The C(match) argument controls the method used to match against the current active configuration. By default, the configuration commands config are matched against the active config and the deltas are loaded line by line. If the C(match) argument is set to C(none) the active configuration is ignored - and the configuration is always loaded. If the C(match) argument is set to C(smart) - both the active configuration and the target configuration are simlulated + and the configuration is always loaded. If the C(match) argument is set to + C(smart) both the active configuration and the target configuration are simlulated and the results compared to bring the target device into a reliable and - reproducable state. + reproducable state. Using C(smart), the special value C(...) indicates that + this keys value should not be changed and any preexisting siblings should not + be removed from the target. type: str default: line choices: - line - smart - none backup: description: - The C(backup) argument will backup the current devices active configuration to the Ansible control host prior to making any changes. If the C(backup_options) value is not given, the backup file will be located in the backup folder in the playbook root directory or role root directory, if playbook is part of an ansible role. If the directory does not exist, it is created. type: bool default: no comment: description: - Allows a commit description to be specified to be included when the configuration is committed. If the configuration is not changed or committed, this argument is ignored. default: configured by vyos_config type: str config: description: - The C(config) argument specifies the base configuration to use to compare against the desired configuration. If this value is not specified, the module will automatically retrieve the current active configuration from the remote device. The configuration lines in the option value should be similar to how it will appear if present in the running-configuration of the device including indentation to ensure idempotency and correct diff. type: str save: description: - The C(save) argument controls whether or not changes made to the active configuration are saved to disk. This is independent of committing the config. When set to True, the active configuration is saved. type: bool default: no backup_options: description: - This is a dict object containing configurable options related to backup file path. The value of this option is read only when C(backup) is set to I(yes), if C(backup) is set to I(no) this option will be silently ignored. suboptions: filename: description: - The filename to be used to store the backup configuration. If the filename is not given it will be generated based on the hostname, current time and date in format defined by _config.@ type: str dir_path: description: - This option provides the path ending with directory name in which the backup configuration file will be stored. If the directory does not exist it will be first created and the filename is either the value of C(filename) or default filename as described in C(filename) options description. If the path value is not given in that case a I(backup) directory will be created in the current working directory and backup configuration will be copied in C(filename) within I(backup) directory. type: path type: dict """ EXAMPLES = """ - name: configure the remote device vyos.vyos.vyos_config: lines: - set system host-name {{ inventory_hostname }} - set service lldp - delete service dhcp-server - name: backup and load from file vyos.vyos.vyos_config: src: vyos.cfg backup: yes - name: render a Jinja2 template onto the VyOS router vyos.vyos.vyos_config: match: smart src: vyos_template.j2 - name: for idempotency, use full-form commands vyos.vyos.vyos_config: lines: # - set int eth eth2 description 'OUTSIDE' - set interface ethernet eth2 description 'OUTSIDE' - name: configurable backup path vyos.vyos.vyos_config: backup: yes backup_options: filename: backup.cfg dir_path: /home/user """ RETURN = """ commands: description: The list of configuration commands sent to the device returned: always type: list sample: ['...', '...'] filtered: description: The list of configuration commands removed to avoid a load failure returned: always type: list sample: ['...', '...'] backup_path: description: The full path to the backup file returned: when backup is yes type: str sample: /playbooks/ansible/backup/vyos_config.2016-07-16@22:28:34 filename: description: The name of the backup file returned: when backup is yes and filename is not specified in backup options type: str sample: vyos_config.2016-07-16@22:28:34 shortname: description: The full path to the backup file excluding the timestamp returned: when backup is yes and filename is not specified in backup options type: str sample: /playbooks/ansible/backup/vyos_config date: description: The date extracted from the backup file name returned: when backup is yes type: str sample: "2016-07-16" time: description: The time extracted from the backup file name returned: when backup is yes type: str sample: "22:28:34" """ import re from ansible.module_utils._text import to_text from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.connection import ConnectionError from ansible_collections.vyos.vyos.plugins.module_utils.network.vyos.vyos import ( load_config, get_config, run_commands, ) from ansible_collections.vyos.vyos.plugins.module_utils.network.vyos.vyos import ( vyos_argument_spec, get_connection, ) DEFAULT_COMMENT = "configured by vyos_config" CONFIG_FILTERS = [ re.compile(r"set system login user \S+ authentication encrypted-password") ] def get_candidate(module): contents = module.params["src"] or module.params["lines"] if module.params["src"]: contents = contents.splitlines() if len(contents) > 0: line = contents[0].split() if len(line) > 0 and line[0] in ("set", "delete"): contents = format_commands(contents) contents = "\n".join(contents) return contents def format_commands(commands): """ This function format the input commands and removes the prepend white spaces for command lines having 'set' or 'delete' and it skips empty lines. :param commands: :return: list of commands """ return [ line.strip() if line.split()[0] in ("set", "delete") else line for line in commands if len(line.strip()) > 0 ] def diff_config(commands, config): config = [str(c).replace("'", "") for c in config.splitlines()] updates = list() visited = set() for line in commands: item = str(line).replace("'", "") if not item.startswith("set") and not item.startswith("delete"): raise ValueError("line must start with either `set` or `delete`") elif item.startswith("set") and item not in config: updates.append(line) elif item.startswith("delete"): if not config: updates.append(line) else: item = re.sub(r"delete", "set", item) for entry in config: if entry.startswith(item) and line not in visited: updates.append(line) visited.add(line) return list(updates) def sanitize_config(config, result): result["filtered"] = list() index_to_filter = list() for regex in CONFIG_FILTERS: for index, line in enumerate(list(config)): if regex.search(line): result["filtered"].append(line) index_to_filter.append(index) # Delete all filtered configs for filter_index in sorted(index_to_filter, reverse=True): del config[filter_index] def run(module, result): # get the current active config from the node or passed in via # the config param config = module.params["config"] or get_config(module) # create the candidate config object from the arguments candidate = get_candidate(module) # create loadable config that includes only the configuration updates connection = get_connection(module) try: response = connection.get_diff( candidate=candidate, running=config, diff_match=module.params["match"], ) except ConnectionError as exc: module.fail_json(msg=to_text(exc, errors="surrogate_then_replace")) commands = response.get("config_diff") sanitize_config(commands, result) result["commands"] = commands commit = not module.check_mode comment = module.params["comment"] diff = None if commands: diff = load_config(module, commands, commit=commit, comment=comment) if result.get("filtered"): result["warnings"].append( "Some configuration commands were " "removed, please see the filtered key" ) result["changed"] = True if module._diff: result["diff"] = {"prepared": diff} def main(): backup_spec = dict(filename=dict(), dir_path=dict(type="path")) argument_spec = dict( src=dict(type="path"), lines=dict(type="list", elements="str"), match=dict(default="line", choices=["line", "smart", "none"]), comment=dict(default=DEFAULT_COMMENT), config=dict(), backup=dict(type="bool", default=False), backup_options=dict(type="dict", options=backup_spec), save=dict(type="bool", default=False), ) argument_spec.update(vyos_argument_spec) mutually_exclusive = [("lines", "src")] module = AnsibleModule( argument_spec=argument_spec, mutually_exclusive=mutually_exclusive, supports_check_mode=True, ) warnings = list() result = dict(changed=False, warnings=warnings) if module.params["backup"]: result["__backup__"] = get_config(module=module) if any((module.params["src"], module.params["lines"])): run(module, result) if module.params["save"]: diff = run_commands(module, commands=["configure", "compare saved"])[1] if diff != "[edit]": if not module.check_mode: run_commands(module, commands=["save"]) result["changed"] = True run_commands(module, commands=["exit"]) if result.get("changed") and any( (module.params["src"], module.params["lines"]) ): msg = ( "To ensure idempotency and correct diff the input configuration lines should be" " similar to how they appear if present in" " the running configuration on device" ) if module.params["src"]: msg += " including the indentation" if "warnings" in result: result["warnings"].append(msg) else: result["warnings"] = msg module.exit_json(**result) if __name__ == "__main__": main() diff --git a/tests/unit/cliconf/test_utils_vyosconf.py b/tests/unit/cliconf/test_utils_vyosconf.py index 34b49c38..9d354968 100644 --- a/tests/unit/cliconf/test_utils_vyosconf.py +++ b/tests/unit/cliconf/test_utils_vyosconf.py @@ -1,160 +1,160 @@ # # This file is part of Ansible # # Ansible is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Ansible is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . # from __future__ import absolute_import, division, print_function __metaclass__ = type import unittest from ansible_collections.vyos.vyos.plugins.cliconf_utils.vyosconf import ( VyosConf, ) class TestListElements(unittest.TestCase): def test_add(self): conf = VyosConf() conf.set_entry(["a", "b"], "c") self.assertEqual(conf.config, {"a": {"b": {"c": {}}}}) conf.set_entry(["a", "b"], "d") self.assertEqual(conf.config, {"a": {"b": {"c": {}, "d": {}}}}) conf.set_entry(["a", "c"], "b") self.assertEqual( conf.config, {"a": {"b": {"c": {}, "d": {}}, "c": {"b": {}}}} ) conf.set_entry(["a", "c", "b"], "d") self.assertEqual( conf.config, {"a": {"b": {"c": {}, "d": {}}, "c": {"b": {"d": {}}}}}, ) def test_del(self): conf = VyosConf() conf.set_entry(["a", "b"], "c") conf.set_entry(["a", "c", "b"], "d") conf.set_entry(["a", "b"], "d") self.assertEqual( conf.config, {"a": {"b": {"c": {}, "d": {}}, "c": {"b": {"d": {}}}}}, ) conf.del_entry(["a", "c", "b"], "d") self.assertEqual(conf.config, {"a": {"b": {"c": {}, "d": {}}}}) conf.set_entry(["a", "b", "c"], "d") conf.del_entry(["a", "b", "c"], "d") self.assertEqual(conf.config, {"a": {"b": {"d": {}}}}) def test_parse(self): conf = VyosConf() self.assertListEqual( conf.parse_line("set a b c"), ["set", ["a", "b"], "c"] ) self.assertListEqual( conf.parse_line('set a b "c"'), ["set", ["a", "b"], "c"] ) self.assertListEqual( conf.parse_line("set a b 'c d'"), ["set", ["a", "b"], "c d"] ) self.assertListEqual( conf.parse_line("set a b 'c'"), ["set", ["a", "b"], "c"] ) self.assertListEqual( conf.parse_line("delete a b 'c'"), ["delete", ["a", "b"], "c"] ) self.assertListEqual( conf.parse_line("del a b 'c'"), ["del", ["a", "b"], "c"] ) self.assertListEqual( conf.parse_line("set a b '\"c'"), ["set", ["a", "b"], '"c'] ) self.assertListEqual( conf.parse_line("set a b 'c' #this is a comment"), ["set", ["a", "b"], "c"], ) self.assertListEqual( conf.parse_line("set a b '#c'"), ["set", ["a", "b"], "#c"] ) def test_run_commands(self): self.assertEqual( VyosConf(["set a b 'c'", "set a c 'b'"]).config, {"a": {"b": {"c": {}}, "c": {"b": {}}}}, ) self.assertEqual( VyosConf(["set a b c 'd'", "set a c 'b'", "del a b c d"]).config, {"a": {"c": {"b": {}}}}, ) def test_build_commands(self): self.assertEqual( sorted( VyosConf( [ "set a b 'c a'", "set a c a", "set a c b", "delete a c a", ] ).build_commands() ), sorted(["set a b 'c a'", "set a c b"]), ) def test_check_commands(self): conf = VyosConf(["set a b 'c a'", "set a c b"]) self.assertListEqual( conf.check_commands( ["set a b 'c a'", "del a c b", "set a b 'c'", "del a a a"] ), [True, False, False, True], ) def test_diff_commands_to(self): conf = VyosConf(["set a b 'c a'", "set a c b"]) self.assertListEqual( conf.diff_commands_to(VyosConf(["set a c b"])), ["delete a b"] ) self.assertListEqual( conf.diff_commands_to(VyosConf(["set a b 'c a'", "set a c b"])), [] ) self.assertListEqual( conf.diff_commands_to( VyosConf( [ - "set a b !", + "set a b ...", ] ) ), ["delete a c"], ) self.assertListEqual( - conf.diff_commands_to(VyosConf(["set a !", "set a d e"])), + conf.diff_commands_to(VyosConf(["set a ...", "set a d e"])), ["set a d e"], ) self.assertListEqual( conf.diff_commands_to(VyosConf(["set a b", "set a c b"])), ["delete a b 'c a'"], ) self.assertListEqual( conf.diff_commands_to(VyosConf(["set a b 'a c'", "set a c b"])), ["delete a b 'c a'", "set a b 'a c'"], ) if __name__ == "__main__": unittest.main()