diff --git a/.gitignore b/.gitignore index 01333d5b1..c597d9c84 100644 --- a/.gitignore +++ b/.gitignore @@ -1,153 +1,155 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg .idea/ .idea .idea/* *.iml # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ cover # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # dotenv .env # virtualenv .venv venv/ ENV/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # Autogenerated files templates-cfg/* templates-op/* tests/templates/* # Debian packaging debian/files debian/tmp debian/debhelper-build-stamp debian/.debhelper/ debian/vyos-1x debian/vyos-1x-vmware debian/vyos-1x-smoketest debian/*.postinst.debhelper debian/*.prerm.debhelper debian/*.postrm.debhelper debian/*.substvars # Sonar Cloud .scannerwork /.vs # SlickEdit *.vpj *.vpw *.vpwhist *.vtg # VS Code .vscode/* !.vscode/settings.json # VIM *.swp # vyos-1x JSON version data/component-versions.json # vyos-1x XML cache python/vyos/xml_ref/cache.py python/vyos/xml_ref/pkg_cache/*_cache.py +python/vyos/xml_ref/op_cache.py +python/vyos/xml_ref/pkg_cache/*_op_cache.py # autogenerated vyos-configd JSON definition data/configd-include.json # We do not use pip Pipfile Pipfile.lock diff --git a/Makefile b/Makefile index 685c8f150..c83380be5 100644 --- a/Makefile +++ b/Makefile @@ -1,120 +1,122 @@ TMPL_DIR := templates-cfg OP_TMPL_DIR := templates-op BUILD_DIR := build DATA_DIR := data SHIM_DIR := src/shim LIBS := -lzmq CFLAGS := BUILD_ARCH := $(shell dpkg-architecture -q DEB_BUILD_ARCH) J2LINT := $(shell command -v j2lint 2> /dev/null) PYLINT_FILES := $(shell git ls-files *.py src/migration-scripts) config_xml_src = $(wildcard interface-definitions/*.xml.in) config_xml_obj = $(config_xml_src:.xml.in=.xml) op_xml_src = $(wildcard op-mode-definitions/*.xml.in) op_xml_obj = $(op_xml_src:.xml.in=.xml) %.xml: %.xml.in @echo Generating $(BUILD_DIR)/$@ from $< mkdir -p $(BUILD_DIR)/$(dir $@) $(CURDIR)/scripts/transclude-template $< > $(BUILD_DIR)/$@ .PHONY: interface_definitions .ONESHELL: interface_definitions: $(config_xml_obj) mkdir -p $(TMPL_DIR) $(CURDIR)/scripts/override-default $(BUILD_DIR)/interface-definitions find $(BUILD_DIR)/interface-definitions -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-templates {} $(CURDIR)/schema/interface_definition.rng $(TMPL_DIR) || exit 1 $(CURDIR)/python/vyos/xml_ref/generate_cache.py --xml-dir $(BUILD_DIR)/interface-definitions || exit 1 # XXX: delete top level node.def's that now live in other packages # IPSec VPN EAP-RADIUS does not support source-address rm -rf $(TMPL_DIR)/vpn/ipsec/remote-access/radius/source-address # T2472 - EIGRP support rm -rf $(TMPL_DIR)/protocols/eigrp # T2773 - EIGRP support for VRF rm -rf $(TMPL_DIR)/vrf/name/node.tag/protocols/eigrp # XXX: test if there are empty node.def files - this is not allowed as these # could mask help strings or mandatory priority statements find $(TMPL_DIR) -name node.def -type f -empty -exec false {} + || sh -c 'echo "There are empty node.def files! Check your interface definitions." && exit 1' ifeq ($(BUILD_ARCH),arm64) # There is currently no telegraf support in VyOS for ARM64, remove CLI definitions rm -rf $(TMPL_DIR)/service/monitoring/telegraf endif .PHONY: op_mode_definitions .ONESHELL: op_mode_definitions: $(op_xml_obj) mkdir -p $(OP_TMPL_DIR) find $(BUILD_DIR)/op-mode-definitions/ -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-op-templates {} $(CURDIR)/schema/op-mode-definition.rng $(OP_TMPL_DIR) || exit 1 + $(CURDIR)/python/vyos/xml_ref/generate_op_cache.py --xml-dir $(BUILD_DIR)/op-mode-definitions || exit 1 + # XXX: tcpdump, ping, traceroute and mtr must be able to recursivly call themselves as the # options are provided from the scripts themselves ln -s ../node.tag $(OP_TMPL_DIR)/ping/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/traceroute/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/mtr/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/monitor/traceroute/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/monitor/traffic/interface/node.tag/node.tag/ # XXX: test if there are empty node.def files - this is not allowed as these # could mask help strings or mandatory priority statements find $(OP_TMPL_DIR) -name node.def -type f -empty -exec false {} + || sh -c 'echo "There are empty node.def files! Check your interface definitions." && exit 1' .PHONY: vyshim vyshim: $(MAKE) -C $(SHIM_DIR) .PHONY: all all: clean interface_definitions op_mode_definitions test j2lint vyshim generate-configd-include-json .PHONY: clean clean: rm -rf $(BUILD_DIR) rm -rf $(TMPL_DIR) rm -rf $(OP_TMPL_DIR) $(MAKE) -C $(SHIM_DIR) clean .PHONY: test test: generate-configd-include-json set -e; python3 -m compileall -q -x '/vmware-tools/scripts/, /ppp/' . PYTHONPATH=python/ python3 -m "nose" --with-xunit src --with-coverage --cover-erase --cover-xml --cover-package src/conf_mode,src/op_mode,src/completion,src/helpers,src/validators,src/tests --verbose .PHONY: check_migration_scripts_executable .ONESHELL: check_migration_scripts_executable: @echo "Checking if migration scripts have executable bit set..." find src/migration-scripts -type f -not -executable -print -exec false {} + || sh -c 'echo "Found files that are not executable! Add permissions." && exit 1' .PHONY: j2lint j2lint: ifndef J2LINT $(error "j2lint binary not found, consider installing: pip install git+https://github.com/aristanetworks/j2lint.git@341b5d5db86") endif $(J2LINT) data/ .PHONY: sonar sonar: sonar-scanner -X -Dsonar.login=${SONAR_TOKEN} .PHONY: unused-imports unused-imports: @pylint --disable=all --enable=W0611 $(PYLINT_FILES) deb: dpkg-buildpackage -uc -us -tc -b .PHONY: generate-configd-include-json generate-configd-include-json: @scripts/generate-configd-include-json.py .PHONY: schema schema: trang -I rnc -O rng schema/interface_definition.rnc schema/interface_definition.rng trang -I rnc -O rng schema/op-mode-definition.rnc schema/op-mode-definition.rng diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 25ee45391..dec619d3e 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -1,62 +1,63 @@ # Copyright 2018-2024 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. import os base_dir = '/usr/libexec/vyos/' directories = { 'base' : base_dir, 'data' : '/usr/share/vyos/', 'conf_mode' : f'{base_dir}/conf_mode', 'op_mode' : f'{base_dir}/op_mode', 'services' : f'{base_dir}/services', 'config' : '/opt/vyatta/etc/config', 'migrate' : '/opt/vyatta/etc/config-migrate/migrate', 'activate' : f'{base_dir}/activate', 'log' : '/var/log/vyatta', 'templates' : '/usr/share/vyos/templates/', 'certbot' : '/config/auth/letsencrypt', 'api_schema': f'{base_dir}/services/api/graphql/graphql/schema/', 'api_client_op': f'{base_dir}/services/api/graphql/graphql/client_op/', 'api_templates': f'{base_dir}/services/api/graphql/session/templates/', 'vyos_udev_dir' : '/run/udev/vyos', 'isc_dhclient_dir' : '/run/dhclient', 'dhcp6_client_dir' : '/run/dhcp6c', - 'vyos_configdir' : '/opt/vyatta/config' + 'vyos_configdir' : '/opt/vyatta/config', + 'completion_dir' : f'{base_dir}/completion' } config_status = '/tmp/vyos-config-status' api_config_state = '/run/http-api-state' cfg_group = 'vyattacfg' cfg_vintage = 'vyos' commit_lock = os.path.join(directories['vyos_configdir'], '.lock') component_version_json = os.path.join(directories['data'], 'component-versions.json') config_default = os.path.join(directories['data'], 'config.boot.default') rt_symbolic_names = { # Standard routing tables for Linux & reserved IDs for VyOS 'default': 253, # Confusingly, a final fallthru, not the default. 'main': 254, # The actual global table used by iproute2 unless told otherwise. 'local': 255, # Special kernel loopback table. } rt_global_vrf = rt_symbolic_names['main'] rt_global_table = rt_symbolic_names['main'] diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index 2ba3da4e8..91ce394f7 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -1,89 +1,112 @@ # Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. from typing import Optional, Union, TYPE_CHECKING from vyos.xml_ref import definition +from vyos.xml_ref import op_definition if TYPE_CHECKING: from vyos.config import ConfigDict def load_reference(cache=[]): if cache: return cache[0] xml = definition.Xml() try: from vyos.xml_ref.cache import reference except Exception: raise ImportError('no xml reference cache !!') if not reference: raise ValueError('empty xml reference cache !!') xml.define(reference) cache.append(xml) return xml def is_tag(path: list) -> bool: return load_reference().is_tag(path) def is_tag_value(path: list) -> bool: return load_reference().is_tag_value(path) def is_multi(path: list) -> bool: return load_reference().is_multi(path) def is_valueless(path: list) -> bool: return load_reference().is_valueless(path) def is_leaf(path: list) -> bool: return load_reference().is_leaf(path) def owner(path: list) -> str: return load_reference().owner(path) def priority(path: list) -> str: return load_reference().priority(path) def cli_defined(path: list, node: str, non_local=False) -> bool: return load_reference().cli_defined(path, node, non_local=non_local) def component_version() -> dict: return load_reference().component_version() def default_value(path: list) -> Optional[Union[str, list]]: return load_reference().default_value(path) def multi_to_list(rpath: list, conf: dict) -> dict: return load_reference().multi_to_list(rpath, conf) def get_defaults(path: list, get_first_key=False, recursive=False) -> dict: return load_reference().get_defaults(path, get_first_key=get_first_key, recursive=recursive) def relative_defaults(rpath: list, conf: dict, get_first_key=False, recursive=False) -> dict: return load_reference().relative_defaults(rpath, conf, get_first_key=get_first_key, recursive=recursive) def from_source(d: dict, path: list) -> bool: return definition.from_source(d, path) def ext_dict_merge(source: dict, destination: Union[dict, 'ConfigDict']): return definition.ext_dict_merge(source, destination) + +def load_op_reference(op_cache=[]): + if op_cache: + return op_cache[0] + + op_xml = op_definition.OpXml() + + try: + from vyos.xml_ref.op_cache import op_reference + except Exception: + raise ImportError('no xml op reference cache !!') + + if not op_reference: + raise ValueError('empty xml op reference cache !!') + + op_xml.define(op_reference) + op_cache.append(op_xml) + + return op_xml + +def get_op_ref_path(path: list) -> list[op_definition.PathData]: + return load_op_reference()._get_op_ref_path(path) diff --git a/python/vyos/xml_ref/generate_op_cache.py b/python/vyos/xml_ref/generate_op_cache.py new file mode 100755 index 000000000..e93b07974 --- /dev/null +++ b/python/vyos/xml_ref/generate_op_cache.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# + +import re +import sys +import json +import glob +from argparse import ArgumentParser +from argparse import ArgumentTypeError +from os.path import join +from os.path import abspath +from os.path import dirname +from xml.etree import ElementTree as ET +from xml.etree.ElementTree import Element +from typing import TypeAlias +from typing import Optional + +_here = dirname(__file__) + +sys.path.append(join(_here, '..')) +from defaults import directories + +from op_definition import NodeData +from op_definition import PathData + +xml_op_cache_json = 'xml_op_cache.json' +xml_op_tmp = join('/tmp', xml_op_cache_json) +op_ref_cache = abspath(join(_here, 'op_cache.py')) + +OptElement: TypeAlias = Optional[Element] +DEBUG = False + + +def translate_exec(s: str) -> str: + s = s.replace('${vyos_op_scripts_dir}', directories['op_mode']) + s = s.replace('${vyos_libexec_dir}', directories['base']) + return s + + +def translate_position(s: str, pos: list[str]) -> str: + pos = pos.copy() + pat: re.Pattern = re.compile(r'(?:\")?\${?([0-9]+)}?(?:\")?') + t: str = pat.sub(r'_place_holder_\1_', s) + + # preferred to .format(*list) to avoid collisions with braces + for i, p in enumerate(pos): + t = t.replace(f'_place_holder_{i+1}_', p) + + return t + + +def translate_command(s: str, pos: list[str]) -> str: + s = translate_exec(s) + s = translate_position(s, pos) + return s + + +def translate_op_script(s: str) -> str: + s = s.replace('${vyos_completion_dir}', directories['completion_dir']) + s = s.replace('${vyos_op_scripts_dir}', directories['op_mode']) + return s + + +def insert_node(n: Element, l: list[PathData], path = None) -> None: + # pylint: disable=too-many-locals,too-many-branches + prop: OptElement = n.find('properties') + children: OptElement = n.find('children') + command: OptElement = n.find('command') + # name is not None as required by schema + name: str = n.get('name', 'schema_error') + node_type: str = n.tag + if path is None: + path = [] + + path.append(name) + if node_type == 'tagNode': + path.append(f'{name}-tag_value') + + help_prop: OptElement = None if prop is None else prop.find('help') + help_text = None if help_prop is None else help_prop.text + command_text = None if command is None else command.text + if command_text is not None: + command_text = translate_command(command_text, path) + + comp_help = None + if prop is not None: + che = prop.findall("completionHelp") + for c in che: + lists = c.findall("list") + paths = c.findall("path") + scripts = c.findall("script") + + comp_help = {} + list_l = [] + for i in lists: + list_l.append(i.text) + path_l = [] + for i in paths: + path_str = re.sub(r'\s+', '/', i.text) + path_l.append(path_str) + script_l = [] + for i in scripts: + script_str = translate_op_script(i.text) + script_l.append(script_str) + + comp_help['list'] = list_l + comp_help['fs_path'] = path_l + comp_help['script'] = script_l + + for d in l: + if name in list(d): + break + else: + d = {} + l.append(d) + + inner_l = d.setdefault(name, []) + + inner_d: PathData = {'node_data': NodeData(node_type=node_type, + help_text=help_text, + comp_help=comp_help, + command=command_text, + path=path)} + inner_l.append(inner_d) + + if children is not None: + inner_nodes = children.iterfind("*") + for inner_n in inner_nodes: + inner_path = path[:] + insert_node(inner_n, inner_l, inner_path) + + +def parse_file(file_path, l): + tree = ET.parse(file_path) + root = tree.getroot() + for n in root.iterfind("*"): + insert_node(n, l) + + +def main(): + parser = ArgumentParser(description='generate dict from xml defintions') + parser.add_argument('--xml-dir', type=str, required=True, + help='transcluded xml op-mode-definition file') + + args = vars(parser.parse_args()) + + xml_dir = abspath(args['xml_dir']) + + l = [] + + for fname in glob.glob(f'{xml_dir}/*.xml'): + parse_file(fname, l) + + with open(xml_op_tmp, 'w') as f: + json.dump(l, f, indent=2) + + with open(op_ref_cache, 'w') as f: + f.write(f'op_reference = {str(l)}') + +if __name__ == '__main__': + main() diff --git a/python/vyos/xml_ref/op_definition.py b/python/vyos/xml_ref/op_definition.py new file mode 100644 index 000000000..914f3a105 --- /dev/null +++ b/python/vyos/xml_ref/op_definition.py @@ -0,0 +1,49 @@ +# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +from typing import TypedDict +from typing import TypeAlias +from typing import Optional +from typing import Union + + +class NodeData(TypedDict): + node_type: Optional[str] + help_text: Optional[str] + comp_help: Optional[dict[str, list]] + command: Optional[str] + path: Optional[list[str]] + + +PathData: TypeAlias = dict[str, Union[NodeData|list['PathData']]] + + +class OpXml: + def __init__(self): + self.op_ref = {} + + def define(self, op_ref: list[PathData]) -> None: + self.op_ref = op_ref + + def _get_op_ref_path(self, path: list[str]) -> list[PathData]: + def _get_path_list(path: list[str], l: list[PathData]) -> list[PathData]: + if not path: + return l + for d in l: + if path[0] in list(d): + return _get_path_list(path[1:], d[path[0]]) + return [] + l = self.op_ref + return _get_path_list(path, l)