diff --git a/.gitignore b/.gitignore index 507daceee..01333d5b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,151 +1,153 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg .idea/ .idea .idea/* *.iml # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ cover # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # dotenv .env # virtualenv .venv venv/ ENV/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # Autogenerated files templates-cfg/* templates-op/* tests/templates/* # Debian packaging debian/files debian/tmp debian/debhelper-build-stamp debian/.debhelper/ debian/vyos-1x debian/vyos-1x-vmware debian/vyos-1x-smoketest debian/*.postinst.debhelper debian/*.prerm.debhelper debian/*.postrm.debhelper debian/*.substvars # Sonar Cloud .scannerwork /.vs # SlickEdit *.vpj *.vpw *.vpwhist *.vtg # VS Code .vscode/* !.vscode/settings.json # VIM *.swp # vyos-1x JSON version data/component-versions.json # vyos-1x XML cache python/vyos/xml_ref/cache.py python/vyos/xml_ref/pkg_cache/*_cache.py +# autogenerated vyos-configd JSON definition +data/configd-include.json # We do not use pip Pipfile Pipfile.lock diff --git a/Makefile b/Makefile index 509b47858..1e0a12714 100644 --- a/Makefile +++ b/Makefile @@ -1,133 +1,126 @@ TMPL_DIR := templates-cfg OP_TMPL_DIR := templates-op BUILD_DIR := build DATA_DIR := data SHIM_DIR := src/shim LIBS := -lzmq CFLAGS := BUILD_ARCH := $(shell dpkg-architecture -q DEB_BUILD_ARCH) J2LINT := $(shell command -v j2lint 2> /dev/null) PYLINT_FILES := $(shell git ls-files *.py src/migration-scripts) config_xml_src = $(wildcard interface-definitions/*.xml.in) config_xml_obj = $(config_xml_src:.xml.in=.xml) op_xml_src = $(wildcard op-mode-definitions/*.xml.in) op_xml_obj = $(op_xml_src:.xml.in=.xml) %.xml: %.xml.in @echo Generating $(BUILD_DIR)/$@ from $< mkdir -p $(BUILD_DIR)/$(dir $@) $(CURDIR)/scripts/transclude-template $< > $(BUILD_DIR)/$@ .PHONY: interface_definitions .ONESHELL: interface_definitions: $(config_xml_obj) mkdir -p $(TMPL_DIR) $(CURDIR)/scripts/override-default $(BUILD_DIR)/interface-definitions find $(BUILD_DIR)/interface-definitions -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-templates {} $(CURDIR)/schema/interface_definition.rng $(TMPL_DIR) || exit 1 $(CURDIR)/python/vyos/xml_ref/generate_cache.py --xml-dir $(BUILD_DIR)/interface-definitions || exit 1 # XXX: delete top level node.def's that now live in other packages # IPSec VPN EAP-RADIUS does not support source-address rm -rf $(TMPL_DIR)/vpn/ipsec/remote-access/radius/source-address # T2472 - EIGRP support rm -rf $(TMPL_DIR)/protocols/eigrp # T2773 - EIGRP support for VRF rm -rf $(TMPL_DIR)/vrf/name/node.tag/protocols/eigrp # XXX: test if there are empty node.def files - this is not allowed as these # could mask help strings or mandatory priority statements find $(TMPL_DIR) -name node.def -type f -empty -exec false {} + || sh -c 'echo "There are empty node.def files! Check your interface definitions." && exit 1' ifeq ($(BUILD_ARCH),arm64) # There is currently no telegraf support in VyOS for ARM64, remove CLI definitions rm -rf $(TMPL_DIR)/service/monitoring/telegraf endif .PHONY: op_mode_definitions .ONESHELL: op_mode_definitions: $(op_xml_obj) mkdir -p $(OP_TMPL_DIR) find $(BUILD_DIR)/op-mode-definitions/ -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-op-templates {} $(CURDIR)/schema/op-mode-definition.rng $(OP_TMPL_DIR) || exit 1 # XXX: delete top level op mode node.def's that now live in other packages rm -f $(OP_TMPL_DIR)/add/node.def rm -f $(OP_TMPL_DIR)/clear/interfaces/node.def rm -f $(OP_TMPL_DIR)/clear/node.def rm -f $(OP_TMPL_DIR)/delete/node.def # XXX: tcpdump, ping, traceroute and mtr must be able to recursivly call themselves as the # options are provided from the scripts themselves ln -s ../node.tag $(OP_TMPL_DIR)/ping/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/traceroute/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/mtr/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/monitor/traceroute/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/monitor/traffic/interface/node.tag/node.tag/ # XXX: test if there are empty node.def files - this is not allowed as these # could mask help strings or mandatory priority statements find $(OP_TMPL_DIR) -name node.def -type f -empty -exec false {} + || sh -c 'echo "There are empty node.def files! Check your interface definitions." && exit 1' .PHONY: vyshim vyshim: $(MAKE) -C $(SHIM_DIR) .PHONY: all -all: clean interface_definitions op_mode_definitions check test j2lint vyshim check_migration_scripts_executable - -.PHONY: check -.ONESHELL: -check: - @echo "Checking which CLI scripts are not enabled to work with vyos-configd..." - @for file in `ls src/conf_mode -I__pycache__` - do - if ! grep -q $$file data/configd-include.json; then - echo "* $$file" - fi - done +all: clean interface_definitions op_mode_definitions test j2lint vyshim check_migration_scripts_executable generate-configd-include-json .PHONY: clean clean: rm -rf $(BUILD_DIR) rm -rf $(TMPL_DIR) rm -rf $(OP_TMPL_DIR) $(MAKE) -C $(SHIM_DIR) clean .PHONY: test -test: +test: generate-configd-include-json set -e; python3 -m compileall -q -x '/vmware-tools/scripts/, /ppp/' . PYTHONPATH=python/ python3 -m "nose" --with-xunit src --with-coverage --cover-erase --cover-xml --cover-package src/conf_mode,src/op_mode,src/completion,src/helpers,src/validators,src/tests --verbose .PHONY: check_migration_scripts_executable .ONESHELL: check_migration_scripts_executable: @echo "Checking if migration scripts have executable bit set..." find src/migration-scripts -type f -not -executable -print -exec false {} + || sh -c 'echo "Found files that are not executable! Add permissions." && exit 1' .PHONY: j2lint j2lint: ifndef J2LINT $(error "j2lint binary not found, consider installing: pip install git+https://github.com/aristanetworks/j2lint.git@341b5d5db86") endif $(J2LINT) data/ .PHONY: sonar sonar: sonar-scanner -X -Dsonar.login=${SONAR_TOKEN} .PHONY: unused-imports unused-imports: @pylint --disable=all --enable=W0611 $(PYLINT_FILES) deb: dpkg-buildpackage -uc -us -tc -b +.PHONY: generate-configd-include-json +generate-configd-include-json: + @scripts/generate-configd-include-json.py + .PHONY: schema schema: trang -I rnc -O rng schema/interface_definition.rnc schema/interface_definition.rng trang -I rnc -O rng schema/op-mode-definition.rnc schema/op-mode-definition.rng diff --git a/data/configd-include.json b/data/configd-include.json deleted file mode 100644 index 224a9c390..000000000 --- a/data/configd-include.json +++ /dev/null @@ -1,115 +0,0 @@ -[ -"container.py", -"firewall.py", -"high-availability.py", -"interfaces_bonding.py", -"interfaces_bridge.py", -"interfaces_dummy.py", -"interfaces_ethernet.py", -"interfaces_geneve.py", -"interfaces_input.py", -"interfaces_l2tpv3.py", -"interfaces_loopback.py", -"interfaces_macsec.py", -"interfaces_openvpn.py", -"interfaces_pppoe.py", -"interfaces_pseudo-ethernet.py", -"interfaces_sstpc.py", -"interfaces_tunnel.py", -"interfaces_virtual-ethernet.py", -"interfaces_vti.py", -"interfaces_vxlan.py", -"interfaces_wireguard.py", -"interfaces_wireless.py", -"interfaces_wwan.py", -"load-balancing_reverse-proxy.py", -"load-balancing_wan.py", -"nat.py", -"nat64.py", -"nat66.py", -"netns.py", -"pki.py", -"policy.py", -"policy_route.py", -"policy_local-route.py", -"protocols_babel.py", -"protocols_bfd.py", -"protocols_bgp.py", -"protocols_eigrp.py", -"protocols_failover.py", -"protocols_igmp-proxy.py", -"protocols_isis.py", -"protocols_mpls.py", -"protocols_nhrp.py", -"protocols_ospf.py", -"protocols_ospfv3.py", -"protocols_pim.py", -"protocols_pim6.py", -"protocols_rip.py", -"protocols_ripng.py", -"protocols_rpki.py", -"protocols_segment-routing.py", -"protocols_static.py", -"protocols_static_arp.py", -"protocols_static_multicast.py", -"protocols_static_neighbor-proxy.py", -"qos.py", -"service_aws_glb.py", -"service_broadcast-relay.py", -"service_config-sync.py", -"service_conntrack-sync.py", -"service_console-server.py", -"service_dhcp-relay.py", -"service_dhcp-server.py", -"service_dhcpv6-relay.py", -"service_dhcpv6-server.py", -"service_dns_dynamic.py", -"service_dns_forwarding.py", -"service_event-handler.py", -"service_https.py", -"service_ids_ddos-protection.py", -"service_ipoe-server.py", -"service_lldp.py", -"service_mdns_repeater.py", -"service_monitoring_telegraf.py", -"service_monitoring_zabbix-agent.py", -"service_ndp-proxy.py", -"service_ntp.py", -"service_pppoe-server.py", -"service_router-advert.py", -"service_salt-minion.py", -"service_sla.py", -"service_snmp.py", -"service_ssh.py", -"service_stunnel.py", -"service_tftp-server.py", -"service_webproxy.py", -"system_acceleration.py", -"system_config-management.py", -"system_conntrack.py", -"system_console.py", -"system_flow-accounting.py", -"system_frr.py", -"system_host-name.py", -"system_ip.py", -"system_ipv6.py", -"system_lcd.py", -"system_login.py", -"system_login_banner.py", -"system_logs.py", -"system_option.py", -"system_proxy.py", -"system_sflow.py", -"system_sysctl.py", -"system_syslog.py", -"system_task-scheduler.py", -"system_timezone.py", -"system_update-check.py", -"system_wireless.py", -"vpn_ipsec.py", -"vpn_l2tp.py", -"vpn_openconnect.py", -"vpn_pptp.py", -"vpn_sstp.py", -"vrf.py" -] diff --git a/scripts/generate-configd-include-json.py b/scripts/generate-configd-include-json.py new file mode 100755 index 000000000..b4b627fce --- /dev/null +++ b/scripts/generate-configd-include-json.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +from jinja2 import Template + +conf_scripts = 'src/conf_mode' +configd_include = 'data/configd-include.json' + +configd_template = Template("""[ +{% for file in files %} +"{{ file }}"{{ "," if not loop.last else "" }} +{% endfor %} +] +""", trim_blocks=True) + +files = [f for f in os.listdir(conf_scripts) if os.path.isfile(f'{conf_scripts}/{f}')] +files = sorted(files) + +tmp = {'files' : files} +with open(configd_include, 'w') as f: + f.write(configd_template.render(tmp))