diff --git a/data/configd-include.json b/data/configd-include.json index 648655a8b..456211caa 100644 --- a/data/configd-include.json +++ b/data/configd-include.json @@ -1,89 +1,90 @@ [ "arp.py", "bcast_relay.py", "container.py", "conntrack.py", "conntrack_sync.py", "dhcp_relay.py", "dhcp_server.py", "dhcpv6_relay.py", "dhcpv6_server.py", "dns_forwarding.py", "dynamic_dns.py", "firewall.py", "flow_accounting_conf.py", "high-availability.py", "host_name.py", "https.py", "igmp_proxy.py", "intel_qat.py", "interfaces-bonding.py", "interfaces-bridge.py", "interfaces-dummy.py", "interfaces-ethernet.py", "interfaces-geneve.py", "interfaces-l2tpv3.py", "interfaces-loopback.py", "interfaces-macsec.py", "interfaces-openvpn.py", "interfaces-pppoe.py", "interfaces-pseudo-ethernet.py", "interfaces-sstpc.py", "interfaces-tunnel.py", "interfaces-vti.py", "interfaces-vxlan.py", "interfaces-wireguard.py", "interfaces-wireless.py", "interfaces-wwan.py", "lldp.py", "nat.py", "nat66.py", "netns.py", "ntp.py", "pki.py", "policy.py", "policy-local-route.py", "protocols_bfd.py", "protocols_bgp.py", "protocols_igmp.py", "protocols_isis.py", "protocols_mpls.py", "protocols_nhrp.py", "protocols_ospf.py", "protocols_ospfv3.py", "protocols_pim.py", "protocols_rip.py", "protocols_ripng.py", "protocols_rpki.py", "protocols_static.py", "protocols_static_multicast.py", "qos.py", "salt-minion.py", "service_console-server.py", "service_ids_fastnetmon.py", "service_ipoe-server.py", "service_mdns-repeater.py", "service_monitoring_telegraf.py", "service_pppoe-server.py", "service_router-advert.py", "service_upnp.py", "ssh.py", "system-ip.py", "system-ipv6.py", "system-login-banner.py", "system-logs.py", "system-option.py", "system-proxy.py", +"system_sflow.py", "system_sysctl.py", "system-syslog.py", "system-timezone.py", "system_console.py", "system_lcd.py", "task_scheduler.py", "tftp_server.py", "vpn_l2tp.py", "vpn_pptp.py", "vpn_sstp.py", "vrf.py", "vrf_vni.py" ] diff --git a/data/templates/sflow/hsflowd.conf.j2 b/data/templates/sflow/hsflowd.conf.j2 new file mode 100644 index 000000000..7ce6554d7 --- /dev/null +++ b/data/templates/sflow/hsflowd.conf.j2 @@ -0,0 +1,28 @@ +# Genereated by /usr/libexec/vyos/conf_mode/system_sflow.py +# Parameters http://sflow.net/host-sflow-linux-config.php + +sflow { +{% if polling is vyos_defined %} + polling={{ polling }} +{% endif %} +{% if sampling_rate is vyos_defined %} + sampling={{ sampling_rate }} + sampling.bps_ratio=0 +{% endif %} +{% if agent_address is vyos_defined %} + agentIP={{ agent_address }} +{% endif %} +{% if agent_interface is vyos_defined %} + agent={{ agent_interface }} +{% endif %} +{% if server is vyos_defined %} +{% for server, server_config in server.items() %} + collector { ip = {{ server }} udpport = {{ server_config.port }} } +{% endfor %} +{% endif %} +{% if interface is vyos_defined %} +{% for iface in interface %} + pcap { dev={{ iface }} } +{% endfor %} +{% endif %} +} diff --git a/data/templates/sflow/override.conf.j2 b/data/templates/sflow/override.conf.j2 new file mode 100644 index 000000000..f2a982528 --- /dev/null +++ b/data/templates/sflow/override.conf.j2 @@ -0,0 +1,16 @@ +[Unit] +After= +After=vyos-router.service +ConditionPathExists= +ConditionPathExists=/run/sflow/hsflowd.conf + +[Service] +EnvironmentFile= +ExecStart= +ExecStart=/usr/sbin/hsflowd -m %m -d -f /run/sflow/hsflowd.conf +WorkingDirectory= +WorkingDirectory=/run/sflow +PIDFile= +PIDFile=/run/sflow/hsflowd.pid +Restart=always +RestartSec=10 diff --git a/debian/control b/debian/control index c3854252f..028b7cd43 100644 --- a/debian/control +++ b/debian/control @@ -1,204 +1,205 @@ Source: vyos-1x Section: contrib/net Priority: extra Maintainer: VyOS Package Maintainers <maintainers@vyos.net> Build-Depends: debhelper (>= 9), dh-python, fakeroot, gcc-multilib [amd64], clang [amd64], iproute2, llvm [amd64], libbpf-dev [amd64], libelf-dev (>= 0.2) [amd64], libpcap-dev [amd64], build-essential, libvyosconfig0 (>= 0.0.7), libzmq3-dev, python3, python3-coverage, python3-lxml, python3-netifaces, python3-nose, python3-jinja2, python3-psutil, python3-setuptools, python3-sphinx, python3-xmltodict, quilt, whois Standards-Version: 3.9.6 Package: vyos-1x Architecture: amd64 arm64 Depends: ${python3:Depends}, accel-ppp, avahi-daemon, beep, bmon, bsdmainutils, charon-systemd, conntrack, conntrackd, conserver-client, conserver-server, console-data, cron, curl, dbus, ddclient (>= 3.9.1), dropbear, easy-rsa, etherwake, ethtool, fdisk, fastnetmon [amd64], file, frr (>= 7.5), frr-pythontools, frr-rpki-rtrlib, frr-snmp, fuse-overlayfs, libpam-google-authenticator, grc, hostapd, + hsflowd, hvinfo, igmpproxy, ipaddrcheck, iperf, iperf3, iproute2 (>= 6.0.0), iputils-arping, isc-dhcp-client, isc-dhcp-relay, isc-dhcp-server, iw, keepalived (>=2.0.5), lcdproc, lcdproc-extra-drivers, libatomic1, libbpf1 [amd64], libcharon-extra-plugins (>=5.9), libcharon-extauth-plugins (>=5.9), libndp-tools, libnetfilter-conntrack3, libnfnetlink0, libpam-radius-auth (>= 1.5.0), libqmi-utils, libstrongswan-extra-plugins (>=5.9), libstrongswan-standard-plugins (>=5.9), libvyosconfig0, lldpd, lm-sensors, lsscsi, minisign, modemmanager, mtr-tiny, ndisc6, ndppd, netplug, nfct, nftables (>= 0.9.3), nginx-light, chrony, nvme-cli, ocserv, opennhrp, openssh-server, openssl, openvpn, openvpn-auth-ldap, openvpn-auth-radius, openvpn-otp, owamp-client, owamp-server, pciutils, pdns-recursor, pmacct (>= 1.6.0), podman, pppoe, procps, python3, python3-certbot-nginx, python3-cryptography, python3-hurry.filesize, python3-inotify, python3-isc-dhcp-leases, python3-jinja2, python3-jmespath, python3-netaddr, python3-netifaces, python3-paramiko, python3-passlib, python3-psutil, python3-pyhumps, python3-pystache, python3-pyudev, python3-six, python3-tabulate, python3-vici (>= 5.7.2), python3-voluptuous, python3-xmltodict, python3-zmq, qrencode, radvd, salt-minion, sed, smartmontools, snmp, snmpd, squashfs-tools, squid, squidclient, squidguard, sshguard, ssl-cert, sstp-client, strongswan (>= 5.9), strongswan-swanctl (>= 5.9), stunnel4, sudo, systemd, telegraf (>= 1.20), tcpdump, tcptraceroute, telnet, tftpd-hpa, traceroute, tuned, twamp-client, twamp-server, udp-broadcast-relay, uidmap, usb-modeswitch, usbutils, vyatta-bash, vyatta-cfg, vyos-http-api-tools, vyos-utils, wide-dhcpv6-client, wireguard-tools, wireless-regdb, wpasupplicant (>= 0.6.7), ndppd, miniupnpd-nftables Description: VyOS configuration scripts and data VyOS configuration scripts, interface definitions, and everything Package: vyos-1x-vmware Architecture: amd64 Depends: vyos-1x, open-vm-tools Description: VyOS configuration scripts and data for VMware Adds configuration files required for VyOS running on VMware hosts. Package: vyos-1x-smoketest Architecture: all Depends: skopeo, snmp, vyos-1x Description: VyOS build sanity checking toolkit diff --git a/interface-definitions/system-sflow.xml.in b/interface-definitions/system-sflow.xml.in new file mode 100644 index 000000000..a53c99937 --- /dev/null +++ b/interface-definitions/system-sflow.xml.in @@ -0,0 +1,103 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- sflow configuration --> +<interfaceDefinition> + <node name="system"> + <children> + <node name="sflow" owner="${vyos_conf_scripts_dir}/system_sflow.py"> + <properties> + <help>sFlow settings</help> + <priority>990</priority> + </properties> + <children> + <leafNode name="agent-address"> + <properties> + <help>sFlow agent IPv4 or IPv6 address</help> + <completionHelp> + <list>auto</list> + <script>${vyos_completion_dir}/list_local_ips.sh --both</script> + </completionHelp> + <valueHelp> + <format>ipv4</format> + <description>sFlow IPv4 agent address</description> + </valueHelp> + <valueHelp> + <format>ipv6</format> + <description>sFlow IPv6 agent address</description> + </valueHelp> + <constraint> + <validator name="ipv4-address"/> + <validator name="ipv6-address"/> + <validator name="ipv6-link-local"/> + </constraint> + </properties> + </leafNode> + <leafNode name="agent-interface"> + <properties> + <help>IP address associated with this interface</help> + <completionHelp> + <script>${vyos_completion_dir}/list_interfaces</script> + </completionHelp> + <valueHelp> + <format>txt</format> + <description>Interface name</description> + </valueHelp> + <constraint> + #include <include/constraint/interface-name.xml.in> + </constraint> + </properties> + </leafNode> + #include <include/generic-interface-multi.xml.i> + <leafNode name="polling"> + <properties> + <help>Schedule counter-polling in seconds</help> + <valueHelp> + <format>u32:1-600</format> + <description>Polling rate in seconds</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 1-600"/> + </constraint> + </properties> + <defaultValue>30</defaultValue> + </leafNode> + <leafNode name="sampling-rate"> + <properties> + <help>sFlow sampling-rate</help> + <valueHelp> + <format>u32:1-65535</format> + <description>Sampling rate (1 in N packets)</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 1-65535"/> + </constraint> + </properties> + <defaultValue>1000</defaultValue> + </leafNode> + <tagNode name="server"> + <properties> + <help>sFlow destination server</help> + <valueHelp> + <format>ipv4</format> + <description>IPv4 server to export sFlow</description> + </valueHelp> + <valueHelp> + <format>ipv6</format> + <description>IPv6 server to export sFlow</description> + </valueHelp> + <constraint> + <validator name="ipv4-address"/> + <validator name="ipv6-address"/> + </constraint> + </properties> + <children> + #include <include/port-number.xml.i> + <leafNode name="port"> + <defaultValue>6343</defaultValue> + </leafNode> + </children> + </tagNode> + </children> + </node> + </children> + </node> +</interfaceDefinition> diff --git a/smoketest/scripts/cli/test_system_sflow.py b/smoketest/scripts/cli/test_system_sflow.py new file mode 100755 index 000000000..b593c21e6 --- /dev/null +++ b/smoketest/scripts/cli/test_system_sflow.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import unittest + +from base_vyostest_shim import VyOSUnitTestSHIM + +from vyos.configsession import ConfigSessionError +from vyos.ifconfig import Section +from vyos.util import cmd +from vyos.util import process_named_running +from vyos.util import read_file + +PROCESS_NAME = 'hsflowd' +base_path = ['system', 'sflow'] + +hsflowd_conf = '/run/sflow/hsflowd.conf' + + +class TestSystemFlowAccounting(VyOSUnitTestSHIM.TestCase): + + @classmethod + def setUpClass(cls): + super(TestSystemFlowAccounting, cls).setUpClass() + + # ensure we can also run this test on a live system - so lets clean + # out the current configuration :) + cls.cli_delete(cls, base_path) + + def tearDown(self): + # after service removal process must no longer run + self.assertTrue(process_named_running(PROCESS_NAME)) + + self.cli_delete(base_path) + self.cli_commit() + + # after service removal process must no longer run + self.assertFalse(process_named_running(PROCESS_NAME)) + + def test_sflow(self): + agent_address = '192.0.2.5' + agent_interface = 'eth0' + polling = '24' + sampling_rate = '128' + server = '192.0.2.254' + port = '8192' + + self.cli_set( + ['interfaces', 'dummy', 'dum0', 'address', f'{agent_address}/24']) + self.cli_set(base_path + ['agent-address', agent_address]) + self.cli_set(base_path + ['agent-interface', agent_interface]) + + # You need to configure at least one interface for sflow + with self.assertRaises(ConfigSessionError): + self.cli_commit() + for interface in Section.interfaces('ethernet'): + self.cli_set(base_path + ['interface', interface]) + + self.cli_set(base_path + ['polling', polling]) + self.cli_set(base_path + ['sampling-rate', sampling_rate]) + self.cli_set(base_path + ['server', server, 'port', port]) + + # commit changes + self.cli_commit() + + # verify configuration + hsflowd = read_file(hsflowd_conf) + + self.assertIn(f'polling={polling}', hsflowd) + self.assertIn(f'sampling={sampling_rate}', hsflowd) + self.assertIn(f'agentIP={agent_address}', hsflowd) + self.assertIn(f'agent={agent_interface}', hsflowd) + self.assertIn(f'collector {{ ip = {server} udpport = {port} }}', hsflowd) + + for interface in Section.interfaces('ethernet'): + self.assertIn(f'pcap {{ dev={interface} }}', hsflowd) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/conf_mode/system_sflow.py b/src/conf_mode/system_sflow.py new file mode 100755 index 000000000..2e19a5d76 --- /dev/null +++ b/src/conf_mode/system_sflow.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os + +from sys import exit + +from vyos.config import Config +from vyos.configdict import dict_merge +from vyos.template import render +from vyos.util import call +from vyos.validate import is_addr_assigned +from vyos.xml import defaults +from vyos import ConfigError +from vyos import airbag +airbag.enable() + +hsflowd_conf_path = '/run/sflow/hsflowd.conf' +systemd_service = 'hsflowd.service' +systemd_override = f'/run/systemd/system/{systemd_service}.d/override.conf' + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + base = ['system', 'sflow'] + if not conf.exists(base): + return None + + sflow = conf.get_config_dict(base, + key_mangling=('-', '_'), + get_first_key=True) + + # We have gathered the dict representation of the CLI, but there are default + # options which we need to update into the dictionary retrived. + default_values = defaults(base) + + sflow = dict_merge(default_values, sflow) + + # Ignore default XML values if config doesn't exists + # Delete key from dict + if 'port' in sflow['server']: + del sflow['server']['port'] + + return sflow + + +def verify(sflow): + if not sflow: + return None + + # Check if configured sflow agent-address exist in the system + if 'agent_address' in sflow: + tmp = sflow['agent_address'] + if not is_addr_assigned(tmp): + raise ConfigError( + f'Configured "sflow agent-address {tmp}" does not exist in the system!' + ) + + # Check if at least one interface is configured + if 'interface' not in sflow: + raise ConfigError( + 'sFlow requires at least one interface to be configured!') + + # Check if at least one server is configured + if 'server' not in sflow: + raise ConfigError('You need to configure at least one sFlow server!') + + # return True if all checks were passed + return True + + +def generate(sflow): + if not sflow: + return None + + render(hsflowd_conf_path, 'sflow/hsflowd.conf.j2', sflow) + render(systemd_override, 'sflow/override.conf.j2', sflow) + # Reload systemd manager configuration + call('systemctl daemon-reload') + + +def apply(sflow): + if not sflow: + # Stop flow-accounting daemon and remove configuration file + call(f'systemctl stop {systemd_service}') + if os.path.exists(hsflowd_conf_path): + os.unlink(hsflowd_conf_path) + return + + # Start/reload flow-accounting daemon + call(f'systemctl restart {systemd_service}') + + +if __name__ == '__main__': + try: + config = get_config() + verify(config) + generate(config) + apply(config) + except ConfigError as e: + print(e) + exit(1)