diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py
index 870d7cfda..5a353b110 100644
--- a/python/vyos/configdict.py
+++ b/python/vyos/configdict.py
@@ -1,666 +1,666 @@
# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
"""
A library for retrieving value dicts from VyOS configs in a declarative fashion.
"""
import os
import json
from vyos.utils.dict import dict_search
from vyos.utils.process import cmd
def retrieve_config(path_hash, base_path, config):
"""
Retrieves a VyOS config as a dict according to a declarative description
The description dict, passed in the first argument, must follow this format:
``field_name : <path, type, [inner_options_dict]>``.
Supported types are: ``str`` (for normal nodes),
``list`` (returns a list of strings, for multi nodes),
``bool`` (returns True if valueless node exists),
``dict`` (for tag nodes, returns a dict indexed by node names,
according to description in the third item of the tuple).
Args:
path_hash (dict): Declarative description of the config to retrieve
base_path (list): A base path to prepend to all option paths
config (vyos.config.Config): A VyOS config object
Returns:
dict: config dict
"""
config_hash = {}
for k in path_hash:
if type(path_hash[k]) != tuple:
raise ValueError("In field {0}: expected a tuple, got a value {1}".format(k, str(path_hash[k])))
if len(path_hash[k]) < 2:
raise ValueError("In field {0}: field description must be a tuple of at least two items, path (list) and type".format(k))
path = path_hash[k][0]
if type(path) != list:
raise ValueError("In field {0}: path must be a list, not a {1}".format(k, type(path)))
typ = path_hash[k][1]
if type(typ) != type:
raise ValueError("In field {0}: type must be a type, not a {1}".format(k, type(typ)))
path = base_path + path
path_str = " ".join(path)
if typ == str:
config_hash[k] = config.return_value(path_str)
elif typ == list:
config_hash[k] = config.return_values(path_str)
elif typ == bool:
config_hash[k] = config.exists(path_str)
elif typ == dict:
try:
inner_hash = path_hash[k][2]
except IndexError:
raise ValueError("The type of the \'{0}\' field is dict, but inner options hash is missing from the tuple".format(k))
config_hash[k] = {}
nodes = config.list_nodes(path_str)
for node in nodes:
config_hash[k][node] = retrieve_config(inner_hash, path + [node], config)
return config_hash
def dict_merge(source, destination):
""" Merge two dictionaries. Only keys which are not present in destination
will be copied from source, anything else will be kept untouched. Function
will return a new dict which has the merged key/value pairs. """
from copy import deepcopy
tmp = deepcopy(destination)
for key, value in source.items():
if key not in tmp:
tmp[key] = value
elif isinstance(source[key], dict):
tmp[key] = dict_merge(source[key], tmp[key])
return tmp
def list_diff(first, second):
""" Diff two dictionaries and return only unique items """
second = set(second)
return [item for item in first if item not in second]
def is_node_changed(conf, path):
"""
Check if any key under path has been changed and return True.
If nothing changed, return false
"""
from vyos.configdiff import get_config_diff
D = get_config_diff(conf, key_mangling=('-', '_'))
return D.is_node_changed(path)
def leaf_node_changed(conf, path):
"""
Check if a leaf node was altered. If it has been altered - values has been
changed, or it was added/removed, we will return a list containing the old
value(s). If nothing has been changed, None is returned.
NOTE: path must use the real CLI node name (e.g. with a hyphen!)
"""
from vyos.configdiff import get_config_diff
D = get_config_diff(conf, key_mangling=('-', '_'))
(new, old) = D.get_value_diff(path)
if new != old:
if isinstance(old, dict):
# valueLess nodes return {} if node is deleted
return True
if old is None and isinstance(new, dict):
# valueLess nodes return {} if node was added
return True
if old is None:
return []
if isinstance(old, str):
return [old]
if isinstance(old, list):
if isinstance(new, str):
new = [new]
elif isinstance(new, type(None)):
new = []
return list_diff(old, new)
return None
def node_changed(conf, path, key_mangling=None, recursive=False, expand_nodes=None) -> list:
"""
Check if node under path (or anything under path if recursive=True) was changed. By default
we only check if a node or subnode (recursive) was deleted from path. If expand_nodes
is set to Diff.ADD we can also check if something was added to the path.
If nothing changed, an empty list is returned.
"""
from vyos.configdiff import get_config_diff
from vyos.configdiff import Diff
# to prevent circular dependencies we assign the default here
if not expand_nodes: expand_nodes = Diff.DELETE
D = get_config_diff(conf, key_mangling)
# get_child_nodes_diff() will return dict_keys()
tmp = D.get_child_nodes_diff(path, expand_nodes=expand_nodes, recursive=recursive)
output = []
if expand_nodes & Diff.DELETE:
output.extend(list(tmp['delete'].keys()))
if expand_nodes & Diff.ADD:
output.extend(list(tmp['add'].keys()))
# remove duplicate keys from list, this happens when a node (e.g. description) is altered
output = list(dict.fromkeys(output))
return output
def get_removed_vlans(conf, path, dict):
"""
Common function to parse a dictionary retrieved via get_config_dict() and
determine any added/removed VLAN interfaces - be it 802.1q or Q-in-Q.
"""
from vyos.configdiff import get_config_diff, Diff
# Check vif, vif-s/vif-c VLAN interfaces for removal
D = get_config_diff(conf, key_mangling=('-', '_'))
D.set_level(conf.get_level())
# get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
keys = D.get_child_nodes_diff(path + ['vif'], expand_nodes=Diff.DELETE)['delete'].keys()
if keys: dict['vif_remove'] = [*keys]
# get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
keys = D.get_child_nodes_diff(path + ['vif-s'], expand_nodes=Diff.DELETE)['delete'].keys()
if keys: dict['vif_s_remove'] = [*keys]
for vif in dict.get('vif_s', {}).keys():
keys = D.get_child_nodes_diff(path + ['vif-s', vif, 'vif-c'], expand_nodes=Diff.DELETE)['delete'].keys()
if keys: dict['vif_s'][vif]['vif_c_remove'] = [*keys]
return dict
def is_member(conf, interface, intftype=None):
"""
Checks if passed interface is member of other interface of specified type.
intftype is optional, if not passed it will search all known types
(currently bridge and bonding)
Returns: dict
empty -> Interface is not a member
key -> Interface is a member of this interface
"""
ret_val = {}
intftypes = ['bonding', 'bridge']
if intftype not in intftypes + [None]:
raise ValueError((
f'unknown interface type "{intftype}" or it cannot '
f'have member interfaces'))
intftype = intftypes if intftype == None else [intftype]
for iftype in intftype:
base = ['interfaces', iftype]
for intf in conf.list_nodes(base):
member = base + [intf, 'member', 'interface', interface]
if conf.exists(member):
tmp = conf.get_config_dict(member, key_mangling=('-', '_'),
get_first_key=True,
no_tag_node_value_mangle=True)
ret_val.update({intf : tmp})
return ret_val
def is_mirror_intf(conf, interface, direction=None):
"""
Check whether the passed interface is used for port mirroring. Direction
is optional, if not passed it will search all known direction
(currently ingress and egress)
Returns:
None -> Interface is not a monitor interface
Array() -> This interface is a monitor interface of interfaces
"""
from vyos.ifconfig import Section
directions = ['ingress', 'egress']
if direction not in directions + [None]:
raise ValueError(f'Unknown interface mirror direction "{direction}"')
direction = directions if direction == None else [direction]
ret_val = None
base = ['interfaces']
for dir in direction:
for iftype in conf.list_nodes(base):
iftype_base = base + [iftype]
for intf in conf.list_nodes(iftype_base):
mirror = iftype_base + [intf, 'mirror', dir, interface]
if conf.exists(mirror):
path = ['interfaces', Section.section(intf), intf]
tmp = conf.get_config_dict(path, key_mangling=('-', '_'),
get_first_key=True)
ret_val = {intf : tmp}
return ret_val
def has_address_configured(conf, intf):
"""
Checks if interface has an address configured.
Checks the following config nodes:
'address', 'ipv6 address eui64', 'ipv6 address autoconf'
Returns True if interface has address configured, False if it doesn't.
"""
from vyos.ifconfig import Section
ret = False
old_level = conf.get_level()
conf.set_level([])
intfpath = ['interfaces', Section.get_config_path(intf)]
if (conf.exists([intfpath, 'address']) or
conf.exists([intfpath, 'ipv6', 'address', 'autoconf']) or
conf.exists([intfpath, 'ipv6', 'address', 'eui64'])):
ret = True
conf.set_level(old_level)
return ret
def has_vrf_configured(conf, intf):
"""
Checks if interface has a VRF configured.
Returns True if interface has VRF configured, False if it doesn't.
"""
from vyos.ifconfig import Section
ret = False
old_level = conf.get_level()
conf.set_level([])
if conf.exists(['interfaces', Section.get_config_path(intf), 'vrf']):
ret = True
conf.set_level(old_level)
return ret
def has_vlan_subinterface_configured(conf, intf):
"""
Checks if interface has an VLAN subinterface configured.
Checks the following config nodes:
'vif', 'vif-s'
Return True if interface has VLAN subinterface configured.
"""
from vyos.ifconfig import Section
ret = False
intfpath = ['interfaces', Section.section(intf), intf]
if (conf.exists(intfpath + ['vif']) or conf.exists(intfpath + ['vif-s'])):
ret = True
return ret
def is_source_interface(conf, interface, intftype=None):
"""
Checks if passed interface is configured as source-interface of other
interfaces of specified type. intftype is optional, if not passed it will
search all known types (currently pppoe, macsec, pseudo-ethernet, tunnel
and vxlan)
Returns:
None -> Interface is not a member
interface name -> Interface is a member of this interface
False -> interface type cannot have members
"""
ret_val = None
intftypes = ['macsec', 'pppoe', 'pseudo-ethernet', 'tunnel', 'vxlan']
if not intftype:
intftype = intftypes
if isinstance(intftype, str):
intftype = [intftype]
elif not isinstance(intftype, list):
raise ValueError(f'Interface type "{type(intftype)}" must be either str or list!')
if not all(x in intftypes for x in intftype):
raise ValueError(f'unknown interface type "{intftype}" or it can not '
'have a source-interface')
for it in intftype:
base = ['interfaces', it]
for intf in conf.list_nodes(base):
src_intf = base + [intf, 'source-interface']
if conf.exists(src_intf) and interface in conf.return_values(src_intf):
ret_val = intf
break
return ret_val
def get_dhcp_interfaces(conf, vrf=None):
""" Common helper functions to retrieve all interfaces from current CLI
sessions that have DHCP configured. """
dhcp_interfaces = {}
dict = conf.get_config_dict(['interfaces'], get_first_key=True)
if not dict:
return dhcp_interfaces
def check_dhcp(config):
ifname = config['ifname']
tmp = {}
if 'address' in config and 'dhcp' in config['address']:
options = {}
if dict_search('dhcp_options.default_route_distance', config) != None:
options.update({'dhcp_options' : config['dhcp_options']})
if 'vrf' in config:
if vrf == config['vrf']: tmp.update({ifname : options})
else:
if vrf is None: tmp.update({ifname : options})
return tmp
for section, interface in dict.items():
for ifname in interface:
# always reset config level, as get_interface_dict() will alter it
conf.set_level([])
# we already have a dict representation of the config from get_config_dict(),
# but with the extended information from get_interface_dict() we also
# get the DHCP client default-route-distance default option if not specified.
_, ifconfig = get_interface_dict(conf, ['interfaces', section], ifname)
tmp = check_dhcp(ifconfig)
dhcp_interfaces.update(tmp)
# check per VLAN interfaces
for vif, vif_config in ifconfig.get('vif', {}).items():
tmp = check_dhcp(vif_config)
dhcp_interfaces.update(tmp)
# check QinQ VLAN interfaces
for vif_s, vif_s_config in ifconfig.get('vif_s', {}).items():
tmp = check_dhcp(vif_s_config)
dhcp_interfaces.update(tmp)
for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items():
tmp = check_dhcp(vif_c_config)
dhcp_interfaces.update(tmp)
return dhcp_interfaces
def get_pppoe_interfaces(conf, vrf=None):
""" Common helper functions to retrieve all interfaces from current CLI
sessions that have DHCP configured. """
pppoe_interfaces = {}
conf.set_level([])
for ifname in conf.list_nodes(['interfaces', 'pppoe']):
# always reset config level, as get_interface_dict() will alter it
conf.set_level([])
# we already have a dict representation of the config from get_config_dict(),
# but with the extended information from get_interface_dict() we also
# get the DHCP client default-route-distance default option if not specified.
_, ifconfig = get_interface_dict(conf, ['interfaces', 'pppoe'], ifname)
options = {}
if 'default_route_distance' in ifconfig:
options.update({'default_route_distance' : ifconfig['default_route_distance']})
if 'no_default_route' in ifconfig:
options.update({'no_default_route' : {}})
if 'vrf' in ifconfig:
if vrf == ifconfig['vrf']: pppoe_interfaces.update({ifname : options})
else:
if vrf is None: pppoe_interfaces.update({ifname : options})
return pppoe_interfaces
def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pki=False):
"""
Common utility function to retrieve and mangle the interfaces configuration
from the CLI input nodes. All interfaces have a common base where value
retrival is identical. This function must be used whenever possible when
working on the interfaces node!
Return a dictionary with the necessary interface config keys.
"""
if not ifname:
from vyos import ConfigError
# determine tagNode instance
if 'VYOS_TAGNODE_VALUE' not in os.environ:
raise ConfigError('Interface (VYOS_TAGNODE_VALUE) not specified')
ifname = os.environ['VYOS_TAGNODE_VALUE']
# Check if interface has been removed. We must use exists() as
# get_config_dict() will always return {} - even when an empty interface
# node like the following exists.
# +macsec macsec1 {
# +}
if not config.exists(base + [ifname]):
dict = config.get_config_dict(base + [ifname], key_mangling=('-', '_'),
get_first_key=True,
no_tag_node_value_mangle=True)
dict.update({'deleted' : {}})
else:
# Get config_dict with default values
dict = config.get_config_dict(base + [ifname], key_mangling=('-', '_'),
get_first_key=True,
no_tag_node_value_mangle=True,
with_defaults=True,
with_recursive_defaults=recursive_defaults,
with_pki=with_pki)
# If interface does not request an IPv4 DHCP address there is no need
# to keep the dhcp-options key
if 'address' not in dict or 'dhcp' not in dict['address']:
if 'dhcp_options' in dict:
del dict['dhcp_options']
# Add interface instance name into dictionary
dict.update({'ifname': ifname})
# Check if QoS policy applied on this interface - See ifconfig.interface.set_mirror_redirect()
if config.exists(['qos', 'interface', ifname]):
dict.update({'traffic_policy': {}})
address = leaf_node_changed(config, base + [ifname, 'address'])
if address: dict.update({'address_old' : address})
# Check if we are a member of a bridge device
bridge = is_member(config, ifname, 'bridge')
if bridge: dict.update({'is_bridge_member' : bridge})
# Check if it is a monitor interface
mirror = is_mirror_intf(config, ifname)
if mirror: dict.update({'is_mirror_intf' : mirror})
# Check if we are a member of a bond device
bond = is_member(config, ifname, 'bonding')
if bond: dict.update({'is_bond_member' : bond})
# Check if any DHCP options changed which require a client restat
dhcp = is_node_changed(config, base + [ifname, 'dhcp-options'])
if dhcp: dict.update({'dhcp_options_changed' : {}})
# Changine interface VRF assignemnts require a DHCP restart, too
dhcp = is_node_changed(config, base + [ifname, 'vrf'])
if dhcp: dict.update({'dhcp_options_changed' : {}})
# Some interfaces come with a source_interface which must also not be part
# of any other bond or bridge interface as it is exclusivly assigned as the
# Kernels "lower" interface to this new "virtual/upper" interface.
if 'source_interface' in dict:
# Check if source interface is member of another bridge
tmp = is_member(config, dict['source_interface'], 'bridge')
if tmp: dict.update({'source_interface_is_bridge_member' : tmp})
# Check if source interface is member of another bridge
tmp = is_member(config, dict['source_interface'], 'bonding')
if tmp: dict.update({'source_interface_is_bond_member' : tmp})
mac = leaf_node_changed(config, base + [ifname, 'mac'])
if mac: dict.update({'mac_old' : mac})
eui64 = leaf_node_changed(config, base + [ifname, 'ipv6', 'address', 'eui64'])
if eui64:
tmp = dict_search('ipv6.address', dict)
if not tmp:
dict.update({'ipv6': {'address': {'eui64_old': eui64}}})
else:
dict['ipv6']['address'].update({'eui64_old': eui64})
for vif, vif_config in dict.get('vif', {}).items():
# Add subinterface name to dictionary
dict['vif'][vif].update({'ifname' : f'{ifname}.{vif}'})
if config.exists(['qos', 'interface', f'{ifname}.{vif}']):
dict['vif'][vif].update({'traffic_policy': {}})
if 'deleted' not in dict:
address = leaf_node_changed(config, base + [ifname, 'vif', vif, 'address'])
if address: dict['vif'][vif].update({'address_old' : address})
# If interface does not request an IPv4 DHCP address there is no need
# to keep the dhcp-options key
if 'address' not in dict['vif'][vif] or 'dhcp' not in dict['vif'][vif]['address']:
if 'dhcp_options' in dict['vif'][vif]:
del dict['vif'][vif]['dhcp_options']
# Check if we are a member of a bridge device
bridge = is_member(config, f'{ifname}.{vif}', 'bridge')
if bridge: dict['vif'][vif].update({'is_bridge_member' : bridge})
# Check if any DHCP options changed which require a client restat
dhcp = is_node_changed(config, base + [ifname, 'vif', vif, 'dhcp-options'])
if dhcp: dict['vif'][vif].update({'dhcp_options_changed' : {}})
for vif_s, vif_s_config in dict.get('vif_s', {}).items():
# Add subinterface name to dictionary
dict['vif_s'][vif_s].update({'ifname' : f'{ifname}.{vif_s}'})
if config.exists(['qos', 'interface', f'{ifname}.{vif_s}']):
dict['vif_s'][vif_s].update({'traffic_policy': {}})
if 'deleted' not in dict:
address = leaf_node_changed(config, base + [ifname, 'vif-s', vif_s, 'address'])
if address: dict['vif_s'][vif_s].update({'address_old' : address})
# If interface does not request an IPv4 DHCP address there is no need
# to keep the dhcp-options key
if 'address' not in dict['vif_s'][vif_s] or 'dhcp' not in \
dict['vif_s'][vif_s]['address']:
if 'dhcp_options' in dict['vif_s'][vif_s]:
del dict['vif_s'][vif_s]['dhcp_options']
# Check if we are a member of a bridge device
bridge = is_member(config, f'{ifname}.{vif_s}', 'bridge')
if bridge: dict['vif_s'][vif_s].update({'is_bridge_member' : bridge})
# Check if any DHCP options changed which require a client restat
dhcp = is_node_changed(config, base + [ifname, 'vif-s', vif_s, 'dhcp-options'])
if dhcp: dict['vif_s'][vif_s].update({'dhcp_options_changed' : {}})
for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items():
# Add subinterface name to dictionary
dict['vif_s'][vif_s]['vif_c'][vif_c].update({'ifname' : f'{ifname}.{vif_s}.{vif_c}'})
if config.exists(['qos', 'interface', f'{ifname}.{vif_s}.{vif_c}']):
dict['vif_s'][vif_s]['vif_c'][vif_c].update({'traffic_policy': {}})
if 'deleted' not in dict:
address = leaf_node_changed(config, base + [ifname, 'vif-s', vif_s, 'vif-c', vif_c, 'address'])
if address: dict['vif_s'][vif_s]['vif_c'][vif_c].update(
{'address_old' : address})
# If interface does not request an IPv4 DHCP address there is no need
# to keep the dhcp-options key
if 'address' not in dict['vif_s'][vif_s]['vif_c'][vif_c] or 'dhcp' \
not in dict['vif_s'][vif_s]['vif_c'][vif_c]['address']:
if 'dhcp_options' in dict['vif_s'][vif_s]['vif_c'][vif_c]:
del dict['vif_s'][vif_s]['vif_c'][vif_c]['dhcp_options']
# Check if we are a member of a bridge device
bridge = is_member(config, f'{ifname}.{vif_s}.{vif_c}', 'bridge')
if bridge: dict['vif_s'][vif_s]['vif_c'][vif_c].update(
{'is_bridge_member' : bridge})
# Check if any DHCP options changed which require a client restat
dhcp = is_node_changed(config, base + [ifname, 'vif-s', vif_s, 'vif-c', vif_c, 'dhcp-options'])
if dhcp: dict['vif_s'][vif_s]['vif_c'][vif_c].update({'dhcp_options_changed' : {}})
# Check vif, vif-s/vif-c VLAN interfaces for removal
dict = get_removed_vlans(config, base + [ifname], dict)
return ifname, dict
def get_vlan_ids(interface):
"""
Get the VLAN ID of the interface bound to the bridge
"""
vlan_ids = set()
bridge_status = cmd('bridge -j vlan show', shell=True)
vlan_filter_status = json.loads(bridge_status)
if vlan_filter_status is not None:
for interface_status in vlan_filter_status:
ifname = interface_status['ifname']
if interface == ifname:
vlans_status = interface_status['vlans']
for vlan_status in vlans_status:
vlan_id = vlan_status['vlan']
vlan_ids.add(vlan_id)
return vlan_ids
def get_accel_dict(config, base, chap_secrets, with_pki=False):
"""
Common utility function to retrieve and mangle the Accel-PPP configuration
from different CLI input nodes. All Accel-PPP services have a common base
where value retrival is identical. This function must be used whenever
possible when working with Accel-PPP services!
Return a dictionary with the necessary interface config keys.
"""
- from vyos.cpu import get_core_count
+ from vyos.utils.cpu import get_core_count
from vyos.template import is_ipv4
dict = config.get_config_dict(base, key_mangling=('-', '_'),
get_first_key=True,
no_tag_node_value_mangle=True,
with_recursive_defaults=True,
with_pki=with_pki)
# set CPUs cores to process requests
dict.update({'thread_count' : get_core_count()})
# we need to store the path to the secrets file
dict.update({'chap_secrets_file' : chap_secrets})
# We can only have two IPv4 and three IPv6 nameservers - also they are
# configured in a different way in the configuration, this is why we split
# the configuration
if 'name_server' in dict:
ns_v4 = []
ns_v6 = []
for ns in dict['name_server']:
if is_ipv4(ns): ns_v4.append(ns)
else: ns_v6.append(ns)
dict.update({'name_server_ipv4' : ns_v4, 'name_server_ipv6' : ns_v6})
del dict['name_server']
# Check option "disable-accounting" per server and replace default value from '1813' to '0'
for server in (dict_search('authentication.radius.server', dict) or []):
if 'disable_accounting' in dict['authentication']['radius']['server'][server]:
dict['authentication']['radius']['server'][server]['acct_port'] = '0'
return dict
diff --git a/python/vyos/utils/__init__.py b/python/vyos/utils/__init__.py
index 12ef2d3b8..1cd062a11 100644
--- a/python/vyos/utils/__init__.py
+++ b/python/vyos/utils/__init__.py
@@ -1,30 +1,31 @@
-# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
from vyos.utils import assertion
from vyos.utils import auth
from vyos.utils import boot
from vyos.utils import commit
from vyos.utils import convert
+from vyos.utils import cpu
from vyos.utils import dict
from vyos.utils import file
from vyos.utils import io
from vyos.utils import kernel
from vyos.utils import list
from vyos.utils import misc
from vyos.utils import network
from vyos.utils import permission
from vyos.utils import process
from vyos.utils import system
diff --git a/python/vyos/cpu.py b/python/vyos/utils/cpu.py
similarity index 99%
rename from python/vyos/cpu.py
rename to python/vyos/utils/cpu.py
index cae5f5f4d..3bea5ac12 100644
--- a/python/vyos/cpu.py
+++ b/python/vyos/utils/cpu.py
@@ -1,102 +1,101 @@
# Copyright (C) 2022-2024 VyOS maintainers and contributors
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
"""
Retrieves (or at least attempts to retrieve) the total number of real CPU cores
installed in a Linux system.
The issue of core count is complicated by existence of SMT, e.g. Intel's Hyper Threading.
GNU nproc returns the number of LOGICAL cores,
which is 2x of the real cores if SMT is enabled.
The idea is to find all physical CPUs and add up their core counts.
It has special cases for x86_64 and MAY work correctly on other architectures,
but nothing is certain.
"""
import re
-
def _read_cpuinfo():
with open('/proc/cpuinfo', 'r') as f:
lines = f.read().strip()
return re.split(r'\n+', lines)
def _split_line(l):
l = l.strip()
parts = re.split(r'\s*:\s*', l)
return (parts[0], ":".join(parts[1:]))
def _find_cpus(cpuinfo_lines):
# Make a dict because it's more convenient to work with later,
# when we need to find physicall distinct CPUs there.
cpus = {}
cpu_number = 0
for l in cpuinfo_lines:
key, value = _split_line(l)
if key == 'processor':
cpu_number = value
cpus[cpu_number] = {}
else:
cpus[cpu_number][key] = value
return cpus
def _find_physical_cpus():
cpus = _find_cpus(_read_cpuinfo())
phys_cpus = {}
for num in cpus:
if 'physical id' in cpus[num]:
# On at least some architectures, CPUs in different sockets
# have different 'physical id' field, e.g. on x86_64.
phys_id = cpus[num]['physical id']
if phys_id not in phys_cpus:
phys_cpus[phys_id] = cpus[num]
else:
# On other architectures, e.g. on ARM, there's no such field.
# We just assume they are different CPUs,
# whether single core ones or cores of physical CPUs.
phys_cpus[num] = cpus[num]
return phys_cpus
def get_cpus():
""" Returns a list of /proc/cpuinfo entries that belong to different CPUs.
"""
cpus_dict = _find_physical_cpus()
return list(cpus_dict.values())
def get_core_count():
""" Returns the total number of physical CPU cores
(even if Hyper-Threading or another SMT is enabled and has inflated
the number of cores in /proc/cpuinfo)
"""
physical_cpus = _find_physical_cpus()
core_count = 0
for num in physical_cpus:
# Some architectures, e.g. x86_64, include a field for core count.
# Since we found unique physical CPU entries, we can sum their core counts.
if 'cpu cores' in physical_cpus[num]:
core_count += int(physical_cpus[num]['cpu cores'])
else:
core_count += 1
return core_count
diff --git a/smoketest/scripts/cli/base_accel_ppp_test.py b/smoketest/scripts/cli/base_accel_ppp_test.py
index 212dc58ab..c6f6cb804 100644
--- a/smoketest/scripts/cli/base_accel_ppp_test.py
+++ b/smoketest/scripts/cli/base_accel_ppp_test.py
@@ -1,648 +1,648 @@
# Copyright (C) 2020-2024 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import re
from base_vyostest_shim import VyOSUnitTestSHIM
from configparser import ConfigParser
from vyos.configsession import ConfigSessionError
from vyos.template import is_ipv4
-from vyos.cpu import get_core_count
+from vyos.utils.cpu import get_core_count
from vyos.utils.process import process_named_running
from vyos.utils.process import cmd
class BasicAccelPPPTest:
class TestCase(VyOSUnitTestSHIM.TestCase):
@classmethod
def setUpClass(cls):
cls._process_name = "accel-pppd"
super(BasicAccelPPPTest.TestCase, cls).setUpClass()
# ensure we can also run this test on a live system - so lets clean
# out the current configuration :)
cls.cli_delete(cls, cls._base_path)
def setUp(self):
self._gateway = "192.0.2.1"
# ensure we can also run this test on a live system - so lets clean
# out the current configuration :)
self.cli_delete(self._base_path)
def tearDown(self):
# Check for running process
self.assertTrue(process_named_running(self._process_name))
self.cli_delete(self._base_path)
self.cli_commit()
# Check for running process
self.assertFalse(process_named_running(self._process_name))
def set(self, path):
self.cli_set(self._base_path + path)
def delete(self, path):
self.cli_delete(self._base_path + path)
def basic_protocol_specific_config(self):
"""
An astract method.
Initialize protocol scpecific configureations.
"""
self.assertFalse(True, msg="Function must be defined")
def initial_auth_config(self):
"""
Initialization of default authentication for all protocols
"""
self.set(
[
"authentication",
"local-users",
"username",
"vyos",
"password",
"vyos",
]
)
self.set(["authentication", "mode", "local"])
def initial_gateway_config(self):
"""
Initialization of default gateway
"""
self.set(["gateway-address", self._gateway])
def initial_pool_config(self):
"""
Initialization of default client ip pool
"""
first_pool = "SIMPLE-POOL"
self.set(["client-ip-pool", first_pool, "range", "192.0.2.0/24"])
self.set(["default-pool", first_pool])
def basic_config(self, is_auth=True, is_gateway=True, is_client_pool=True):
"""
Initialization of basic configuration
:param is_auth: authentication initialization
:type is_auth: bool
:param is_gateway: gateway initialization
:type is_gateway: bool
:param is_client_pool: client ip pool initialization
:type is_client_pool: bool
"""
self.basic_protocol_specific_config()
if is_auth:
self.initial_auth_config()
if is_gateway:
self.initial_gateway_config()
if is_client_pool:
self.initial_pool_config()
def getConfig(self, start, end="cli"):
"""
Return part of configuration from line
where the first injection of start keyword to the line
where the first injection of end keyowrd
:param start: start keyword
:type start: str
:param end: end keyword
:type end: str
:return: part of config
:rtype: str
"""
command = f'cat {self._config_file} | sed -n "/^\[{start}/,/^\[{end}/p"'
out = cmd(command)
return out
def verify(self, conf):
self.assertEqual(conf["core"]["thread-count"], str(get_core_count()))
def test_accel_name_servers(self):
# Verify proper Name-Server configuration for IPv4 and IPv6
self.basic_config()
nameserver = ["192.0.2.1", "192.0.2.2", "2001:db8::1"]
for ns in nameserver:
self.set(["name-server", ns])
# commit changes
self.cli_commit()
# Validate configuration values
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
# IPv4 and IPv6 nameservers must be checked individually
for ns in nameserver:
if is_ipv4(ns):
self.assertIn(ns, [conf["dns"]["dns1"], conf["dns"]["dns2"]])
else:
self.assertEqual(conf["ipv6-dns"][ns], None)
def test_accel_local_authentication(self):
# Test configuration of local authentication
self.basic_config()
# upload / download limit
user = "test"
password = "test2"
static_ip = "100.100.100.101"
upload = "5000"
download = "10000"
self.set(
[
"authentication",
"local-users",
"username",
user,
"password",
password,
]
)
self.set(
[
"authentication",
"local-users",
"username",
user,
"static-ip",
static_ip,
]
)
self.set(
[
"authentication",
"local-users",
"username",
user,
"rate-limit",
"upload",
upload,
]
)
# upload rate-limit requires also download rate-limit
with self.assertRaises(ConfigSessionError):
self.cli_commit()
self.set(
[
"authentication",
"local-users",
"username",
user,
"rate-limit",
"download",
download,
]
)
# commit changes
self.cli_commit()
# Validate configuration values
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
# check proper path to chap-secrets file
self.assertEqual(conf["chap-secrets"]["chap-secrets"], self._chap_secrets)
# basic verification
self.verify(conf)
# check local users
tmp = cmd(f"sudo cat {self._chap_secrets}")
regex = f"{user}\s+\*\s+{password}\s+{static_ip}\s+{download}/{upload}"
tmp = re.findall(regex, tmp)
self.assertTrue(tmp)
# Check local-users default value(s)
self.delete(
["authentication", "local-users", "username", user, "static-ip"]
)
# commit changes
self.cli_commit()
# check local users
tmp = cmd(f"sudo cat {self._chap_secrets}")
regex = f"{user}\s+\*\s+{password}\s+\*\s+{download}/{upload}"
tmp = re.findall(regex, tmp)
self.assertTrue(tmp)
def test_accel_radius_authentication(self):
# Test configuration of RADIUS authentication for PPPoE server
self.basic_config()
radius_server = "192.0.2.22"
radius_key = "secretVyOS"
radius_port = "2000"
radius_port_acc = "3000"
acct_interim_jitter = '10'
acct_interim_interval = '10'
acct_timeout = '30'
self.set(["authentication", "mode", "radius"])
self.set(
["authentication", "radius", "server", radius_server, "key", radius_key]
)
self.set(
[
"authentication",
"radius",
"server",
radius_server,
"port",
radius_port,
]
)
self.set(
[
"authentication",
"radius",
"server",
radius_server,
"acct-port",
radius_port_acc,
]
)
self.set(
[
"authentication",
"radius",
"acct-interim-jitter",
acct_interim_jitter,
]
)
self.set(
[
"authentication",
"radius",
"accounting-interim-interval",
acct_interim_interval,
]
)
self.set(
[
"authentication",
"radius",
"acct-timeout",
acct_timeout,
]
)
coa_server = "4.4.4.4"
coa_key = "testCoA"
self.set(
["authentication", "radius", "dynamic-author", "server", coa_server]
)
self.set(["authentication", "radius", "dynamic-author", "key", coa_key])
nas_id = "VyOS-PPPoE"
nas_ip = "7.7.7.7"
self.set(["authentication", "radius", "nas-identifier", nas_id])
self.set(["authentication", "radius", "nas-ip-address", nas_ip])
source_address = "1.2.3.4"
self.set(["authentication", "radius", "source-address", source_address])
# commit changes
self.cli_commit()
# Validate configuration values
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
# basic verification
self.verify(conf)
# check auth
self.assertTrue(conf["radius"].getboolean("verbose"))
self.assertEqual(conf["radius"]["acct-timeout"], acct_timeout)
self.assertEqual(conf["radius"]["acct-interim-interval"], acct_interim_interval)
self.assertEqual(conf["radius"]["acct-interim-jitter"], acct_interim_jitter)
self.assertEqual(conf["radius"]["timeout"], "3")
self.assertEqual(conf["radius"]["max-try"], "3")
self.assertEqual(
conf["radius"]["dae-server"], f"{coa_server}:1700,{coa_key}"
)
self.assertEqual(conf["radius"]["nas-identifier"], nas_id)
self.assertEqual(conf["radius"]["nas-ip-address"], nas_ip)
self.assertEqual(conf["radius"]["bind"], source_address)
server = conf["radius"]["server"].split(",")
self.assertEqual(radius_server, server[0])
self.assertEqual(radius_key, server[1])
self.assertEqual(f"auth-port={radius_port}", server[2])
self.assertEqual(f"acct-port={radius_port_acc}", server[3])
self.assertEqual(f"req-limit=0", server[4])
self.assertEqual(f"fail-time=0", server[5])
#
# Disable Radius Accounting
#
self.delete(
["authentication", "radius", "server", radius_server, "acct-port"]
)
self.set(
[
"authentication",
"radius",
"server",
radius_server,
"disable-accounting",
]
)
self.set(
[
"authentication",
"radius",
"server",
radius_server,
"backup",
]
)
self.set(
[
"authentication",
"radius",
"server",
radius_server,
"priority",
"10",
]
)
# commit changes
self.cli_commit()
conf.read(self._config_file)
server = conf["radius"]["server"].split(",")
self.assertEqual(radius_server, server[0])
self.assertEqual(radius_key, server[1])
self.assertEqual(f"auth-port={radius_port}", server[2])
self.assertEqual(f"acct-port=0", server[3])
self.assertEqual(f"req-limit=0", server[4])
self.assertEqual(f"fail-time=0", server[5])
self.assertIn('weight=10', server)
self.assertIn('backup', server)
def test_accel_ipv4_pool(self):
self.basic_config(is_gateway=False, is_client_pool=False)
gateway = "192.0.2.1"
subnet = "172.16.0.0/24"
first_pool = "POOL1"
second_pool = "POOL2"
range = "192.0.2.10-192.0.2.20"
range_config = "192.0.2.10-20"
self.set(["gateway-address", gateway])
self.set(["client-ip-pool", first_pool, "range", subnet])
self.set(["client-ip-pool", first_pool, "next-pool", second_pool])
self.set(["client-ip-pool", second_pool, "range", range])
self.set(["default-pool", first_pool])
# commit changes
self.cli_commit()
# Validate configuration values
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
self.assertEqual(
f"{first_pool},next={second_pool}", conf["ip-pool"][f"{subnet},name"]
)
self.assertEqual(second_pool, conf["ip-pool"][f"{range_config},name"])
self.assertEqual(gateway, conf["ip-pool"]["gw-ip-address"])
self.assertEqual(first_pool, conf[self._protocol_section]["ip-pool"])
def test_accel_next_pool(self):
# T5099 required specific order
self.basic_config(is_gateway=False, is_client_pool=False)
gateway = "192.0.2.1"
first_pool = "VyOS-pool1"
first_subnet = "192.0.2.0/25"
second_pool = "Vyos-pool2"
second_subnet = "203.0.113.0/25"
third_pool = "Vyos-pool3"
third_subnet = "198.51.100.0/24"
self.set(["gateway-address", gateway])
self.set(["client-ip-pool", first_pool, "range", first_subnet])
self.set(["client-ip-pool", first_pool, "next-pool", second_pool])
self.set(["client-ip-pool", second_pool, "range", second_subnet])
self.set(["client-ip-pool", second_pool, "next-pool", third_pool])
self.set(["client-ip-pool", third_pool, "range", third_subnet])
# commit changes
self.cli_commit()
config = self.getConfig("ip-pool")
pool_config = f"""gw-ip-address={gateway}
{third_subnet},name={third_pool}
{second_subnet},name={second_pool},next={third_pool}
{first_subnet},name={first_pool},next={second_pool}"""
self.assertIn(pool_config, config)
def test_accel_ipv6_pool(self):
# Test configuration of IPv6 client pools
self.basic_config(is_gateway=False, is_client_pool=False)
# Enable IPv6
allow_ipv6 = 'allow'
self.set(['ppp-options', 'ipv6', allow_ipv6])
pool_name = 'ipv6_test_pool'
prefix_1 = '2001:db8:fffe::/56'
prefix_mask = '64'
prefix_2 = '2001:db8:ffff::/56'
client_prefix_1 = f'{prefix_1},{prefix_mask}'
client_prefix_2 = f'{prefix_2},{prefix_mask}'
self.set(
['client-ipv6-pool', pool_name, 'prefix', prefix_1, 'mask',
prefix_mask])
self.set(
['client-ipv6-pool', pool_name, 'prefix', prefix_2, 'mask',
prefix_mask])
delegate_1_prefix = '2001:db8:fff1::/56'
delegate_2_prefix = '2001:db8:fff2::/56'
delegate_mask = '64'
self.set(
['client-ipv6-pool', pool_name, 'delegate', delegate_1_prefix,
'delegation-prefix', delegate_mask])
self.set(
['client-ipv6-pool', pool_name, 'delegate', delegate_2_prefix,
'delegation-prefix', delegate_mask])
# commit changes
self.cli_commit()
# Validate configuration values
conf = ConfigParser(allow_no_value=True, delimiters='=',
strict=False)
conf.read(self._config_file)
for tmp in ['ipv6pool', 'ipv6_nd', 'ipv6_dhcp']:
self.assertEqual(conf['modules'][tmp], None)
self.assertEqual(conf['ppp']['ipv6'], allow_ipv6)
config = self.getConfig("ipv6-pool")
pool_config = f"""{client_prefix_1},name={pool_name}
{client_prefix_2},name={pool_name}
delegate={delegate_1_prefix},{delegate_mask},name={pool_name}
delegate={delegate_2_prefix},{delegate_mask},name={pool_name}"""
self.assertIn(pool_config, config)
def test_accel_ppp_options(self):
# Test configuration of local authentication for PPPoE server
self.basic_config()
# other settings
mppe = 'require'
self.set(['ppp-options', 'disable-ccp'])
self.set(['ppp-options', 'mppe', mppe])
# min-mtu
min_mtu = '1400'
self.set(['ppp-options', 'min-mtu', min_mtu])
# mru
mru = '9000'
self.set(['ppp-options', 'mru', mru])
# interface-cache
interface_cache = '128000'
self.set(['ppp-options', 'interface-cache', interface_cache])
# ipv6
allow_ipv6 = 'allow'
allow_ipv4 = 'require'
random = 'random'
lcp_failure = '4'
lcp_interval = '40'
lcp_timeout = '100'
self.set(['ppp-options', 'ipv4', allow_ipv4])
self.set(['ppp-options', 'ipv6', allow_ipv6])
self.set(['ppp-options', 'ipv6-interface-id', random])
self.set(['ppp-options', 'ipv6-accept-peer-interface-id'])
self.set(['ppp-options', 'ipv6-peer-interface-id', random])
self.set(['ppp-options', 'lcp-echo-failure', lcp_failure])
self.set(['ppp-options', 'lcp-echo-interval', lcp_interval])
self.set(['ppp-options', 'lcp-echo-timeout', lcp_timeout])
# commit changes
self.cli_commit()
# Validate configuration values
conf = ConfigParser(allow_no_value=True, delimiters='=')
conf.read(self._config_file)
self.assertEqual(conf['chap-secrets']['gw-ip-address'], self._gateway)
# check ppp
self.assertEqual(conf['ppp']['mppe'], mppe)
self.assertEqual(conf['ppp']['min-mtu'], min_mtu)
self.assertEqual(conf['ppp']['mru'], mru)
self.assertEqual(conf['ppp']['ccp'],'0')
# check interface-cache
self.assertEqual(conf['ppp']['unit-cache'], interface_cache)
#check ipv6
for tmp in ['ipv6pool', 'ipv6_nd', 'ipv6_dhcp']:
self.assertEqual(conf['modules'][tmp], None)
self.assertEqual(conf['ppp']['ipv6'], allow_ipv6)
self.assertEqual(conf['ppp']['ipv6-intf-id'], random)
self.assertEqual(conf['ppp']['ipv6-peer-intf-id'], random)
self.assertTrue(conf['ppp'].getboolean('ipv6-accept-peer-intf-id'))
self.assertEqual(conf['ppp']['lcp-echo-failure'], lcp_failure)
self.assertEqual(conf['ppp']['lcp-echo-interval'], lcp_interval)
self.assertEqual(conf['ppp']['lcp-echo-timeout'], lcp_timeout)
def test_accel_wins_server(self):
self.basic_config()
winsservers = ["192.0.2.1", "192.0.2.2"]
for wins in winsservers:
self.set(["wins-server", wins])
self.cli_commit()
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
for ws in winsservers:
self.assertIn(ws, [conf["wins"]["wins1"], conf["wins"]["wins2"]])
def test_accel_snmp(self):
self.basic_config()
self.set(['snmp', 'master-agent'])
self.cli_commit()
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
self.assertEqual(conf['modules']['net-snmp'], None)
self.assertEqual(conf['snmp']['master'],'1')
def test_accel_shaper(self):
self.basic_config()
fwmark = '2'
self.set(['shaper', 'fwmark', fwmark])
self.cli_commit()
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
self.assertEqual(conf['modules']['shaper'], None)
self.assertEqual(conf['shaper']['verbose'], '1')
self.assertEqual(conf['shaper']['down-limiter'], 'tbf')
self.assertEqual(conf['shaper']['fwmark'], fwmark)
def test_accel_limits(self):
self.basic_config()
burst = '100'
timeout = '20'
limits = '1/min'
self.set(['limits', 'connection-limit', limits])
self.set(['limits', 'timeout', timeout])
self.set(['limits', 'burst', burst])
self.cli_commit()
conf = ConfigParser(allow_no_value=True, delimiters="=", strict=False)
conf.read(self._config_file)
self.assertEqual(conf['modules']['connlimit'], None)
self.assertEqual(conf['connlimit']['limit'], limits)
self.assertEqual(conf['connlimit']['burst'], burst)
self.assertEqual(conf['connlimit']['timeout'], timeout)
def test_accel_log_level(self):
self.basic_config()
self.cli_commit()
# check default value
conf = ConfigParser(allow_no_value=True)
conf.read(self._config_file)
self.assertEqual(conf['log']['level'], '3')
for log_level in range(0, 5):
self.set(['log', 'level', str(log_level)])
self.cli_commit()
# Validate configuration values
conf = ConfigParser(allow_no_value=True)
conf.read(self._config_file)
self.assertEqual(conf['log']['level'], str(log_level))
diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py
index a969626a9..ded370a7a 100755
--- a/src/conf_mode/container.py
+++ b/src/conf_mode/container.py
@@ -1,521 +1,521 @@
#!/usr/bin/env python3
#
# Copyright (C) 2021-2024 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from decimal import Decimal
from hashlib import sha256
from ipaddress import ip_address
from ipaddress import ip_network
from json import dumps as json_write
from vyos.base import Warning
from vyos.config import Config
from vyos.configdict import dict_merge
from vyos.configdict import node_changed
from vyos.configdict import is_node_changed
from vyos.configverify import verify_vrf
from vyos.ifconfig import Interface
-from vyos.cpu import get_core_count
+from vyos.utils.cpu import get_core_count
from vyos.utils.file import write_file
from vyos.utils.process import call
from vyos.utils.process import cmd
from vyos.utils.process import run
from vyos.utils.network import interface_exists
from vyos.template import bracketize_ipv6
from vyos.template import inc_ip
from vyos.template import is_ipv4
from vyos.template import is_ipv6
from vyos.template import render
from vyos.xml_ref import default_value
from vyos import ConfigError
from vyos import airbag
airbag.enable()
config_containers = '/etc/containers/containers.conf'
config_registry = '/etc/containers/registries.conf'
config_storage = '/etc/containers/storage.conf'
systemd_unit_path = '/run/systemd/system'
def _cmd(command):
if os.path.exists('/tmp/vyos.container.debug'):
print(command)
return cmd(command)
def network_exists(name):
# Check explicit name for network, returns True if network exists
c = _cmd(f'podman network ls --quiet --filter name=^{name}$')
return bool(c)
# Common functions
def get_config(config=None):
if config:
conf = config
else:
conf = Config()
base = ['container']
container = conf.get_config_dict(base, key_mangling=('-', '_'),
no_tag_node_value_mangle=True,
get_first_key=True,
with_recursive_defaults=True)
for name in container.get('name', []):
# T5047: Any container related configuration changed? We only
# wan't to restart the required containers and not all of them ...
tmp = is_node_changed(conf, base + ['name', name])
if tmp:
if 'container_restart' not in container:
container['container_restart'] = [name]
else:
container['container_restart'].append(name)
# registry is a tagNode with default values - merge the list from
# default_values['registry'] into the tagNode variables
if 'registry' not in container:
container.update({'registry': {}})
default_values = default_value(base + ['registry'])
for registry in default_values:
tmp = {registry: {}}
container['registry'] = dict_merge(tmp, container['registry'])
# Delete container network, delete containers
tmp = node_changed(conf, base + ['network'])
if tmp: container.update({'network_remove': tmp})
tmp = node_changed(conf, base + ['name'])
if tmp: container.update({'container_remove': tmp})
return container
def verify(container):
# bail out early - looks like removal from running config
if not container:
return None
# Add new container
if 'name' in container:
for name, container_config in container['name'].items():
# Container image is a mandatory option
if 'image' not in container_config:
raise ConfigError(f'Container image for "{name}" is mandatory!')
# Check if requested container image exists locally. If it does not
# exist locally - inform the user. This is required as there is a
# shared container image storage accross all VyOS images. A user can
# delete a container image from the system, boot into another version
# of VyOS and then it would fail to boot. This is to prevent any
# configuration error when container images are deleted from the
# global storage. A per image local storage would be a super waste
# of diskspace as there will be a full copy (up tu several GB/image)
# on upgrade. This is the "cheapest" and fastest solution in terms
# of image upgrade and deletion.
image = container_config['image']
if run(f'podman image exists {image}') != 0:
Warning(f'Image "{image}" used in container "{name}" does not exist ' \
f'locally. Please use "add container image {image}" to add it ' \
f'to the system! Container "{name}" will not be started!')
if 'cpu_quota' in container_config:
cores = get_core_count()
if Decimal(container_config['cpu_quota']) > cores:
raise ConfigError(f'Cannot set limit to more cores than available "{name}"!')
if 'network' in container_config:
if len(container_config['network']) > 1:
raise ConfigError(f'Only one network can be specified for container "{name}"!')
# Check if the specified container network exists
network_name = list(container_config['network'])[0]
if network_name not in container.get('network', {}):
raise ConfigError(f'Container network "{network_name}" does not exist!')
if 'address' in container_config['network'][network_name]:
cnt_ipv4 = 0
cnt_ipv6 = 0
for address in container_config['network'][network_name]['address']:
network = None
if is_ipv4(address):
try:
network = [x for x in container['network'][network_name]['prefix'] if is_ipv4(x)][0]
cnt_ipv4 += 1
except:
raise ConfigError(f'Network "{network_name}" does not contain an IPv4 prefix!')
elif is_ipv6(address):
try:
network = [x for x in container['network'][network_name]['prefix'] if is_ipv6(x)][0]
cnt_ipv6 += 1
except:
raise ConfigError(f'Network "{network_name}" does not contain an IPv6 prefix!')
# Specified container IP address must belong to network prefix
if ip_address(address) not in ip_network(network):
raise ConfigError(f'Used container address "{address}" not in network "{network}"!')
# We can not use the first IP address of a network prefix as this is used by podman
if ip_address(address) == ip_network(network)[1]:
raise ConfigError(f'IP address "{address}" can not be used for a container, ' \
'reserved for the container engine!')
if cnt_ipv4 > 1 or cnt_ipv6 > 1:
raise ConfigError(f'Only one IP address per address family can be used for ' \
f'container "{name}". {cnt_ipv4} IPv4 and {cnt_ipv6} IPv6 address(es)!')
if 'device' in container_config:
for dev, dev_config in container_config['device'].items():
if 'source' not in dev_config:
raise ConfigError(f'Device "{dev}" has no source path configured!')
if 'destination' not in dev_config:
raise ConfigError(f'Device "{dev}" has no destination path configured!')
source = dev_config['source']
if not os.path.exists(source):
raise ConfigError(f'Device "{dev}" source path "{source}" does not exist!')
if 'sysctl' in container_config and 'parameter' in container_config['sysctl']:
for var, cfg in container_config['sysctl']['parameter'].items():
if 'value' not in cfg:
raise ConfigError(f'sysctl parameter {var} has no value assigned!')
if var.startswith('net.') and 'allow_host_networks' in container_config:
raise ConfigError(f'sysctl parameter {var} cannot be set when using host networking!')
if 'environment' in container_config:
for var, cfg in container_config['environment'].items():
if 'value' not in cfg:
raise ConfigError(f'Environment variable {var} has no value assigned!')
if 'label' in container_config:
for var, cfg in container_config['label'].items():
if 'value' not in cfg:
raise ConfigError(f'Label variable {var} has no value assigned!')
if 'volume' in container_config:
for volume, volume_config in container_config['volume'].items():
if 'source' not in volume_config:
raise ConfigError(f'Volume "{volume}" has no source path configured!')
if 'destination' not in volume_config:
raise ConfigError(f'Volume "{volume}" has no destination path configured!')
source = volume_config['source']
if not os.path.exists(source):
raise ConfigError(f'Volume "{volume}" source path "{source}" does not exist!')
if 'port' in container_config:
for tmp in container_config['port']:
if not {'source', 'destination'} <= set(container_config['port'][tmp]):
raise ConfigError(f'Both "source" and "destination" must be specified for a port mapping!')
# If 'allow-host-networks' or 'network' not set.
if 'allow_host_networks' not in container_config and 'network' not in container_config:
raise ConfigError(f'Must either set "network" or "allow-host-networks" for container "{name}"!')
# Can not set both allow-host-networks and network at the same time
if {'allow_host_networks', 'network'} <= set(container_config):
raise ConfigError(
f'"allow-host-networks" and "network" for "{name}" cannot be both configured at the same time!')
# gid cannot be set without uid
if 'gid' in container_config and 'uid' not in container_config:
raise ConfigError(f'Cannot set "gid" without "uid" for container')
# Add new network
if 'network' in container:
for network, network_config in container['network'].items():
v4_prefix = 0
v6_prefix = 0
# If ipv4-prefix not defined for user-defined network
if 'prefix' not in network_config:
raise ConfigError(f'prefix for network "{network}" must be defined!')
for prefix in network_config['prefix']:
if is_ipv4(prefix):
v4_prefix += 1
elif is_ipv6(prefix):
v6_prefix += 1
if v4_prefix > 1:
raise ConfigError(f'Only one IPv4 prefix can be defined for network "{network}"!')
if v6_prefix > 1:
raise ConfigError(f'Only one IPv6 prefix can be defined for network "{network}"!')
# Verify VRF exists
verify_vrf(network_config)
# A network attached to a container can not be deleted
if {'network_remove', 'name'} <= set(container):
for network in container['network_remove']:
for c, c_config in container['name'].items():
if 'network' in c_config and network in c_config['network']:
raise ConfigError(f'Can not remove network "{network}", used by container "{c}"!')
if 'registry' in container:
for registry, registry_config in container['registry'].items():
if 'authentication' not in registry_config:
continue
if not {'username', 'password'} <= set(registry_config['authentication']):
raise ConfigError('Container registry requires both username and password to be set!')
return None
def generate_run_arguments(name, container_config):
image = container_config['image']
cpu_quota = container_config['cpu_quota']
memory = container_config['memory']
shared_memory = container_config['shared_memory']
restart = container_config['restart']
# Add sysctl options
sysctl_opt = ''
if 'sysctl' in container_config and 'parameter' in container_config['sysctl']:
for k, v in container_config['sysctl']['parameter'].items():
sysctl_opt += f" --sysctl {k}={v['value']}"
# Add capability options. Should be in uppercase
capabilities = ''
if 'capability' in container_config:
for cap in container_config['capability']:
cap = cap.upper().replace('-', '_')
capabilities += f' --cap-add={cap}'
# Add a host device to the container /dev/x:/dev/x
device = ''
if 'device' in container_config:
for dev, dev_config in container_config['device'].items():
source_dev = dev_config['source']
dest_dev = dev_config['destination']
device += f' --device={source_dev}:{dest_dev}'
# Check/set environment options "-e foo=bar"
env_opt = ''
if 'environment' in container_config:
for k, v in container_config['environment'].items():
env_opt += f" --env \"{k}={v['value']}\""
# Check/set label options "--label foo=bar"
label = ''
if 'label' in container_config:
for k, v in container_config['label'].items():
label += f" --label \"{k}={v['value']}\""
hostname = ''
if 'host_name' in container_config:
hostname = container_config['host_name']
hostname = f'--hostname {hostname}'
# Publish ports
port = ''
if 'port' in container_config:
protocol = ''
for portmap in container_config['port']:
protocol = container_config['port'][portmap]['protocol']
sport = container_config['port'][portmap]['source']
dport = container_config['port'][portmap]['destination']
listen_addresses = container_config['port'][portmap].get('listen_address', [])
# If listen_addresses is not empty, include them in the publish command
if listen_addresses:
for listen_address in listen_addresses:
port += f' --publish {bracketize_ipv6(listen_address)}:{sport}:{dport}/{protocol}'
else:
# If listen_addresses is empty, just include the standard publish command
port += f' --publish {sport}:{dport}/{protocol}'
# Set uid and gid
uid = ''
if 'uid' in container_config:
uid = container_config['uid']
if 'gid' in container_config:
uid += ':' + container_config['gid']
uid = f'--user {uid}'
# Bind volume
volume = ''
if 'volume' in container_config:
for vol, vol_config in container_config['volume'].items():
svol = vol_config['source']
dvol = vol_config['destination']
mode = vol_config['mode']
prop = vol_config['propagation']
volume += f' --volume {svol}:{dvol}:{mode},{prop}'
host_pid = ''
if 'allow_host_pid' in container_config:
host_pid = '--pid host'
container_base_cmd = f'--detach --interactive --tty --replace {capabilities} --cpus {cpu_quota} {sysctl_opt} ' \
f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \
f'--name {name} {hostname} {device} {port} {volume} {env_opt} {label} {uid} {host_pid}'
entrypoint = ''
if 'entrypoint' in container_config:
# it needs to be json-formatted with single quote on the outside
entrypoint = json_write(container_config['entrypoint'].split()).replace('"', """)
entrypoint = f'--entrypoint '{entrypoint}''
command = ''
if 'command' in container_config:
command = container_config['command'].strip()
command_arguments = ''
if 'arguments' in container_config:
command_arguments = container_config['arguments'].strip()
if 'allow_host_networks' in container_config:
return f'{container_base_cmd} --net host {entrypoint} {image} {command} {command_arguments}'.strip()
ip_param = ''
networks = ",".join(container_config['network'])
for network in container_config['network']:
if 'address' not in container_config['network'][network]:
continue
for address in container_config['network'][network]['address']:
if is_ipv6(address):
ip_param += f' --ip6 {address}'
else:
ip_param += f' --ip {address}'
return f'{container_base_cmd} --no-healthcheck --net {networks} {ip_param} {entrypoint} {image} {command} {command_arguments}'.strip()
def generate(container):
# bail out early - looks like removal from running config
if not container:
for file in [config_containers, config_registry, config_storage]:
if os.path.exists(file):
os.unlink(file)
return None
if 'network' in container:
for network, network_config in container['network'].items():
tmp = {
'name': network,
'id': sha256(f'{network}'.encode()).hexdigest(),
'driver': 'bridge',
'network_interface': f'pod-{network}',
'subnets': [],
'ipv6_enabled': False,
'internal': False,
'dns_enabled': True,
'ipam_options': {
'driver': 'host-local'
}
}
for prefix in network_config['prefix']:
net = {'subnet': prefix, 'gateway': inc_ip(prefix, 1)}
tmp['subnets'].append(net)
if is_ipv6(prefix):
tmp['ipv6_enabled'] = True
write_file(f'/etc/containers/networks/{network}.json', json_write(tmp, indent=2))
render(config_containers, 'container/containers.conf.j2', container)
render(config_registry, 'container/registries.conf.j2', container)
render(config_storage, 'container/storage.conf.j2', container)
if 'name' in container:
for name, container_config in container['name'].items():
if 'disable' in container_config:
continue
file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service')
run_args = generate_run_arguments(name, container_config)
render(file_path, 'container/systemd-unit.j2', {'name': name, 'run_args': run_args, },
formater=lambda _: _.replace(""", '"').replace("'", "'"))
return None
def apply(container):
# Delete old containers if needed. We can't delete running container
# Option "--force" allows to delete containers with any status
if 'container_remove' in container:
for name in container['container_remove']:
file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service')
call(f'systemctl stop vyos-container-{name}.service')
if os.path.exists(file_path):
os.unlink(file_path)
call('systemctl daemon-reload')
# Delete old networks if needed
if 'network_remove' in container:
for network in container['network_remove']:
call(f'podman network rm {network} >/dev/null 2>&1')
# Add container
disabled_new = False
if 'name' in container:
for name, container_config in container['name'].items():
image = container_config['image']
if run(f'podman image exists {image}') != 0:
# container image does not exist locally - user already got
# informed by a WARNING in verfiy() - bail out early
continue
if 'disable' in container_config:
# check if there is a container by that name running
tmp = _cmd('podman ps -a --format "{{.Names}}"')
if name in tmp:
file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service')
call(f'systemctl stop vyos-container-{name}.service')
if os.path.exists(file_path):
disabled_new = True
os.unlink(file_path)
continue
if 'container_restart' in container and name in container['container_restart']:
cmd(f'systemctl restart vyos-container-{name}.service')
if disabled_new:
call('systemctl daemon-reload')
# Start network and assign it to given VRF if requested. this can only be done
# after the containers got started as the podman network interface will
# only be enabled by the first container and yet I do not know how to enable
# the network interface in advance
if 'network' in container:
for network, network_config in container['network'].items():
network_name = f'pod-{network}'
# T5147: Networks are started only as soon as there is a consumer.
# If only a network is created in the first place, no need to assign
# it to a VRF as there's no consumer, yet.
if interface_exists(network_name):
tmp = Interface(network_name)
tmp.set_vrf(network_config.get('vrf', ''))
tmp.add_ipv6_eui64_address('fe80::/64')
return None
if __name__ == '__main__':
try:
c = get_config()
verify(c)
generate(c)
apply(c)
except ConfigError as e:
print(e)
exit(1)
diff --git a/src/op_mode/cpu.py b/src/op_mode/cpu.py
index d53663c17..1a0f7392f 100755
--- a/src/op_mode/cpu.py
+++ b/src/op_mode/cpu.py
@@ -1,82 +1,82 @@
#!/usr/bin/env python3
#
-# Copyright (C) 2016-2022 VyOS maintainers and contributors
+# Copyright (C) 2016-2024 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
-import vyos.cpu
import vyos.opmode
+from vyos.utils.cpu import get_cpus
+from vyos.utils.cpu import get_core_count
from jinja2 import Template
cpu_template = Template("""
{% for cpu in cpus %}
{% if 'physical id' in cpu %}CPU socket: {{cpu['physical id']}}{% endif %}
{% if 'vendor_id' in cpu %}CPU Vendor: {{cpu['vendor_id']}}{% endif %}
{% if 'model name' in cpu %}Model: {{cpu['model name']}}{% endif %}
{% if 'cpu cores' in cpu %}Cores: {{cpu['cpu cores']}}{% endif %}
{% if 'cpu MHz' in cpu %}Current MHz: {{cpu['cpu MHz']}}{% endif %}
{% endfor %}
""")
cpu_summary_template = Template("""
Physical CPU cores: {{count}}
CPU model(s): {{models | join(", ")}}
""")
def _get_raw_data():
- return vyos.cpu.get_cpus()
+ return get_cpus()
def _format_cpus(cpu_data):
env = {'cpus': cpu_data}
return cpu_template.render(env).strip()
def _get_summary_data():
- count = vyos.cpu.get_core_count()
- cpu_data = vyos.cpu.get_cpus()
+ count = get_core_count()
+ cpu_data = get_cpus()
models = [c['model name'] for c in cpu_data]
env = {'count': count, "models": models}
return env
def _format_cpu_summary(summary_data):
return cpu_summary_template.render(summary_data).strip()
def show(raw: bool):
cpu_data = _get_raw_data()
if raw:
return cpu_data
else:
return _format_cpus(cpu_data)
def show_summary(raw: bool):
cpu_summary_data = _get_summary_data()
if raw:
return cpu_summary_data
else:
return _format_cpu_summary(cpu_summary_data)
if __name__ == '__main__':
try:
res = vyos.opmode.run(sys.modules[__name__])
if res:
print(res)
except (ValueError, vyos.opmode.Error) as e:
print(e)
sys.exit(1)
-
diff --git a/src/op_mode/uptime.py b/src/op_mode/uptime.py
index 059a4c3f6..559eed24c 100755
--- a/src/op_mode/uptime.py
+++ b/src/op_mode/uptime.py
@@ -1,82 +1,82 @@
#!/usr/bin/env python3
#
-# Copyright (C) 2021-2023 VyOS maintainers and contributors
+# Copyright (C) 2021-2024 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
import vyos.opmode
def _get_uptime_seconds():
from re import search
from vyos.utils.file import read_file
data = read_file("/proc/uptime")
seconds = search("([0-9\.]+)\s", data).group(1)
return int(float(seconds))
def _get_load_averages():
from re import search
+ from vyos.utils.cpu import get_core_count
from vyos.utils.process import cmd
- from vyos.cpu import get_core_count
data = cmd("uptime")
matches = search(r"load average:\s*(?P<one>[0-9\.]+)\s*,\s*(?P<five>[0-9\.]+)\s*,\s*(?P<fifteen>[0-9\.]+)\s*", data)
core_count = get_core_count()
res = {}
res[1] = float(matches["one"]) / core_count
res[5] = float(matches["five"]) / core_count
res[15] = float(matches["fifteen"]) / core_count
return res
def _get_raw_data():
from vyos.utils.convert import seconds_to_human
res = {}
res["uptime_seconds"] = _get_uptime_seconds()
res["uptime"] = seconds_to_human(_get_uptime_seconds(), separator=' ')
res["load_average"] = _get_load_averages()
return res
def _get_formatted_output(data):
out = "Uptime: {}\n\n".format(data["uptime"])
avgs = data["load_average"]
out += "Load averages:\n"
out += "1 minute: {:.01f}%\n".format(avgs[1]*100)
out += "5 minutes: {:.01f}%\n".format(avgs[5]*100)
out += "15 minutes: {:.01f}%\n".format(avgs[15]*100)
return out
def show(raw: bool):
uptime_data = _get_raw_data()
if raw:
return uptime_data
else:
return _get_formatted_output(uptime_data)
if __name__ == '__main__':
try:
res = vyos.opmode.run(sys.modules[__name__])
if res:
print(res)
except (ValueError, vyos.opmode.Error) as e:
print(e)
sys.exit(1)