diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py
index 41252d8f5..73bd9ea96 100644
--- a/python/vyos/configdep.py
+++ b/python/vyos/configdep.py
@@ -1,180 +1,190 @@
# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library. If not, see <http://www.gnu.org/licenses/>.
import os
import json
import typing
from inspect import stack
from graphlib import TopologicalSorter, CycleError
from vyos.utils.system import load_as_module
from vyos.configdict import dict_merge
from vyos.defaults import directories
from vyos.configsource import VyOSError
from vyos import ConfigError
# https://peps.python.org/pep-0484/#forward-references
# for type 'Config'
if typing.TYPE_CHECKING:
from vyos.config import Config
dependency_dir = os.path.join(directories['data'],
'config-mode-dependencies')
-dependent_func: dict[str, list[typing.Callable]] = {}
+local_dependent_func: dict[str, list[typing.Callable]] = {}
DEBUG = False
+FORCE_LOCAL = False
def debug_print(s: str):
if DEBUG:
print(s)
def canon_name(name: str) -> str:
return os.path.splitext(name)[0].replace('-', '_')
def canon_name_of_path(path: str) -> str:
script = os.path.basename(path)
return canon_name(script)
def caller_name() -> str:
return stack()[2].filename
def name_of(f: typing.Callable) -> str:
return f.__name__
def names_of(l: list[typing.Callable]) -> list[str]:
return [name_of(f) for f in l]
def remove_redundant(l: list[typing.Callable]) -> list[typing.Callable]:
names = set()
for e in reversed(l):
_ = l.remove(e) if name_of(e) in names else names.add(name_of(e))
def append_uniq(l: list[typing.Callable], e: typing.Callable):
"""Append an element, removing earlier occurrences
The list of dependencies is generally short and traversing the list on
each append is preferable to the cost of redundant script invocation.
"""
l.append(e)
remove_redundant(l)
def read_dependency_dict(dependency_dir: str = dependency_dir) -> dict:
res = {}
for dep_file in os.listdir(dependency_dir):
if not dep_file.endswith('.json'):
continue
path = os.path.join(dependency_dir, dep_file)
with open(path) as f:
d = json.load(f)
if dep_file == 'vyos-1x.json':
res = dict_merge(res, d)
else:
res = dict_merge(d, res)
return res
def get_dependency_dict(config: 'Config') -> dict:
if hasattr(config, 'cached_dependency_dict'):
d = getattr(config, 'cached_dependency_dict')
else:
d = read_dependency_dict()
setattr(config, 'cached_dependency_dict', d)
return d
def run_config_mode_script(script: str, config: 'Config'):
path = os.path.join(directories['conf_mode'], script)
name = canon_name(script)
mod = load_as_module(name, path)
config.set_level([])
try:
c = mod.get_config(config)
mod.verify(c)
mod.generate(c)
mod.apply(c)
except (VyOSError, ConfigError) as e:
raise ConfigError(repr(e))
def def_closure(target: str, config: 'Config',
tagnode: typing.Optional[str] = None) -> typing.Callable:
script = target + '.py'
def func_impl():
if tagnode:
os.environ['VYOS_TAGNODE_VALUE'] = tagnode
run_config_mode_script(script, config)
return func_impl
def set_dependents(case: str, config: 'Config',
tagnode: typing.Optional[str] = None):
d = get_dependency_dict(config)
k = canon_name_of_path(caller_name())
tag_ext = f'_{tagnode}' if tagnode is not None else ''
- l = dependent_func.setdefault(k, [])
+ if hasattr(config, 'dependent_func') and not FORCE_LOCAL:
+ dependent_func = getattr(config, 'dependent_func')
+ l = dependent_func.setdefault('vyos_configd', [])
+ else:
+ dependent_func = local_dependent_func
+ l = dependent_func.setdefault(k, [])
for target in d[k][case]:
func = def_closure(target, config, tagnode)
func.__name__ = f'{target}{tag_ext}'
append_uniq(l, func)
debug_print(f'set_dependents: caller {k}, dependents {names_of(l)}')
-def call_dependents():
+def call_dependents(dependent_func: dict = None):
k = canon_name_of_path(caller_name())
- l = dependent_func.get(k, [])
+ if dependent_func is None or FORCE_LOCAL:
+ dependent_func = local_dependent_func
+ l = dependent_func.get(k, [])
+ else:
+ l = dependent_func.get('vyos_configd', [])
debug_print(f'call_dependents: caller {k}, dependents {names_of(l)}')
while l:
f = l.pop(0)
debug_print(f'calling: {f.__name__}')
f()
def called_as_dependent() -> bool:
st = stack()[1:]
for f in st:
if f.filename == __file__:
return True
return False
def graph_from_dependency_dict(d: dict) -> dict:
g = {}
for k in list(d):
g[k] = set()
# add the dependencies for every sub-case; should there be cases
# that are mutally exclusive in the future, the graphs will be
# distinguished
for el in list(d[k]):
g[k] |= set(d[k][el])
return g
def is_acyclic(d: dict) -> bool:
g = graph_from_dependency_dict(d)
ts = TopologicalSorter(g)
try:
# get node iterator
order = ts.static_order()
# try iteration
_ = [*order]
except CycleError:
return False
return True
def check_dependency_graph(dependency_dir: str = dependency_dir,
supplement: str = None) -> bool:
d = read_dependency_dict(dependency_dir=dependency_dir)
if supplement is not None:
with open(supplement) as f:
d = dict_merge(json.load(f), d)
return is_acyclic(d)
diff --git a/src/services/vyos-configd b/src/services/vyos-configd
index 355182b26..648a017d5 100755
--- a/src/services/vyos-configd
+++ b/src/services/vyos-configd
@@ -1,289 +1,302 @@
#!/usr/bin/env python3
#
-# Copyright (C) 2020-2023 VyOS maintainers and contributors
+# Copyright (C) 2020-2024 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import grp
import re
import json
+import typing
import logging
import signal
import importlib.util
import zmq
from contextlib import contextmanager
from vyos.defaults import directories
from vyos.utils.boot import boot_configuration_complete
from vyos.configsource import ConfigSourceString
from vyos.configsource import ConfigSourceError
+from vyos.configdep import call_dependents
from vyos.config import Config
from vyos import ConfigError
CFG_GROUP = 'vyattacfg'
script_stdout_log = '/tmp/vyos-configd-script-stdout'
debug = True
logger = logging.getLogger(__name__)
logs_handler = logging.StreamHandler()
logger.addHandler(logs_handler)
if debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
SOCKET_PATH = "ipc:///run/vyos-configd.sock"
# Response error codes
R_SUCCESS = 1
R_ERROR_COMMIT = 2
R_ERROR_DAEMON = 4
R_PASS = 8
vyos_conf_scripts_dir = directories['conf_mode']
configd_include_file = os.path.join(directories['data'], 'configd-include.json')
configd_env_set_file = os.path.join(directories['data'], 'vyos-configd-env-set')
configd_env_unset_file = os.path.join(directories['data'], 'vyos-configd-env-unset')
# sourced on entering config session
configd_env_file = '/etc/default/vyos-configd-env'
session_out = None
session_mode = None
def key_name_from_file_name(f):
return os.path.splitext(f)[0]
def module_name_from_key(k):
return k.replace('-', '_')
def path_from_file_name(f):
return os.path.join(vyos_conf_scripts_dir, f)
# opt-in to be run by daemon
with open(configd_include_file) as f:
try:
include = json.load(f)
except OSError as e:
logger.critical(f"configd include file error: {e}")
sys.exit(1)
except json.JSONDecodeError as e:
logger.critical(f"JSON load error: {e}")
sys.exit(1)
# import conf_mode scripts
(_, _, filenames) = next(iter(os.walk(vyos_conf_scripts_dir)))
filenames.sort()
load_filenames = [f for f in filenames if f in include]
imports = [key_name_from_file_name(f) for f in load_filenames]
module_names = [module_name_from_key(k) for k in imports]
paths = [path_from_file_name(f) for f in load_filenames]
to_load = list(zip(module_names, paths))
modules = []
for x in to_load:
spec = importlib.util.spec_from_file_location(x[0], x[1])
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
modules.append(module)
conf_mode_scripts = dict(zip(imports, modules))
exclude_set = {key_name_from_file_name(f) for f in filenames if f not in include}
include_set = {key_name_from_file_name(f) for f in filenames if f in include}
@contextmanager
def stdout_redirected(filename, mode):
saved_stdout_fd = None
destination_file = None
try:
sys.stdout.flush()
saved_stdout_fd = os.dup(sys.stdout.fileno())
destination_file = open(filename, mode)
os.dup2(destination_file.fileno(), sys.stdout.fileno())
yield
finally:
if saved_stdout_fd is not None:
os.dup2(saved_stdout_fd, sys.stdout.fileno())
os.close(saved_stdout_fd)
if destination_file is not None:
destination_file.close()
def explicit_print(path, mode, msg):
try:
with open(path, mode) as f:
f.write(f"\n{msg}\n\n")
except OSError:
logger.critical("error explicit_print")
def run_script(script, config, args) -> int:
script.argv = args
config.set_level([])
try:
c = script.get_config(config)
script.verify(c)
script.generate(c)
script.apply(c)
except ConfigError as e:
logger.critical(e)
explicit_print(session_out, session_mode, str(e))
return R_ERROR_COMMIT
except Exception as e:
logger.critical(e)
return R_ERROR_DAEMON
return R_SUCCESS
def initialization(socket):
global session_out
global session_mode
# Reset config strings:
active_string = ''
session_string = ''
# check first for resent init msg, in case of client timeout
while True:
msg = socket.recv().decode("utf-8", "ignore")
try:
message = json.loads(msg)
if message["type"] == "init":
resp = "init"
socket.send(resp.encode())
except:
break
# zmq synchronous for ipc from single client:
active_string = msg
resp = "active"
socket.send(resp.encode())
session_string = socket.recv().decode("utf-8", "ignore")
resp = "session"
socket.send(resp.encode())
pid_string = socket.recv().decode("utf-8", "ignore")
resp = "pid"
socket.send(resp.encode())
logger.debug(f"config session pid is {pid_string}")
try:
session_out = os.readlink(f"/proc/{pid_string}/fd/1")
session_mode = 'w'
except FileNotFoundError:
session_out = None
# if not a 'live' session, for example on boot, write to file
if not session_out or not boot_configuration_complete():
session_out = script_stdout_log
session_mode = 'a'
try:
configsource = ConfigSourceString(running_config_text=active_string,
session_config_text=session_string)
except ConfigSourceError as e:
logger.debug(e)
return None
config = Config(config_source=configsource)
+ dependent_func: dict[str, list[typing.Callable]] = {}
+ setattr(config, 'dependent_func', dependent_func)
return config
-def process_node_data(config, data) -> int:
+def process_node_data(config, data, last: bool = False) -> int:
if not config:
logger.critical(f"Empty config")
return R_ERROR_DAEMON
script_name = None
args = []
res = re.match(r'^(VYOS_TAGNODE_VALUE=[^/]+)?.*\/([^/]+).py(.*)', data)
if res.group(1):
env = res.group(1).split('=')
os.environ[env[0]] = env[1]
if res.group(2):
script_name = res.group(2)
if not script_name:
logger.critical(f"Missing script_name")
return R_ERROR_DAEMON
if res.group(3):
args = res.group(3).split()
args.insert(0, f'{script_name}.py')
if script_name not in include_set:
+ # call dependents now if last element of prio queue is run
+ # independent of configd
+ if last:
+ call_dependents(dependent_func=config.dependent_func)
return R_PASS
with stdout_redirected(session_out, session_mode):
result = run_script(conf_mode_scripts[script_name], config, args)
+ if last:
+ call_dependents(dependent_func=config.dependent_func)
+
return result
def remove_if_file(f: str):
try:
os.remove(f)
except FileNotFoundError:
pass
except OSError:
raise
def shutdown():
remove_if_file(configd_env_file)
os.symlink(configd_env_unset_file, configd_env_file)
sys.exit(0)
if __name__ == '__main__':
context = zmq.Context()
socket = context.socket(zmq.REP)
# Set the right permissions on the socket, then change it back
o_mask = os.umask(0)
socket.bind(SOCKET_PATH)
os.umask(o_mask)
cfg_group = grp.getgrnam(CFG_GROUP)
os.setgid(cfg_group.gr_gid)
os.environ['SUDO_USER'] = 'vyos'
os.environ['SUDO_GID'] = str(cfg_group.gr_gid)
def sig_handler(signum, frame):
shutdown()
signal.signal(signal.SIGTERM, sig_handler)
signal.signal(signal.SIGINT, sig_handler)
# Define the vyshim environment variable
remove_if_file(configd_env_file)
os.symlink(configd_env_set_file, configd_env_file)
config = None
while True:
# Wait for next request from client
msg = socket.recv().decode()
logger.debug(f"Received message: {msg}")
message = json.loads(msg)
if message["type"] == "init":
resp = "init"
socket.send(resp.encode())
config = initialization(socket)
elif message["type"] == "node":
- res = process_node_data(config, message["data"])
+ if message["last"]:
+ logger.debug(f'final element of priority queue')
+ res = process_node_data(config, message["data"], message["last"])
response = res.to_bytes(1, byteorder=sys.byteorder)
logger.debug(f"Sending response {res}")
socket.send(response)
else:
logger.critical(f"Unexpected message: {message}")
diff --git a/src/shim/vyshim.c b/src/shim/vyshim.c
index cae8b6152..41723e7a4 100644
--- a/src/shim/vyshim.c
+++ b/src/shim/vyshim.c
@@ -1,301 +1,309 @@
/*
- * Copyright (C) 2020 VyOS maintainers and contributors
+ * Copyright (C) 2020-2024 VyOS maintainers and contributors
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 or later as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <string.h>
#include <sys/time.h>
#include <time.h>
#include <stdint.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <zmq.h>
#include "mkjson.h"
/*
*
*
*/
#if DEBUG
#define DEBUG_ON 1
#else
#define DEBUG_ON 0
#endif
#define debug_print(fmt, ...) \
do { if (DEBUG_ON) fprintf(stderr, fmt, ##__VA_ARGS__); } while (0)
#define debug_call(f) \
do { if (DEBUG_ON) f; } while (0)
#define SOCKET_PATH "ipc:///run/vyos-configd.sock"
#define GET_ACTIVE "cli-shell-api --show-active-only --show-show-defaults --show-ignore-edit showConfig"
#define GET_SESSION "cli-shell-api --show-working-only --show-show-defaults --show-ignore-edit showConfig"
#define COMMIT_MARKER "/var/tmp/initial_in_commit"
+#define QUEUE_MARKER "/var/tmp/last_in_queue"
enum {
SUCCESS = 1 << 0,
ERROR_COMMIT = 1 << 1,
ERROR_DAEMON = 1 << 2,
PASS = 1 << 3
};
volatile int init_alarm = 0;
volatile int timeout = 0;
int initialization(void *);
int pass_through(char **, int);
void timer_handler(int);
double get_posix_clock_time(void);
int main(int argc, char* argv[])
{
// string for node data: conf_mode script and tagnode, if applicable
char string_node_data[256];
string_node_data[0] = '\0';
void *context = zmq_ctx_new();
void *requester = zmq_socket(context, ZMQ_REQ);
int ex_index;
int init_timeout = 0;
+ int last = 0;
debug_print("Connecting to vyos-configd ...\n");
zmq_connect(requester, SOCKET_PATH);
for (int i = 1; i < argc ; i++) {
strncat(&string_node_data[0], argv[i], 127);
}
debug_print("data to send: %s\n", string_node_data);
char *test = strstr(string_node_data, "VYOS_TAGNODE_VALUE");
ex_index = test ? 2 : 1;
if (access(COMMIT_MARKER, F_OK) != -1) {
init_timeout = initialization(requester);
if (!init_timeout) remove(COMMIT_MARKER);
}
// if initial communication failed, pass through execution of script
if (init_timeout) {
int ret = pass_through(argv, ex_index);
return ret;
}
+ if (access(QUEUE_MARKER, F_OK) != -1) {
+ last = 1;
+ remove(QUEUE_MARKER);
+ }
+
char error_code[1];
debug_print("Sending node data ...\n");
- char *string_node_data_msg = mkjson(MKJSON_OBJ, 2,
+ char *string_node_data_msg = mkjson(MKJSON_OBJ, 3,
MKJSON_STRING, "type", "node",
+ MKJSON_BOOL, "last", last,
MKJSON_STRING, "data", &string_node_data[0]);
zmq_send(requester, string_node_data_msg, strlen(string_node_data_msg), 0);
zmq_recv(requester, error_code, 1, 0);
debug_print("Received node data receipt\n");
int err = (int)error_code[0];
free(string_node_data_msg);
zmq_close(requester);
zmq_ctx_destroy(context);
if (err & PASS) {
debug_print("Received PASS\n");
int ret = pass_through(argv, ex_index);
return ret;
}
if (err & ERROR_DAEMON) {
debug_print("Received ERROR_DAEMON\n");
int ret = pass_through(argv, ex_index);
return ret;
}
if (err & ERROR_COMMIT) {
debug_print("Received ERROR_COMMIT\n");
return -1;
}
return 0;
}
int initialization(void* Requester)
{
char *active_str = NULL;
size_t active_len = 0;
char *session_str = NULL;
size_t session_len = 0;
char *empty_string = "\n";
char buffer[16];
struct sigaction sa;
struct itimerval timer, none_timer;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = &timer_handler;
sigaction(SIGALRM, &sa, NULL);
timer.it_value.tv_sec = 0;
timer.it_value.tv_usec = 10000;
timer.it_interval.tv_sec = timer.it_interval.tv_usec = 0;
none_timer.it_value.tv_sec = none_timer.it_value.tv_usec = 0;
none_timer.it_interval.tv_sec = none_timer.it_interval.tv_usec = 0;
double prev_time_value, time_value;
double time_diff;
char *pid_val = getenv("VYATTA_CONFIG_TMP");
strsep(&pid_val, "_");
debug_print("config session pid: %s\n", pid_val);
debug_print("Sending init announcement\n");
char *init_announce = mkjson(MKJSON_OBJ, 1,
MKJSON_STRING, "type", "init");
// check for timeout on initial contact
while (!init_alarm) {
debug_call(prev_time_value = get_posix_clock_time());
setitimer(ITIMER_REAL, &timer, NULL);
zmq_send(Requester, init_announce, strlen(init_announce), 0);
zmq_recv(Requester, buffer, 16, 0);
setitimer(ITIMER_REAL, &none_timer, &timer);
debug_call(time_value = get_posix_clock_time());
debug_print("Received init receipt\n");
debug_call(time_diff = time_value - prev_time_value);
debug_print("time elapse %f\n", time_diff);
break;
}
free(init_announce);
if (timeout) return -1;
FILE *fp_a = popen(GET_ACTIVE, "r");
getdelim(&active_str, &active_len, '\0', fp_a);
int ret = pclose(fp_a);
if (!ret) {
debug_print("Sending active config\n");
zmq_send(Requester, active_str, active_len - 1, 0);
zmq_recv(Requester, buffer, 16, 0);
debug_print("Received active receipt\n");
} else {
debug_print("Sending empty active config\n");
zmq_send(Requester, empty_string, 0, 0);
zmq_recv(Requester, buffer, 16, 0);
debug_print("Received active receipt\n");
}
free(active_str);
FILE *fp_s = popen(GET_SESSION, "r");
getdelim(&session_str, &session_len, '\0', fp_s);
pclose(fp_s);
debug_print("Sending session config\n");
zmq_send(Requester, session_str, session_len - 1, 0);
zmq_recv(Requester, buffer, 16, 0);
debug_print("Received session receipt\n");
free(session_str);
debug_print("Sending config session pid\n");
zmq_send(Requester, pid_val, strlen(pid_val), 0);
zmq_recv(Requester, buffer, 16, 0);
debug_print("Received pid receipt\n");
return 0;
}
int pass_through(char **argv, int ex_index)
{
char **newargv = NULL;
pid_t child_pid;
newargv = &argv[ex_index];
if (ex_index > 1) {
putenv(argv[ex_index - 1]);
}
debug_print("pass-through invoked\n");
if ((child_pid=fork()) < 0) {
debug_print("fork() failed\n");
return -1;
} else if (child_pid == 0) {
if (-1 == execv(argv[ex_index], newargv)) {
debug_print("pass_through execve failed %s: %s\n",
argv[ex_index], strerror(errno));
return -1;
}
} else if (child_pid > 0) {
int status;
pid_t wait_pid = waitpid(child_pid, &status, 0);
if (wait_pid < 0) {
debug_print("waitpid() failed\n");
return -1;
} else if (wait_pid == child_pid) {
if (WIFEXITED(status)) {
debug_print("child exited with code %d\n",
WEXITSTATUS(status));
return WEXITSTATUS(status);
}
}
}
return 0;
}
void timer_handler(int signum)
{
debug_print("timer_handler invoked\n");
timeout = 1;
init_alarm = 1;
return;
}
#ifdef _POSIX_MONOTONIC_CLOCK
double get_posix_clock_time(void)
{
struct timespec ts;
if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
return (double) (ts.tv_sec + ts.tv_nsec / 1000000000.0);
} else {
return 0;
}
}
#else
double get_posix_clock_time(void)
{return (double)0;}
#endif