diff --git a/data/templates/https/nginx.default.j2 b/data/templates/https/nginx.default.j2 index dbb08e187..753c3a5c9 100644 --- a/data/templates/https/nginx.default.j2 +++ b/data/templates/https/nginx.default.j2 @@ -1,56 +1,56 @@ ### Autogenerated by https.py ### # Default server configuration {% for server in server_block_list %} server { # SSL configuration # {% if server.address == '*' %} listen {{ server.port }} ssl; listen [::]:{{ server.port }} ssl; {% else %} listen {{ server.address | bracketize_ipv6 }}:{{ server.port }} ssl; {% endif %} {% for name in server.name %} server_name {{ name }}; {% endfor %} {% if server.certbot %} ssl_certificate {{ server.certbot_dir }}/live/{{ server.certbot_domain_dir }}/fullchain.pem; ssl_certificate_key {{ server.certbot_dir }}/live/{{ server.certbot_domain_dir }}/privkey.pem; include {{ server.certbot_dir }}/options-ssl-nginx.conf; ssl_dhparam {{ server.certbot_dir }}/ssl-dhparams.pem; {% elif server.vyos_cert %} ssl_certificate {{ server.vyos_cert.crt }}; ssl_certificate_key {{ server.vyos_cert.key }}; {% else %} # # Self signed certs generated by the ssl-cert package # Don't use them in a production server! # include snippets/snakeoil.conf; {% endif %} ssl_protocols TLSv1.2 TLSv1.3; # proxy settings for HTTP API, if enabled; 503, if not - location ~ /(retrieve|configure|config-file|image|generate|show|reset|docs|openapi.json|redoc|graphql) { + location ~ /(retrieve|configure|config-file|image|container-image|generate|show|reset|docs|openapi.json|redoc|graphql) { {% if server.api %} {% if server.api.socket %} proxy_pass http://unix:/run/api.sock; {% else %} proxy_pass http://localhost:{{ server.api.port }}; {% endif %} proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 600; proxy_buffering off; {% else %} return 503; {% endif %} } error_page 497 =301 https://$host:{{ server.port }}$request_uri; } {% endfor %} diff --git a/op-mode-definitions/container.xml.in b/op-mode-definitions/container.xml.in index 786bd66d3..ada9a4d59 100644 --- a/op-mode-definitions/container.xml.in +++ b/op-mode-definitions/container.xml.in @@ -1,176 +1,176 @@ <?xml version="1.0"?> <interfaceDefinition> <node name="add"> <children> <node name="container"> <properties> <help>Add container image</help> </properties> <children> <tagNode name="image"> <properties> <help>Pull a new image for container</help> </properties> - <command>sudo podman image pull "${4}"</command> + <command>sudo ${vyos_op_scripts_dir}/container.py add_image --name "${4}"</command> </tagNode> </children> </node> </children> </node> <node name="connect"> <children> <tagNode name="container"> <properties> <help>Attach to a running container</help> <completionHelp> <path>container name</path> </completionHelp> </properties> <command>sudo podman exec --interactive --tty "$3" /bin/sh</command> </tagNode> </children> </node> <node name="delete"> <children> <node name="container"> <properties> <help>Delete container image</help> </properties> <children> <tagNode name="image"> <properties> <help>Delete container image</help> <completionHelp> <script>sudo podman image ls -q</script> </completionHelp> </properties> - <command>sudo podman image rm --force "${4}"</command> + <command>sudo ${vyos_op_scripts_dir}/container.py delete_image --name "${4}"</command> </tagNode> </children> </node> </children> </node> <node name="generate"> <children> <node name="container"> <properties> <help>Generate Container Image</help> </properties> <children> <tagNode name="image"> <properties> <help>Name of container image (tag)</help> </properties> <children> <tagNode name="path"> <properties> <help>Path to Dockerfile</help> <completionHelp> <list><filename></list> </completionHelp> </properties> <command>sudo podman build --net host --layers --force-rm --tag "$4" $6</command> </tagNode> </children> </tagNode> </children> </node> </children> </node> <node name="monitor"> <children> <node name="log"> <children> <tagNode name="container"> <properties> <help>Monitor last lines of container logs</help> <completionHelp> <path>container name</path> </completionHelp> </properties> <command>sudo podman logs --follow --names "$4"</command> </tagNode> </children> </node> </children> </node> <node name="show"> <children> <node name="container"> <properties> <help>Show containers</help> </properties> <command>sudo ${vyos_op_scripts_dir}/container.py show_container</command> <children> <leafNode name="image"> <properties> <help>Show container image</help> </properties> <command>sudo ${vyos_op_scripts_dir}/container.py show_image</command> </leafNode> <tagNode name="log"> <properties> <help>Show logs from a given container</help> <completionHelp> <path>container name</path> </completionHelp> </properties> <command>sudo podman logs --names "$4"</command> </tagNode> <leafNode name="network"> <properties> <help>Show available container networks</help> </properties> <command>sudo ${vyos_op_scripts_dir}/container.py show_network</command> </leafNode> </children> </node> <node name="log"> <children> <tagNode name="container"> <properties> <help>Show logs from a given container</help> <completionHelp> <path>container name</path> </completionHelp> </properties> <command>sudo podman logs --names "$4"</command> </tagNode> </children> </node> </children> </node> <node name="restart"> <children> <tagNode name="container"> <properties> <help>Restart a given container</help> <completionHelp> <path>container name</path> </completionHelp> </properties> <command>sudo ${vyos_op_scripts_dir}/container.py restart --name="$3"</command> </tagNode> </children> </node> <node name="update"> <children> <node name="container"> <properties> <help>Update a container image</help> </properties> <children> <tagNode name="image"> <properties> <help>Update container image</help> <completionHelp> <path>container name</path> </completionHelp> </properties> <command>if cli-shell-api existsActive container name "$4"; then sudo podman pull $(cli-shell-api returnActiveValue container name "$4" image); else echo "Container $4 does not exist"; fi</command> </tagNode> </children> </node> </children> </node> </interfaceDefinition> diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py index 3a60f6d92..9864aa5c5 100644 --- a/python/vyos/configsession.py +++ b/python/vyos/configsession.py @@ -1,206 +1,220 @@ # configsession -- the write API for the VyOS running config # Copyright (C) 2019 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; # either version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License along with this library; # if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import os import re import sys import subprocess from vyos.util import is_systemd_service_running CLI_SHELL_API = '/bin/cli-shell-api' SET = '/opt/vyatta/sbin/my_set' DELETE = '/opt/vyatta/sbin/my_delete' COMMENT = '/opt/vyatta/sbin/my_comment' COMMIT = '/opt/vyatta/sbin/my_commit' DISCARD = '/opt/vyatta/sbin/my_discard' SHOW_CONFIG = ['/bin/cli-shell-api', 'showConfig'] LOAD_CONFIG = ['/bin/cli-shell-api', 'loadFile'] MIGRATE_LOAD_CONFIG = ['/usr/libexec/vyos/vyos-load-config.py'] SAVE_CONFIG = ['/opt/vyatta/sbin/vyatta-save-config.pl'] INSTALL_IMAGE = ['/opt/vyatta/sbin/install-image', '--url'] REMOVE_IMAGE = ['/opt/vyatta/bin/vyatta-boot-image.pl', '--del'] GENERATE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'generate'] SHOW = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'show'] RESET = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'reset'] +ADD = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'add'] +DELETE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'delete'] # Default "commit via" string APP = "vyos-http-api" # When started as a service rather than from a user shell, # the process lacks the VyOS-specific environment that comes # from bash configs, so we have to inject it # XXX: maybe it's better to do via a systemd environment file def inject_vyos_env(env): env['VYATTA_CFG_GROUP_NAME'] = 'vyattacfg' env['VYATTA_USER_LEVEL_DIR'] = '/opt/vyatta/etc/shell/level/admin' env['VYATTA_PROCESS_CLIENT'] = 'gui2_rest' env['VYOS_HEADLESS_CLIENT'] = 'vyos_http_api' env['vyatta_bindir']= '/opt/vyatta/bin' env['vyatta_cfg_templates'] = '/opt/vyatta/share/vyatta-cfg/templates' env['vyatta_configdir'] = '/opt/vyatta/config' env['vyatta_datadir'] = '/opt/vyatta/share' env['vyatta_datarootdir'] = '/opt/vyatta/share' env['vyatta_libdir'] = '/opt/vyatta/lib' env['vyatta_libexecdir'] = '/opt/vyatta/libexec' env['vyatta_op_templates'] = '/opt/vyatta/share/vyatta-op/templates' env['vyatta_prefix'] = '/opt/vyatta' env['vyatta_sbindir'] = '/opt/vyatta/sbin' env['vyatta_sysconfdir'] = '/opt/vyatta/etc' env['vyos_bin_dir'] = '/usr/bin' env['vyos_cfg_templates'] = '/opt/vyatta/share/vyatta-cfg/templates' env['vyos_completion_dir'] = '/usr/libexec/vyos/completion' env['vyos_configdir'] = '/opt/vyatta/config' env['vyos_conf_scripts_dir'] = '/usr/libexec/vyos/conf_mode' env['vyos_datadir'] = '/opt/vyatta/share' env['vyos_datarootdir']= '/opt/vyatta/share' env['vyos_libdir'] = '/opt/vyatta/lib' env['vyos_libexec_dir'] = '/usr/libexec/vyos' env['vyos_op_scripts_dir'] = '/usr/libexec/vyos/op_mode' env['vyos_op_templates'] = '/opt/vyatta/share/vyatta-op/templates' env['vyos_prefix'] = '/opt/vyatta' env['vyos_sbin_dir'] = '/usr/sbin' env['vyos_validators_dir'] = '/usr/libexec/vyos/validators' # if running the vyos-configd daemon, inject the vyshim env var if is_systemd_service_running('vyos-configd.service'): env['vyshim'] = '/usr/sbin/vyshim' return env class ConfigSessionError(Exception): pass class ConfigSession(object): """ The write API of VyOS. """ def __init__(self, session_id, app=APP): """ Creates a new config session. Args: session_id (str): Session identifier app (str): Application name, purely informational Note: The session identifier MUST be globally unique within the system. The best practice is to only have one ConfigSession object per process and used the PID for the session identifier. """ env_str = subprocess.check_output([CLI_SHELL_API, 'getSessionEnv', str(session_id)]) self.__session_id = session_id # Extract actual variables from the chunk of shell it outputs # XXX: it's better to extend cli-shell-api to provide easily readable output env_list = re.findall(r'([A-Z_]+)=([^;\s]+)', env_str.decode()) session_env = os.environ session_env = inject_vyos_env(session_env) for k, v in env_list: session_env[k] = v self.__session_env = session_env self.__session_env["COMMIT_VIA"] = app self.__run_command([CLI_SHELL_API, 'setupSession']) def __del__(self): try: output = subprocess.check_output([CLI_SHELL_API, 'teardownSession'], env=self.__session_env).decode().strip() if output: print("cli-shell-api teardownSession output for sesion {0}: {1}".format(self.__session_id, output), file=sys.stderr) except Exception as e: print("Could not tear down session {0}: {1}".format(self.__session_id, e), file=sys.stderr) def __run_command(self, cmd_list): p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=self.__session_env) (stdout_data, stderr_data) = p.communicate() output = stdout_data.decode() result = p.wait() if result != 0: raise ConfigSessionError(output) return output def get_session_env(self): return self.__session_env def set(self, path, value=None): if not value: value = [] else: value = [value] self.__run_command([SET] + path + value) def delete(self, path, value=None): if not value: value = [] else: value = [value] self.__run_command([DELETE] + path + value) def comment(self, path, value=None): if not value: value = [""] else: value = [value] self.__run_command([COMMENT] + path + value) def commit(self): out = self.__run_command([COMMIT]) return out def discard(self): self.__run_command([DISCARD]) def show_config(self, path, format='raw'): config_data = self.__run_command(SHOW_CONFIG + path) if format == 'raw': return config_data def load_config(self, file_path): out = self.__run_command(LOAD_CONFIG + [file_path]) return out def migrate_and_load_config(self, file_path): out = self.__run_command(MIGRATE_LOAD_CONFIG + [file_path]) return out def save_config(self, file_path): out = self.__run_command(SAVE_CONFIG + [file_path]) return out def install_image(self, url): out = self.__run_command(INSTALL_IMAGE + [url]) return out def remove_image(self, name): out = self.__run_command(REMOVE_IMAGE + [name]) return out def generate(self, path): out = self.__run_command(GENERATE + path) return out def show(self, path): out = self.__run_command(SHOW + path) return out def reset(self, path): out = self.__run_command(RESET + path) return out + + def add_container_image(self, name): + out = self.__run_command(ADD + ['container', 'image'] + [name]) + return out + + def delete_container_image(self, name): + out = self.__run_command(DELETE + ['container', 'image'] + [name]) + return out + + def show_container_image(self): + out = self.__run_command(SHOW + ['container', 'image']) + return out diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 5ff768859..17a9ab581 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -1,224 +1,224 @@ # Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. import re import sys import typing from humps import decamelize class Error(Exception): """ Any error that makes requested operation impossible to complete for reasons unrelated to the user input or script logic. """ pass class UnconfiguredSubsystem(Error): """ Requested operation is valid, but cannot be completed because corresponding subsystem is not configured and running. """ pass class DataUnavailable(Error): """ Requested operation is valid, but cannot be completed because data for it is not available. This error MAY be treated as temporary because such issues are often caused by transient events such as service restarts. """ pass class PermissionDenied(Error): """ Requested operation is valid, but the caller has no permission to perform it. """ pass class IncorrectValue(Error): """ Requested operation is valid, but an argument provided has an incorrect value, preventing successful completion. """ pass class CommitInProgress(Error): """ Requested operation is valid, but not possible at the time due to a commit being in progress. """ pass class InternalError(Error): """ Any situation when VyOS detects that it could not perform an operation correctly due to logic errors in its own code or errors in underlying software. """ pass def _is_op_mode_function_name(name): - if re.match(r"^(show|clear|reset|restart)", name): + if re.match(r"^(show|clear|reset|restart|add|delete)", name): return True else: return False def _is_show(name): if re.match(r"^show", name): return True else: return False def _get_op_mode_functions(module): from inspect import getmembers, isfunction # Get all functions in that module funcs = getmembers(module, isfunction) # getmembers returns (name, func) tuples funcs = list(filter(lambda ft: _is_op_mode_function_name(ft[0]), funcs)) funcs_dict = {} for (name, thunk) in funcs: funcs_dict[name] = thunk return funcs_dict def _is_optional_type(t): # Optional[t] is internally an alias for Union[t, NoneType] # and there's no easy way to get union members it seems if (type(t) == typing._UnionGenericAlias): if (len(t.__args__) == 2): if t.__args__[1] == type(None): return True return False def _get_arg_type(t): """ Returns the type itself if it's a primitive type, or the "real" type of typing.Optional Doesn't work with anything else at the moment! """ if _is_optional_type(t): return t.__args__[0] else: return t def _normalize_field_name(name): # Convert the name to string if it is not # (in some cases they may be numbers) name = str(name) # Replace all separators with underscores name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) # Replace specific characters with textual descriptions name = re.sub(r'@', '_at_', name) name = re.sub(r'%', '_percentage_', name) name = re.sub(r'~', '_tilde_', name) # Force all letters to lowercase name = name.lower() # Remove leading and trailing underscores, if any name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name) # Ensure there are only single underscores name = re.sub(r'_+', '_', name) return name def _normalize_dict_field_names(old_dict): new_dict = {} for key in old_dict: new_key = _normalize_field_name(key) new_dict[new_key] = _normalize_field_names(old_dict[key]) # Sanity check if len(old_dict) != len(new_dict): raise InternalError("Dictionary fields do not allow unique normalization") else: return new_dict def _normalize_field_names(value): if isinstance(value, dict): return _normalize_dict_field_names(value) elif isinstance(value, list): return list(map(lambda v: _normalize_field_names(v), value)) else: return value def run(module): from argparse import ArgumentParser functions = _get_op_mode_functions(module) parser = ArgumentParser() subparsers = parser.add_subparsers(dest="subcommand") for function_name in functions: subparser = subparsers.add_parser(function_name, help=functions[function_name].__doc__) type_hints = typing.get_type_hints(functions[function_name]) if 'return' in type_hints: del type_hints['return'] for opt in type_hints: th = type_hints[opt] if _get_arg_type(th) == bool: subparser.add_argument(f"--{opt}", action='store_true') else: if _is_optional_type(th): subparser.add_argument(f"--{opt}", type=_get_arg_type(th), default=None) else: subparser.add_argument(f"--{opt}", type=_get_arg_type(th), required=True) # Get options as a dict rather than a namespace, # so that we can modify it and pack for passing to functions args = vars(parser.parse_args()) if not args["subcommand"]: print("Subcommand required!") parser.print_usage() sys.exit(1) function_name = args["subcommand"] func = functions[function_name] # Remove the subcommand from the arguments, # it would cause an extra argument error when we pass the dict to a function del args["subcommand"] # Show commands must always get the "raw" argument, # but other commands (clear/reset/restart) should not, # because they produce no output and it makes no sense for them. if ("raw" not in args) and _is_show(function_name): args["raw"] = False if re.match(r"^show", function_name): # Show commands are slightly special: # they may return human-formatted output # or a raw dict that we need to serialize in JSON for printing res = func(**args) if not args["raw"]: return res else: res = decamelize(res) res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) else: # Other functions should not return anything, # although they may print their own warnings or status messages func(**args) diff --git a/src/op_mode/container.py b/src/op_mode/container.py index ecefc556e..d48766a0c 100755 --- a/src/op_mode/container.py +++ b/src/op_mode/container.py @@ -1,84 +1,97 @@ #!/usr/bin/env python3 # # Copyright (C) 2022 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import json import sys from sys import exit from vyos.util import cmd import vyos.opmode def _get_json_data(command: str) -> list: """ Get container command format JSON """ return cmd(f'{command} --format json') def _get_raw_data(command: str) -> list: json_data = _get_json_data(command) data = json.loads(json_data) return data +def add_image(name: str): + from vyos.util import rc_cmd + + rc, output = rc_cmd(f'podman image pull {name}') + if rc != 0: + raise vyos.opmode.InternalError(output) + +def delete_image(name: str): + from vyos.util import rc_cmd + + rc, output = rc_cmd(f'podman image rm --force {name}') + if rc != 0: + raise vyos.opmode.InternalError(output) def show_container(raw: bool): command = 'podman ps --all' container_data = _get_raw_data(command) if raw: return container_data else: return cmd(command) def show_image(raw: bool): command = 'podman image ls' container_data = _get_raw_data('podman image ls') if raw: return container_data else: return cmd(command) def show_network(raw: bool): command = 'podman network ls' container_data = _get_raw_data(command) if raw: return container_data else: return cmd(command) def restart(name: str): from vyos.util import rc_cmd rc, output = rc_cmd(f'systemctl restart vyos-container-{name}.service') if rc != 0: print(output) return None print(f'Container name "{name}" restarted!') return output if __name__ == '__main__': try: res = vyos.opmode.run(sys.modules[__name__]) if res: print(res) except (ValueError, vyos.opmode.Error) as e: print(e) sys.exit(1) diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py index 211f8ce19..c1eb493db 100644 --- a/src/services/api/graphql/libs/op_mode.py +++ b/src/services/api/graphql/libs/op_mode.py @@ -1,101 +1,101 @@ # Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. import os import re import typing import importlib.util from typing import Union from humps import decamelize from vyos.defaults import directories from vyos.util import load_as_module from vyos.opmode import _normalize_field_names def load_op_mode_as_module(name: str): path = os.path.join(directories['op_mode'], name) name = os.path.splitext(name)[0].replace('-', '_') return load_as_module(name, path) def is_op_mode_function_name(name): - if re.match(r"^(show|clear|reset|restart)", name): + if re.match(r"^(show|clear|reset|restart|add|delete)", name): return True return False def is_show_function_name(name): if re.match(r"^show", name): return True return False def _nth_split(delim: str, n: int, s: str): groups = s.split(delim) l = len(groups) if n > l-1 or n < 1: return (s, '') return (delim.join(groups[:n]), delim.join(groups[n:])) def _nth_rsplit(delim: str, n: int, s: str): groups = s.split(delim) l = len(groups) if n > l-1 or n < 1: return (s, '') return (delim.join(groups[:l-n]), delim.join(groups[l-n:])) # Since we have mangled possible hyphens in the file name while constructing # the snake case of the query/mutation name, we will need to recover the # file name by searching with mangling: def _filter_on_mangled(test): def func(elem): mangle = os.path.splitext(elem)[0].replace('-', '_') return test == mangle return func # Find longest name in concatenated string that matches the basename of an # op-mode script. Should one prefer to concatenate in the reverse order # (script_name + '_' + function_name), use _nth_rsplit. def split_compound_op_mode_name(name: str, files: list): for i in range(1, name.count('_') + 1): pair = _nth_split('_', i, name) f = list(filter(_filter_on_mangled(pair[1]), files)) if f: pair = (pair[0], f[0]) return pair return (name, '') def snake_to_pascal_case(name: str) -> str: res = ''.join(map(str.title, name.split('_'))) return res def map_type_name(type_name: type, optional: bool = False) -> str: if type_name == str: return 'String!' if not optional else 'String = null' if type_name == int: return 'Int!' if not optional else 'Int = null' if type_name == bool: return 'Boolean = false' if typing.get_origin(type_name) == list: if not optional: return f'[{map_type_name(typing.get_args(type_name)[0])}]!' return f'[{map_type_name(typing.get_args(type_name)[0])}]' # typing.Optional is typing.Union[_, NoneType] if (typing.get_origin(type_name) is typing.Union and typing.get_args(type_name)[1] == type(None)): return f'{map_type_name(typing.get_args(type_name)[0], optional=True)}' # scalar 'Generic' is defined in schema.graphql return 'Generic' def normalize_output(result: Union[dict, list]) -> Union[dict, list]: return _normalize_field_names(decamelize(result)) diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 60ea9a5ee..f59e089ae 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -1,727 +1,771 @@ #!/usr/share/vyos-http-api-tools/bin/python3 # # Copyright (C) 2019-2021 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # # import os import sys import grp import copy import json import logging import traceback import threading from typing import List, Union, Callable, Dict import uvicorn from fastapi import FastAPI, Depends, Request, Response, HTTPException from fastapi.responses import HTMLResponse from fastapi.exceptions import RequestValidationError from fastapi.routing import APIRoute from pydantic import BaseModel, StrictStr, validator from starlette.middleware.cors import CORSMiddleware from starlette.datastructures import FormData from starlette.formparsers import FormParser, MultiPartParser from multipart.multipart import parse_options_header from ariadne.asgi import GraphQL import vyos.config from vyos.configsession import ConfigSession, ConfigSessionError import api.graphql.state DEFAULT_CONFIG_FILE = '/etc/vyos/http-api.conf' CFG_GROUP = 'vyattacfg' debug = True logger = logging.getLogger(__name__) logs_handler = logging.StreamHandler() logger.addHandler(logs_handler) if debug: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) # Giant lock! lock = threading.Lock() def load_server_config(): with open(DEFAULT_CONFIG_FILE) as f: config = json.load(f) return config def check_auth(key_list, key): key_id = None for k in key_list: if k['key'] == key: key_id = k['id'] return key_id def error(code, msg): resp = {"success": False, "error": msg, "data": None} resp = json.dumps(resp) return HTMLResponse(resp, status_code=code) def success(data): resp = {"success": True, "data": data, "error": None} resp = json.dumps(resp) return HTMLResponse(resp) # Pydantic models for validation # Pydantic will cast when possible, so use StrictStr # validators added as needed for additional constraints # schema_extra adds anotations to OpenAPI, to add examples class ApiModel(BaseModel): key: StrictStr class BaseConfigureModel(BaseModel): op: StrictStr path: List[StrictStr] value: StrictStr = None @validator("path", pre=True, always=True) def check_non_empty(cls, path): assert len(path) > 0 return path class ConfigureModel(ApiModel): op: StrictStr path: List[StrictStr] value: StrictStr = None @validator("path", pre=True, always=True) def check_non_empty(cls, path): assert len(path) > 0 return path class Config: schema_extra = { "example": { "key": "id_key", "op": "set | delete | comment", "path": ['config', 'mode', 'path'], } } class ConfigureListModel(ApiModel): commands: List[BaseConfigureModel] class Config: schema_extra = { "example": { "key": "id_key", "commands": "list of commands", } } class RetrieveModel(ApiModel): op: StrictStr path: List[StrictStr] configFormat: StrictStr = None class Config: schema_extra = { "example": { "key": "id_key", "op": "returnValue | returnValues | exists | showConfig", "path": ['config', 'mode', 'path'], "configFormat": "json (default) | json_ast | raw", } } class ConfigFileModel(ApiModel): op: StrictStr file: StrictStr = None class Config: schema_extra = { "example": { "key": "id_key", "op": "save | load", "file": "filename", } } class ImageModel(ApiModel): op: StrictStr url: StrictStr = None name: StrictStr = None class Config: schema_extra = { "example": { "key": "id_key", "op": "add | delete", "url": "imagelocation", "name": "imagename", } } +class ContainerImageModel(ApiModel): + op: StrictStr + name: StrictStr = None + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "op": "add | delete | show", + "name": "imagename", + } + } + class GenerateModel(ApiModel): op: StrictStr path: List[StrictStr] class Config: schema_extra = { "example": { "key": "id_key", "op": "generate", "path": ["op", "mode", "path"], } } class ShowModel(ApiModel): op: StrictStr path: List[StrictStr] class Config: schema_extra = { "example": { "key": "id_key", "op": "show", "path": ["op", "mode", "path"], } } class ResetModel(ApiModel): op: StrictStr path: List[StrictStr] class Config: schema_extra = { "example": { "key": "id_key", "op": "reset", "path": ["op", "mode", "path"], } } class Success(BaseModel): success: bool data: Union[str, bool, Dict] error: str class Error(BaseModel): success: bool = False data: Union[str, bool, Dict] error: str responses = { 200: {'model': Success}, 400: {'model': Error}, 422: {'model': Error, 'description': 'Validation Error'}, 500: {'model': Error} } def auth_required(data: ApiModel): key = data.key api_keys = app.state.vyos_keys key_id = check_auth(api_keys, key) if not key_id: raise HTTPException(status_code=401, detail="Valid API key is required") app.state.vyos_id = key_id # override Request and APIRoute classes in order to convert form request to json; # do all explicit validation here, for backwards compatability of error messages; # the explicit validation may be dropped, if desired, in favor of native # validation by FastAPI/Pydantic, as is used for application/json requests class MultipartRequest(Request): ERR_MISSING_KEY = False ERR_MISSING_DATA = False ERR_NOT_JSON = False ERR_NOT_DICT = False ERR_NO_OP = False ERR_NO_PATH = False ERR_EMPTY_PATH = False ERR_PATH_NOT_LIST = False ERR_VALUE_NOT_STRING = False ERR_PATH_NOT_LIST_OF_STR = False offending_command = {} exception = None @property def orig_headers(self): self._orig_headers = super().headers return self._orig_headers @property def headers(self): self._headers = super().headers.mutablecopy() self._headers['content-type'] = 'application/json' return self._headers async def form(self) -> FormData: if not hasattr(self, "_form"): assert ( parse_options_header is not None ), "The `python-multipart` library must be installed to use form parsing." content_type_header = self.orig_headers.get("Content-Type") content_type, options = parse_options_header(content_type_header) if content_type == b"multipart/form-data": multipart_parser = MultiPartParser(self.orig_headers, self.stream()) self._form = await multipart_parser.parse() elif content_type == b"application/x-www-form-urlencoded": form_parser = FormParser(self.orig_headers, self.stream()) self._form = await form_parser.parse() else: self._form = FormData() return self._form async def body(self) -> bytes: if not hasattr(self, "_body"): forms = {} merge = {} body = await super().body() self._body = body form_data = await self.form() if form_data: logger.debug("processing form data") for k, v in form_data.multi_items(): forms[k] = v if 'data' not in forms: self.ERR_MISSING_DATA = True else: try: tmp = json.loads(forms['data']) except json.JSONDecodeError as e: self.ERR_NOT_JSON = True self.exception = e tmp = {} if isinstance(tmp, list): merge['commands'] = tmp else: merge = tmp if 'commands' in merge: cmds = merge['commands'] else: cmds = copy.deepcopy(merge) cmds = [cmds] for c in cmds: if not isinstance(c, dict): self.ERR_NOT_DICT = True self.offending_command = c elif 'op' not in c: self.ERR_NO_OP = True self.offending_command = c elif 'path' not in c: self.ERR_NO_PATH = True self.offending_command = c elif not c['path']: self.ERR_EMPTY_PATH = True self.offending_command = c elif not isinstance(c['path'], list): self.ERR_PATH_NOT_LIST = True self.offending_command = c elif not all(isinstance(el, str) for el in c['path']): self.ERR_PATH_NOT_LIST_OF_STR = True self.offending_command = c elif 'value' in c and not isinstance(c['value'], str): self.ERR_VALUE_NOT_STRING = True self.offending_command = c if 'key' not in forms and 'key' not in merge: self.ERR_MISSING_KEY = True if 'key' in forms and 'key' not in merge: merge['key'] = forms['key'] new_body = json.dumps(merge) new_body = new_body.encode() self._body = new_body return self._body class MultipartRoute(APIRoute): def get_route_handler(self) -> Callable: original_route_handler = super().get_route_handler() async def custom_route_handler(request: Request) -> Response: request = MultipartRequest(request.scope, request.receive) endpoint = request.url.path try: response: Response = await original_route_handler(request) except HTTPException as e: return error(e.status_code, e.detail) except Exception as e: if request.ERR_MISSING_KEY: return error(401, "Valid API key is required") if request.ERR_MISSING_DATA: return error(422, "Non-empty data field is required") if request.ERR_NOT_JSON: return error(400, "Failed to parse JSON: {0}".format(request.exception)) if endpoint == '/configure': if request.ERR_NOT_DICT: return error(400, "Malformed command \"{0}\": any command must be a dict".format(json.dumps(request.offending_command))) if request.ERR_NO_OP: return error(400, "Malformed command \"{0}\": missing \"op\" field".format(json.dumps(request.offending_command))) if request.ERR_NO_PATH: return error(400, "Malformed command \"{0}\": missing \"path\" field".format(json.dumps(request.offending_command))) if request.ERR_EMPTY_PATH: return error(400, "Malformed command \"{0}\": empty path".format(json.dumps(request.offending_command))) if request.ERR_PATH_NOT_LIST: return error(400, "Malformed command \"{0}\": \"path\" field must be a list".format(json.dumps(request.offending_command))) if request.ERR_VALUE_NOT_STRING: return error(400, "Malformed command \"{0}\": \"value\" field must be a string".format(json.dumps(request.offending_command))) if request.ERR_PATH_NOT_LIST_OF_STR: return error(400, "Malformed command \"{0}\": \"path\" field must be a list of strings".format(json.dumps(request.offending_command))) if endpoint in ('/retrieve','/generate','/show','/reset'): if request.ERR_NO_OP or request.ERR_NO_PATH: return error(400, "Missing required field. \"op\" and \"path\" fields are required") - if endpoint in ('/config-file', '/image'): + if endpoint in ('/config-file', '/image', '/container-image'): if request.ERR_NO_OP: return error(400, "Missing required field \"op\"") raise e return response return custom_route_handler app = FastAPI(debug=True, title="VyOS API", version="0.1.0", responses={**responses}, dependencies=[Depends(auth_required)]) app.router.route_class = MultipartRoute @app.exception_handler(RequestValidationError) async def validation_exception_handler(request, exc): return error(400, str(exc.errors()[0])) @app.post('/configure') def configure_op(data: Union[ConfigureModel, ConfigureListModel]): session = app.state.vyos_session env = session.get_session_env() config = vyos.config.Config(session_env=env) # Allow users to pass just one command if not isinstance(data, ConfigureListModel): data = [data] else: data = data.commands # We don't want multiple people/apps to be able to commit at once, # or modify the shared session while someone else is doing the same, # so the lock is really global lock.acquire() status = 200 error_msg = None try: for c in data: op = c.op path = c.path if c.value: value = c.value else: value = "" # For vyos.configsession calls that have no separate value arguments, # and for type checking too cfg_path = " ".join(path + [value]).strip() if op == 'set': # XXX: it would be nice to do a strict check for "path already exists", # but there's probably no way to do that session.set(path, value=value) elif op == 'delete': if app.state.vyos_strict and not config.exists(cfg_path): raise ConfigSessionError("Cannot delete [{0}]: path/value does not exist".format(cfg_path)) session.delete(path, value=value) elif op == 'comment': session.comment(path, value=value) else: raise ConfigSessionError("\"{0}\" is not a valid operation".format(op)) # end for session.commit() logger.info(f"Configuration modified via HTTP API using key '{app.state.vyos_id}'") except ConfigSessionError as e: session.discard() status = 400 if app.state.vyos_debug: logger.critical(f"ConfigSessionError:\n {traceback.format_exc()}") error_msg = str(e) except Exception as e: session.discard() logger.critical(traceback.format_exc()) status = 500 # Don't give the details away to the outer world error_msg = "An internal error occured. Check the logs for details." finally: lock.release() if status != 200: return error(status, error_msg) return success(None) @app.post("/retrieve") def retrieve_op(data: RetrieveModel): session = app.state.vyos_session env = session.get_session_env() config = vyos.config.Config(session_env=env) op = data.op path = " ".join(data.path) try: if op == 'returnValue': res = config.return_value(path) elif op == 'returnValues': res = config.return_values(path) elif op == 'exists': res = config.exists(path) elif op == 'showConfig': config_format = 'json' if data.configFormat: config_format = data.configFormat res = session.show_config(path=data.path) if config_format == 'json': config_tree = vyos.configtree.ConfigTree(res) res = json.loads(config_tree.to_json()) elif config_format == 'json_ast': config_tree = vyos.configtree.ConfigTree(res) res = json.loads(config_tree.to_json_ast()) elif config_format == 'raw': pass else: return error(400, "\"{0}\" is not a valid config format".format(config_format)) else: return error(400, "\"{0}\" is not a valid operation".format(op)) except ConfigSessionError as e: return error(400, str(e)) except Exception as e: logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) @app.post('/config-file') def config_file_op(data: ConfigFileModel): session = app.state.vyos_session op = data.op try: if op == 'save': if data.file: path = data.file else: path = '/config/config.boot' res = session.save_config(path) elif op == 'load': if data.file: path = data.file else: return error(400, "Missing required field \"file\"") res = session.migrate_and_load_config(path) res = session.commit() else: return error(400, "\"{0}\" is not a valid operation".format(op)) except ConfigSessionError as e: return error(400, str(e)) except Exception as e: logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) @app.post('/image') def image_op(data: ImageModel): session = app.state.vyos_session op = data.op try: if op == 'add': if data.url: url = data.url else: return error(400, "Missing required field \"url\"") res = session.install_image(url) elif op == 'delete': if data.name: name = data.name else: return error(400, "Missing required field \"name\"") res = session.remove_image(name) else: return error(400, "\"{0}\" is not a valid operation".format(op)) except ConfigSessionError as e: return error(400, str(e)) except Exception as e: logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) +@app.post('/container-image') +def image_op(data: ContainerImageModel): + session = app.state.vyos_session + + op = data.op + + try: + if op == 'add': + if data.name: + name = data.name + else: + return error(400, "Missing required field \"name\"") + res = session.add_container_image(name) + elif op == 'delete': + if data.name: + name = data.name + else: + return error(400, "Missing required field \"name\"") + res = session.delete_container_image(name) + elif op == 'show': + res = session.show_container_image() + else: + return error(400, "\"{0}\" is not a valid operation".format(op)) + except ConfigSessionError as e: + return error(400, str(e)) + except Exception as e: + logger.critical(traceback.format_exc()) + return error(500, "An internal error occured. Check the logs for details.") + + return success(res) + @app.post('/generate') def generate_op(data: GenerateModel): session = app.state.vyos_session op = data.op path = data.path try: if op == 'generate': res = session.generate(path) else: return error(400, "\"{0}\" is not a valid operation".format(op)) except ConfigSessionError as e: return error(400, str(e)) except Exception as e: logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) @app.post('/show') def show_op(data: ShowModel): session = app.state.vyos_session op = data.op path = data.path try: if op == 'show': res = session.show(path) else: return error(400, "\"{0}\" is not a valid operation".format(op)) except ConfigSessionError as e: return error(400, str(e)) except Exception as e: logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) @app.post('/reset') def reset_op(data: ResetModel): session = app.state.vyos_session op = data.op path = data.path try: if op == 'reset': res = session.reset(path) else: return error(400, "\"{0}\" is not a valid operation".format(op)) except ConfigSessionError as e: return error(400, str(e)) except Exception as e: logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) ### # GraphQL integration ### def graphql_init(fast_api_app): from api.graphql.libs.token_auth import get_user_context api.graphql.state.init() api.graphql.state.settings['app'] = app # import after initializaion of state from api.graphql.bindings import generate_schema schema = generate_schema() in_spec = app.state.vyos_introspection if app.state.vyos_origins: origins = app.state.vyos_origins app.add_route('/graphql', CORSMiddleware(GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"), allow_headers=("Authorization",))) else: app.add_route('/graphql', GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec)) ### if __name__ == '__main__': # systemd's user and group options don't work, do it by hand here, # else no one else will be able to commit cfg_group = grp.getgrnam(CFG_GROUP) os.setgid(cfg_group.gr_gid) # Need to set file permissions to 775 too so that every vyattacfg group member # has write access to the running config os.umask(0o002) try: server_config = load_server_config() except Exception as err: logger.critical(f"Failed to load the HTTP API server config: {err}") sys.exit(1) config_session = ConfigSession(os.getpid()) app.state.vyos_session = config_session app.state.vyos_keys = server_config['api_keys'] app.state.vyos_debug = server_config['debug'] app.state.vyos_strict = server_config['strict'] app.state.vyos_origins = server_config.get('cors', {}).get('allow_origin', []) if 'graphql' in server_config: app.state.vyos_graphql = True if isinstance(server_config['graphql'], dict): if 'introspection' in server_config['graphql']: app.state.vyos_introspection = True else: app.state.vyos_introspection = False # default value is merged in conf_mode http-api.py, if not set app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] app.state.vyos_token_exp = server_config['graphql']['authentication']['expiration'] app.state.vyos_secret_len = server_config['graphql']['authentication']['secret_length'] else: app.state.vyos_graphql = False if app.state.vyos_graphql: graphql_init(app) try: if not server_config['socket']: uvicorn.run(app, host=server_config["listen_address"], port=int(server_config["port"]), proxy_headers=True) else: uvicorn.run(app, uds="/run/api.sock", proxy_headers=True) except OSError as err: logger.critical(f"OSError {err}") sys.exit(1)