diff --git a/debian/control b/debian/control index bac9a85..c412f65 100644 --- a/debian/control +++ b/debian/control @@ -5,10 +5,13 @@ Maintainer: Vladimir Vinogradenko Build-Depends: debhelper-compat (= 12), dh-python, python3-all, + python3-aiohttp-rpc, python3-ixhardware, + python3-jsonschema, python3-licenselib, python3-humanfriendly, python3-psutil, + python3-pyroute2, python3-setuptools Standards-Version: 4.4.0 @@ -20,10 +23,13 @@ Depends: ${misc:Depends}, gdisk, openzfs, parted, + python3-aiohttp-rpc, python3-ixhardware, + python3-jsonschema, python3-licenselib, python3-humanfriendly, python3-psutil, + python3-pyroute2, setserial, squashfs-tools, util-linux diff --git a/debian/rules b/debian/rules index 9881e2a..ff66bf9 100755 --- a/debian/rules +++ b/debian/rules @@ -4,3 +4,5 @@ export PYBUILD_NAME=truenas_installer %: dh $@ --with python3 --buildsystem=pybuild +override_dh_installsystemd: + dh_installsystemd --no-start -r --no-restart-after-upgrade --name=truenas-installer diff --git a/debian/truenas-installer.service b/debian/truenas-installer.service new file mode 100644 index 0000000..8aa4044 --- /dev/null +++ b/debian/truenas-installer.service @@ -0,0 +1,8 @@ +[Unit] +Description=TrueNAS Installer + +[Service] +ExecStart=/usr/bin/python3 -m truenas_installer --server + +[Install] +WantedBy=multi-user.target diff --git a/truenas_installer/__main__.py b/truenas_installer/__main__.py index 9c1bc8c..965fd67 100644 --- a/truenas_installer/__main__.py +++ b/truenas_installer/__main__.py @@ -1,13 +1,40 @@ +import argparse +import asyncio + +from aiohttp import web + from ixhardware import parse_dmi from .installer import Installer +from .installer_menu import InstallerMenu +from .server import InstallerRPCServer -if __name__ == "__main__": +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--server", action="store_true") + args = parser.parse_args() + with open("/etc/version") as f: version = f.read().strip() dmi = parse_dmi() installer = Installer(version, dmi) - installer.run() + + if args.server: + rpc_server = InstallerRPCServer(installer) + app = web.Application() + app.router.add_routes([ + web.get("/", rpc_server.handle_http_request), + ]) + app.on_shutdown.append(rpc_server.on_shutdown) + web.run_app(app, port=80) + else: + loop = asyncio.get_event_loop() + loop.create_task(InstallerMenu(installer).run()) + loop.run_forever() + + +if __name__ == "__main__": + main() diff --git a/truenas_installer/install.py b/truenas_installer/install.py index 0bd99eb..dde54ae 100644 --- a/truenas_installer/install.py +++ b/truenas_installer/install.py @@ -14,7 +14,7 @@ BOOT_POOL = "boot-pool" -async def install(disks, create_swap, set_pmbr, authentication, sql, callback): +async def install(disks, create_swap, set_pmbr, authentication, post_install, sql, callback): with installation_lock: try: if not os.path.exists("/etc/hostid"): @@ -27,7 +27,7 @@ async def install(disks, create_swap, set_pmbr, authentication, sql, callback): callback(0, "Creating boot pool") await create_boot_pool([get_partition(disk, 3) for disk in disks]) try: - await run_installer(disks, authentication, sql, callback) + await run_installer(disks, authentication, post_install, sql, callback) finally: await run(["zpool", "export", "-f", BOOT_POOL]) except subprocess.CalledProcessError as e: @@ -108,7 +108,7 @@ async def create_boot_pool(devices): await run(["zfs", "create", "-o", "canmount=off", "-o", "mountpoint=legacy", f"{BOOT_POOL}/grub"]) -async def run_installer(disks, authentication, sql, callback): +async def run_installer(disks, authentication, post_install, sql, callback): with tempfile.TemporaryDirectory() as src: await run(["mount", "/cdrom/TrueNAS-SCALE.update", src, "-t", "squashfs", "-o", "loop"]) try: @@ -117,6 +117,7 @@ async def run_installer(disks, authentication, sql, callback): "disks": disks, "json": True, "pool_name": BOOT_POOL, + "post_install": post_install, "sql": sql, "src": src, } diff --git a/truenas_installer/installer.py b/truenas_installer/installer.py index fff4aa6..3a9282c 100644 --- a/truenas_installer/installer.py +++ b/truenas_installer/installer.py @@ -1,16 +1,8 @@ -import asyncio - -from .installer_menu import InstallerMenu +import os class Installer: def __init__(self, version, dmi): self.version = version self.dmi = dmi - - def run(self): - loop = asyncio.get_event_loop() - - loop.create_task(InstallerMenu(self).run()) - - loop.run_forever() + self.efi = os.path.exists("/sys/firmware/efi") diff --git a/truenas_installer/installer_menu.py b/truenas_installer/installer_menu.py index 7d095e4..7a2d75b 100644 --- a/truenas_installer/installer_menu.py +++ b/truenas_installer/installer_menu.py @@ -104,7 +104,7 @@ async def _install_upgrade_internal(self): create_swap = await dialog_yesno("Swap", "Create 16GB swap partition on boot devices?") set_pmbr = False - if not os.path.exists("/sys/firmware/efi"): + if not self.installer.efi: set_pmbr = await dialog_yesno( "Legacy Boot", ( @@ -117,7 +117,7 @@ async def _install_upgrade_internal(self): sql = await serial_sql() try: - await install(destination_disks, create_swap, set_pmbr, authentication_method, sql, self._callback) + await install(destination_disks, create_swap, set_pmbr, authentication_method, None, sql, self._callback) except InstallError as e: await dialog_msgbox("Installation Error", e.message) return False diff --git a/truenas_installer/network_interfaces.py b/truenas_installer/network_interfaces.py new file mode 100644 index 0000000..6e89c9b --- /dev/null +++ b/truenas_installer/network_interfaces.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass + +from pyroute2 import IPRoute, NetlinkDumpInterrupted + +__all__ = ["list_network_interfaces"] + + +@dataclass +class NetworkInterface: + name: str + + +async def list_network_interfaces(): + max_retries = 3 + for attempt in range(1, max_retries + 1): + try: + with IPRoute() as ipr: + interfaces = [NetworkInterface(dev.get_attr("IFLA_IFNAME")) for dev in ipr.get_links()] + except NetlinkDumpInterrupted: + if attempt < max_retries: + # When the kernel is producing a dump of a kernel structure + # over multiple netlink messages, and the structure changes + # mid-way, NLM_F_DUMP_INTR is added to the header flags. + # This an indication that the requested dump contains + # inconsistent data and must be re-requested. See function + # nl_dump_check_consistent() in include/net/netlink.h. The + # pyroute2 library raises this specific exception for this + # scenario, so we'll try again (up to a max of 3 times). + continue + else: + raise + + return [ + interface for interface in interfaces + if interface.name not in ["lo"] + ] diff --git a/truenas_installer/server/__init__.py b/truenas_installer/server/__init__.py new file mode 100644 index 0000000..f2bf451 --- /dev/null +++ b/truenas_installer/server/__init__.py @@ -0,0 +1,24 @@ +import aiohttp_rpc + +import truenas_installer.server.api +from truenas_installer.server.api.adoption import adoption_middleware +from .error import exception_middleware +from .method import methods + +__all__ = ["InstallerRPCServer"] + + +class InstallerRPCServer(aiohttp_rpc.WsJsonRpcServer): + def __init__(self, installer): + self.installer = installer + super().__init__( + middlewares=( + adoption_middleware, + exception_middleware, + aiohttp_rpc.middlewares.extra_args_middleware, + ), + ) + + for method in methods.values(): + method.server = self + self.add_method(aiohttp_rpc.protocol.JsonRpcMethod(method.call, name=method.name)) diff --git a/truenas_installer/server/api/__init__.py b/truenas_installer/server/api/__init__.py new file mode 100644 index 0000000..5056783 --- /dev/null +++ b/truenas_installer/server/api/__init__.py @@ -0,0 +1,4 @@ +import truenas_installer.server.api.adoption +import truenas_installer.server.api.info +import truenas_installer.server.api.install +import truenas_installer.server.api.power diff --git a/truenas_installer/server/api/adoption.py b/truenas_installer/server/api/adoption.py new file mode 100644 index 0000000..25c7dc5 --- /dev/null +++ b/truenas_installer/server/api/adoption.py @@ -0,0 +1,59 @@ +import errno +import secrets +import typing + +from aiohttp_rpc import errors, protocol + +from truenas_installer.server.error import Error +from truenas_installer.server.method import method + +__all__ = ["is_adopted", "adopt", "authenticate"] + +access_key = None + + +@method(None, {"type": "boolean"}) +async def is_adopted(context): + return access_key is not None + + +@method(None, {"type": "string"}) +async def adopt(context): + global access_key + + if access_key is not None: + raise Error("System is already adopted") + + access_key = secrets.token_urlsafe(32) + + setattr(context.rpc_request.context["http_request"], "_authenticated", True) + + return access_key + + +@method({"type": "string"}, None) +async def authenticate(context, key): + global access_key + + if access_key is None: + raise Error("The system is not adopted") + + if key != access_key: + raise Error("Invalid access key", errno.EINVAL) + + setattr(context.rpc_request.context["http_request"], "_authenticated", True) + + +async def adoption_middleware(request: protocol.JsonRpcRequest, handler: typing.Callable) -> protocol.JsonRpcResponse: + if access_key is not None: + if not ( + request.method_name in ["is_adopted", "authenticate"] or + getattr(request.context["http_request"], "_authenticated", False) + ): + return protocol.JsonRpcResponse( + id=request.id, + jsonrpc=request.jsonrpc, + error=errors.InvalidParams("You must authenticate before making this call", data={"errno": errno.EACCES}), + ) + + return await handler(request) diff --git a/truenas_installer/server/api/info.py b/truenas_installer/server/api/info.py new file mode 100644 index 0000000..0bbe4c4 --- /dev/null +++ b/truenas_installer/server/api/info.py @@ -0,0 +1,54 @@ +from dataclasses import asdict + +from truenas_installer.disks import list_disks as _list_disks +from truenas_installer.network_interfaces import list_network_interfaces as _list_network_interfaces +from truenas_installer.lock import installation_lock +from truenas_installer.server.method import method + +__all__ = ["system_info", "list_disks", "list_network_interfaces"] + + +@method(None, { + "type": "object", + "properties": { + "running": {"type": "boolean"}, + "version": {"type": "string"}, + "efi": {"type": "boolean"}, + }, +}) +async def system_info(context): + return { + "running": installation_lock.locked(), + "version": context.installer.version, + "efi": context.installer.efi, + } + + +@method(None, { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "size": {"type": "number"}, + "model": {"type": "string"}, + "label": {"type": "string"}, + "removable": {"type": "boolean"}, + }, + }, +}) +async def list_disks(context): + return [asdict(disk) for disk in await _list_disks()] + + +@method(None, { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + }, + }, +}) +async def list_network_interfaces(context): + return [asdict(interface) for interface in await _list_network_interfaces()] diff --git a/truenas_installer/server/api/install.py b/truenas_installer/server/api/install.py new file mode 100644 index 0000000..962d750 --- /dev/null +++ b/truenas_installer/server/api/install.py @@ -0,0 +1,94 @@ +import asyncio +import errno +import functools + +from aiohttp_rpc.protocol import JsonRpcRequest + +from truenas_installer.exception import InstallError +from truenas_installer.install import install as install_ +from truenas_installer.serial import serial_sql +from truenas_installer.server.error import Error +from truenas_installer.server.method import method + +__all__ = ["install"] + + +@method({ + "type": "object", + "required": ["disks", "create_swap", "set_pmbr", "authentication"], + "additionalProperties": False, + "properties": { + "disks": { + "type": "array", + "items": {"type": "string"}, + }, + "create_swap": {"type": "boolean"}, + "set_pmbr": {"type": "boolean"}, + "authentication": { + "type": ["object", "null"], + "required": ["username", "password"], + "additionalProperties": False, + "properties": { + "username": { + "type": "string", + "enum": ["admin", "root"], + }, + "password": { + "type": "string", + "minLength": 6, + }, + }, + }, + "post_install": { + "type": "object", + "additionalProperties": False, + "properties": { + "network_interfaces": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": False, + "properties": { + "name": {"type": "string"}, + "aliases": { + "type": "array", + "items": { + "type": "object", + "required": ["type", "address", "netmask"], + "additionalProperties": False, + "properties": { + "type": {"type": "string"}, + "address": {"type": "string"}, + "netmask": {"type": "integer"}, + }, + }, + }, + "ipv4_dhcp": {"type": "boolean"}, + "ipv6_auto": {"type": "boolean"}, + }, + }, + }, + }, + }, + }, +}, None) +async def install(context, params): + try: + await install_(params["disks"], params["create_swap"], params["set_pmbr"], params["authentication"], + params.get("post_install", None), await serial_sql(), + functools.partial(callback, context.server)) + except InstallError as e: + raise Error(e.message, errno.EFAULT) + + +def callback(server, progress, message): + request = server.json_serialize( + JsonRpcRequest( + "installation_progress", + params=[{"progress": progress, "message": message}] + ).dump() + ) + + for other in server.rcp_websockets: + asyncio.ensure_future(other.send_str(request)) diff --git a/truenas_installer/server/api/power.py b/truenas_installer/server/api/power.py new file mode 100644 index 0000000..2c49d40 --- /dev/null +++ b/truenas_installer/server/api/power.py @@ -0,0 +1,17 @@ +import asyncio + +from truenas_installer.server.method import method + +__all__ = ["reboot", "shutdown"] + + +@method(None, None) +async def reboot(context): + process = await asyncio.create_subprocess_exec("reboot") + await process.communicate() + + +@method(None, None) +async def shutdown(context): + process = await asyncio.create_subprocess_exec("shutdown", "now") + await process.communicate() diff --git a/truenas_installer/server/error.py b/truenas_installer/server/error.py new file mode 100644 index 0000000..3461237 --- /dev/null +++ b/truenas_installer/server/error.py @@ -0,0 +1,36 @@ +import errno +import logging +import typing + +from aiohttp_rpc import errors, protocol + +logger = logging.getLogger(__name__) + +__all__ = ["Error", "exception_middleware"] + + +class Error(Exception): + def __init__(self, text, code=errno.EFAULT): + self.text = text + self.code = code + super().__init__(self.text, self.code) + + +async def exception_middleware(request: protocol.JsonRpcRequest, handler: typing.Callable) -> protocol.JsonRpcResponse: + try: + response = await handler(request) + except Error as e: + response = protocol.JsonRpcResponse( + id=request.id, + jsonrpc=request.jsonrpc, + error=errors.InvalidParams(e.text, data={"errno": errno.errorcode.get(e.code)}), + ) + except Exception: + logger.error("Unhandled exception", exc_info=True) + response = protocol.JsonRpcResponse( + id=request.id, + jsonrpc=request.jsonrpc, + error=errors.InternalError(), + ) + + return response diff --git a/truenas_installer/server/method.py b/truenas_installer/server/method.py new file mode 100644 index 0000000..464fcb9 --- /dev/null +++ b/truenas_installer/server/method.py @@ -0,0 +1,62 @@ +# -*- coding=utf-8 -*- +from dataclasses import dataclass +import errno +import logging + +from truenas_installer.server.error import Error + +from jsonschema import validate, ValidationError + +logger = logging.getLogger(__name__) + +__all__ = ["methods", "method"] + +methods = {} + + +@dataclass +class Context: + server: object + rpc_request: object + + +class Method: + def __init__(self, name, schema, result_schema, fn): + self.name = name + self.schema = schema + self.result_schema = result_schema + self.fn = fn + self.server = None + + async def call(self, rpc_request, *args): + args = (Context(self.server, rpc_request),) + + if self.schema is not None: + if len(rpc_request.args) != 1: + raise Error(f"1 parameter required, found {len(rpc_request.args)}", errno.EINVAL) + + param = rpc_request.args[0] + try: + validate(param, self.schema) + except ValidationError as e: + raise Error(str(e), errno.EINVAL) + + args += (param,) + else: + if len(rpc_request.args) != 0: + raise Error(f"0 parameters required, found {len(rpc_request.args)}", errno.EINVAL) + + return await self.fn(*args) + + +def method(schema, result_schema): + def wrapper(fn): + name = fn.__name__ + + if name in methods: + raise RuntimeError(f"Method {name!r} is already registered") + + methods[name] = Method(name, schema, result_schema, fn) + return fn + + return wrapper