From abd7e1fc152d16a53ed66a8892b4bfd85841c925 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron Date: Mon, 5 Sep 2022 12:38:13 +0200 Subject: [PATCH 1/2] services.nix-daemon: refactor into mkRemoteBuilderDesc I like to specify my builders on the go, in my shell. what I do is I export in my shell RUNNER1="xxxx" RUNNER2="yyyyy" which are generated from "mkRemoteBuilderDesc" that I've copy/pasted in my flake. I would like to expose mkRemoteBuilderDesc somehow but not sure how to do so. wip test Update nixos/modules/services/misc/nix-daemon.nix Co-authored-by: Robert Hensing Update nixos/modules/services/misc/nix-daemon.nix Co-authored-by: Robert Hensing --- nixos/modules/services/misc/nix-daemon.nix | 151 ++---------------- .../modules/services/misc/remote-builder.nix | 133 +++++++++++++++ 2 files changed, 146 insertions(+), 138 deletions(-) create mode 100644 nixos/modules/services/misc/remote-builder.nix diff --git a/nixos/modules/services/misc/nix-daemon.nix b/nixos/modules/services/misc/nix-daemon.nix index f37d197f1621d..b23516e8bffa2 100644 --- a/nixos/modules/services/misc/nix-daemon.nix +++ b/nixos/modules/services/misc/nix-daemon.nix @@ -10,6 +10,12 @@ let isNixAtLeast = versionAtLeast (getVersion nixPackage); + remoteBuilder = import ./remote-builder.nix; + + renderRemoteBuilder = machine: (lib.evalModules { + modules = [ ../modules/services/misc/remote-builder.nix machine ] ; + }).config.rendered; + makeNixBuildUser = nr: { name = "nixbld${toString nr}"; value = { @@ -219,126 +225,9 @@ in }; buildMachines = mkOption { - type = types.listOf (types.submodule { - options = { - hostName = mkOption { - type = types.str; - example = "nixbuilder.example.org"; - description = lib.mdDoc '' - The hostname of the build machine. - ''; - }; - protocol = mkOption { - type = types.enum [ null "ssh" "ssh-ng" ]; - default = "ssh"; - example = "ssh-ng"; - description = lib.mdDoc '' - The protocol used for communicating with the build machine. - Use `ssh-ng` if your remote builder and your - local Nix version support that improved protocol. - - Use `null` when trying to change the special localhost builder - without a protocol which is for example used by hydra. - ''; - }; - system = mkOption { - type = types.nullOr types.str; - default = null; - example = "x86_64-linux"; - description = lib.mdDoc '' - The system type the build machine can execute derivations on. - Either this attribute or {var}`systems` must be - present, where {var}`system` takes precedence if - both are set. - ''; - }; - systems = mkOption { - type = types.listOf types.str; - default = [ ]; - example = [ "x86_64-linux" "aarch64-linux" ]; - description = lib.mdDoc '' - The system types the build machine can execute derivations on. - Either this attribute or {var}`system` must be - present, where {var}`system` takes precedence if - both are set. - ''; - }; - sshUser = mkOption { - type = types.nullOr types.str; - default = null; - example = "builder"; - description = lib.mdDoc '' - The username to log in as on the remote host. This user must be - able to log in and run nix commands non-interactively. It must - also be privileged to build derivations, so must be included in - {option}`nix.settings.trusted-users`. - ''; - }; - sshKey = mkOption { - type = types.nullOr types.str; - default = null; - example = "/root/.ssh/id_buildhost_builduser"; - description = lib.mdDoc '' - The path to the SSH private key with which to authenticate on - the build machine. The private key must not have a passphrase. - If null, the building user (root on NixOS machines) must have an - appropriate ssh configuration to log in non-interactively. - - Note that for security reasons, this path must point to a file - in the local filesystem, *not* to the nix store. - ''; - }; - maxJobs = mkOption { - type = types.int; - default = 1; - description = lib.mdDoc '' - The number of concurrent jobs the build machine supports. The - build machine will enforce its own limits, but this allows hydra - to schedule better since there is no work-stealing between build - machines. - ''; - }; - speedFactor = mkOption { - type = types.int; - default = 1; - description = lib.mdDoc '' - The relative speed of this builder. This is an arbitrary integer - that indicates the speed of this builder, relative to other - builders. Higher is faster. - ''; - }; - mandatoryFeatures = mkOption { - type = types.listOf types.str; - default = [ ]; - example = [ "big-parallel" ]; - description = lib.mdDoc '' - A list of features mandatory for this builder. The builder will - be ignored for derivations that don't require all features in - this list. All mandatory features are automatically included in - {var}`supportedFeatures`. - ''; - }; - supportedFeatures = mkOption { - type = types.listOf types.str; - default = [ ]; - example = [ "kvm" "big-parallel" ]; - description = lib.mdDoc '' - A list of features supported by this builder. The builder will - be ignored for derivations that require features not in this - list. - ''; - }; - publicHostKey = mkOption { - type = types.nullOr types.str; - default = null; - description = lib.mdDoc '' - The (base64-encoded) public host key of this builder. The field - is calculated via {command}`base64 -w0 /etc/ssh/ssh_host_type_key.pub`. - If null, SSH will use its regular known-hosts file when connecting. - ''; - }; - }; - }); + # + # remoteBuilder.machineSubmodule; + type = types.listOf (types.submodule ./remote-builder.nix); default = [ ]; description = lib.mdDoc '' This option lists the machines to be used if distributed builds are @@ -677,24 +566,10 @@ in # List of machines for distributed Nix builds in the format # expected by build-remote.pl. environment.etc."nix/machines" = mkIf (cfg.buildMachines != [ ]) { - text = - concatMapStrings - (machine: - (concatStringsSep " " ([ - "${optionalString (machine.protocol != null) "${machine.protocol}://"}${optionalString (machine.sshUser != null) "${machine.sshUser}@"}${machine.hostName}" - (if machine.system != null then machine.system else if machine.systems != [ ] then concatStringsSep "," machine.systems else "-") - (if machine.sshKey != null then machine.sshKey else "-") - (toString machine.maxJobs) - (toString machine.speedFactor) - (let res = (machine.supportedFeatures ++ machine.mandatoryFeatures); - in if (res == []) then "-" else (concatStringsSep "," res)) - (let res = machine.mandatoryFeatures; - in if (res == []) then "-" else (concatStringsSep "," machine.mandatoryFeatures)) - ] - ++ optional (isNixAtLeast "2.4pre") (if machine.publicHostKey != null then machine.publicHostKey else "-"))) - + "\n" - ) - cfg.buildMachines; + text = map + # TODO eval here + concatMapStringsSep "\n" (bm: bm.rendered) cfg.buildMachines + ; }; assertions = diff --git a/nixos/modules/services/misc/remote-builder.nix b/nixos/modules/services/misc/remote-builder.nix new file mode 100644 index 0000000000000..f4c2675726cf5 --- /dev/null +++ b/nixos/modules/services/misc/remote-builder.nix @@ -0,0 +1,133 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + mkRemoteBuilderDesc = config: + concatStringsSep " " ([ + "${optionalString (machine.sshUser != null) "${machine.sshUser}@"}${machine.hostName}" + (if machine.system != null then machine.system else if machine.systems != [ ] then concatStringsSep "," machine.systems else "-") + (if machine.sshKey != null then machine.sshKey else "-") + (toString machine.maxJobs) + (toString machine.speedFactor) + (concatStringsSep "," (machine.supportedFeatures ++ machine.mandatoryFeatures)) + (concatStringsSep "," machine.mandatoryFeatures) + ] + ++ optional (isNixAtLeast "2.4pre") (if machine.publicHostKey != null then machine.publicHostKey else "-")); + + # TODO rename into module one + machineSubmodule = { + options = { + hostName = mkOption { + type = types.str; + example = "nixbuilder.example.org"; + description = lib.mdDoc '' + The hostname of the build machine. + ''; + }; + system = mkOption { + type = types.nullOr types.str; + default = null; + example = "x86_64-linux"; + description = lib.mdDoc '' + The system type the build machine can execute derivations on. + Either this attribute or {var}`systems` must be + present, where {var}`system` takes precedence if + both are set. + ''; + }; + systems = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "x86_64-linux" "aarch64-linux" ]; + description = lib.mdDoc '' + The system types the build machine can execute derivations on. + Either this attribute or {var}`system` must be + present, where {var}`system` takes precedence if + both are set. + ''; + }; + sshUser = mkOption { + type = types.nullOr types.str; + default = null; + example = "builder"; + description = lib.mdDoc '' + The username to log in as on the remote host. This user must be + able to log in and run nix commands non-interactively. It must + also be privileged to build derivations, so must be included in + {option}`nix.settings.trusted-users`. + ''; + }; + sshKey = mkOption { + type = types.nullOr types.str; + default = null; + example = "/root/.ssh/id_buildhost_builduser"; + description = lib.mdDoc '' + The path to the SSH private key with which to authenticate on + the build machine. The private key must not have a passphrase. + If null, the building user (root on NixOS machines) must have an + appropriate ssh configuration to log in non-interactively. + + Note that for security reasons, this path must point to a file + in the local filesystem, *not* to the nix store. + ''; + }; + maxJobs = mkOption { + type = types.int; + default = 1; + description = lib.mdDoc '' + The number of concurrent jobs the build machine supports. The + build machine will enforce its own limits, but this allows hydra + to schedule better since there is no work-stealing between build + machines. + ''; + }; + speedFactor = mkOption { + type = types.int; + default = 1; + description = lib.mdDoc '' + The relative speed of this builder. This is an arbitrary integer + that indicates the speed of this builder, relative to other + builders. Higher is faster. + ''; + }; + mandatoryFeatures = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "big-parallel" ]; + description = lib.mdDoc '' + A list of features mandatory for this builder. The builder will + be ignored for derivations that don't require all features in + this list. All mandatory features are automatically included in + {var}`supportedFeatures`. + ''; + }; + supportedFeatures = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "kvm" "big-parallel" ]; + description = lib.mdDoc '' + A list of features supported by this builder. The builder will + be ignored for derivations that require features not in this + list. + ''; + }; + publicHostKey = mkOption { + type = types.nullOr types.str; + default = null; + description = lib.mdDoc '' + The (base64-encoded) public host key of this builder. The field + is calculated via {command}`base64 -w0 /etc/ssh/ssh_host_type_key.pub`. + If null, SSH will use its regular known-hosts file when connecting. + ''; + }; + rendered = mkOption { + internal = true; + type = types.str; + apply = mkRemoteBuilderDesc config; + }; + }; + }; + in + machineSubmodule From 8dd0c23f8248dfd3b76a6e1e3694b4c42d1cb8b6 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron Date: Sun, 19 Mar 2023 01:41:03 +0100 Subject: [PATCH 2/2] nix-build lib/tests/nix-daemon.nix --show-trace fails : mkRemoteBuilderDesc can't get submodule values --- lib/tests/nix-daemon.nix | 45 +++++++++++++++++++ nixos/modules/services/misc/nix-daemon.nix | 10 ++--- .../modules/services/misc/remote-builder.nix | 34 ++++++++------ nixos/tests/all-tests.nix | 1 + 4 files changed, 72 insertions(+), 18 deletions(-) create mode 100644 lib/tests/nix-daemon.nix diff --git a/lib/tests/nix-daemon.nix b/lib/tests/nix-daemon.nix new file mode 100644 index 0000000000000..2515558952194 --- /dev/null +++ b/lib/tests/nix-daemon.nix @@ -0,0 +1,45 @@ +# to run these tests: +# nix-build lib/tests/nix-daemon.nix +# If it builds, all tests passed +{ pkgs ? import ../.. {}, lib ? pkgs.lib }: + +let + + buildMachine1 = { + hostName = "localhost"; + # todo move it to secrets + # sshUser = "notroot"; + sshKey = "/home/groot/.ssh/id_rsa"; + system = "x86_64-linux"; + maxJobs = 2; + speedFactor = 2; + supportedFeatures = [ "big-parallel" "kvm" ]; + # mandatoryFeatures = [ "perf" ]; + }; + + nixConfModule = { config, ... }: { + + buildMachines = buildMachine1; + + }; + + finalConfig = let + checkedAttrs = (lib.modules.evalModules { + modules = [ + nixConfModule + ({config,...}@args: { + options = { + buildMachines = lib.mkOption { + + description = lib.mdDoc ''PlaceHolder''; + type = lib.types.submodule (import ../../nixos/modules/services/misc/remote-builder.nix (args // { isNixAtLeastPre24 = true; })); + }; + }; + }) + ]; + }).config; + in checkedAttrs; +in + pkgs.writeTextDir "nix-config" + finalConfig.buildMachines.rendered + diff --git a/nixos/modules/services/misc/nix-daemon.nix b/nixos/modules/services/misc/nix-daemon.nix index b23516e8bffa2..7cc94b5645204 100644 --- a/nixos/modules/services/misc/nix-daemon.nix +++ b/nixos/modules/services/misc/nix-daemon.nix @@ -10,11 +10,11 @@ let isNixAtLeast = versionAtLeast (getVersion nixPackage); - remoteBuilder = import ./remote-builder.nix; + # remoteBuilder = import ./remote-builder.nix; - renderRemoteBuilder = machine: (lib.evalModules { - modules = [ ../modules/services/misc/remote-builder.nix machine ] ; - }).config.rendered; + # renderRemoteBuilder = machine: (lib.evalModules { + # modules = [ ../modules/services/misc/remote-builder.nix machine ] ; + # }).config.rendered; makeNixBuildUser = nr: { name = "nixbld${toString nr}"; @@ -227,7 +227,7 @@ in buildMachines = mkOption { # # remoteBuilder.machineSubmodule; - type = types.listOf (types.submodule ./remote-builder.nix); + type = types.listOf (types.submodule ./remote-builder.nix { isNixAtLeastPre24 = (isNixAtLeast "2.4pre"); }); default = [ ]; description = lib.mdDoc '' This option lists the machines to be used if distributed builds are diff --git a/nixos/modules/services/misc/remote-builder.nix b/nixos/modules/services/misc/remote-builder.nix index f4c2675726cf5..14fa23d8179b4 100644 --- a/nixos/modules/services/misc/remote-builder.nix +++ b/nixos/modules/services/misc/remote-builder.nix @@ -1,20 +1,20 @@ -{ config, lib, pkgs, ... }: +{ config, lib, isNixAtLeastPre24, ... }: with lib; let - - mkRemoteBuilderDesc = config: - concatStringsSep " " ([ - "${optionalString (machine.sshUser != null) "${machine.sshUser}@"}${machine.hostName}" - (if machine.system != null then machine.system else if machine.systems != [ ] then concatStringsSep "," machine.systems else "-") - (if machine.sshKey != null then machine.sshKey else "-") - (toString machine.maxJobs) - (toString machine.speedFactor) - (concatStringsSep "," (machine.supportedFeatures ++ machine.mandatoryFeatures)) - (concatStringsSep "," machine.mandatoryFeatures) + mkRemoteBuilderDesc = + # lib.traceSeq (machine) + (concatStringsSep " " ([ + "${optionalString (config.sshUser != null) "${config.sshUser}@"}${config.hostName}" + (if config.system != null then config.system else if config.systems != [ ] then concatStringsSep "," config.systems else "-") + (if config.sshKey != null then config.sshKey else "-") + (toString config.maxJobs) + (toString config.speedFactor) + (concatStringsSep "," (config.supportedFeatures ++ config.mandatoryFeatures)) + (concatStringsSep "," config.mandatoryFeatures) ] - ++ optional (isNixAtLeast "2.4pre") (if machine.publicHostKey != null then machine.publicHostKey else "-")); + ++ optional isNixAtLeastPre24 (if config.publicHostKey != null then config.publicHostKey else "-"))); # TODO rename into module one machineSubmodule = { @@ -124,10 +124,18 @@ let }; rendered = mkOption { internal = true; + readOnly = true; type = types.str; - apply = mkRemoteBuilderDesc config; + # apply = + # x: "toto"; + # mkRemoteBuilderDesc config; }; }; + + config = { + rendered = mkRemoteBuilderDesc config.config; + + }; }; in machineSubmodule diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 8386219f73d83..060f0ec75afe1 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -476,6 +476,7 @@ in { nginx-variants = handleTest ./nginx-variants.nix {}; nifi = handleTestOn ["x86_64-linux"] ./web-apps/nifi.nix {}; nitter = handleTest ./nitter.nix {}; + nix-daemon = handleTest ./nix-daemon.nix {}; nix-ld = handleTest ./nix-ld.nix {}; nix-serve = handleTest ./nix-serve.nix {}; nix-serve-ssh = handleTest ./nix-serve-ssh.nix {};