{ config, lib, pkgs, inputs, ... }@args: with lib; let cfg = config.nyxs-nix.services.k3s; in { imports = [ ./secrets.nix ]; options.nyxs-nix.services.k3s = with types; { enable = mkEnableOption "Enable the vrrp quotes service"; initServer = mkOption { type = bool; default = false; description = "Whether or not the node is the main one or sub node"; }; }; config = mkIf cfg.enable { boot.kernelModules = [ "fuse" ]; programs.fuse.userAllowOther = true; environment.etc."cdi/fuse.json".text = builtins.toJSON { cdiVersion = "1.1.0"; kind = "cdi.nyxerinys.dev/fuse"; devices = [ { name = "rw"; containerEdits = { deviceNodes = [ { path = "/dev/fuse"; type = "c"; fileMode = 438; permissions = "rw"; } ]; }; } ]; }; networking.firewall.allowedTCPPorts = [ 6443 # k3s, api server 2379 # k3s, etcd clients 2380 # k3s, etcd peers 10250 # k3s, kubelet ]; networking.firewall.allowedUDPPorts = [ 8472 # k3s, flannel ]; networking.firewall.allowedTCPPortRanges = [ { from = 30000; to = 32767; } # k3s, nodeport ]; environment.systemPackages = with pkgs; [ nfs-utils openiscsi ]; sops.secrets = { k3s_token = { format = "json"; key = "token"; sopsFile = "${inputs.secrets}/cluster/k3s.json"; }; pg_pass = { format = "json"; key = "pg-pass"; sopsFile = "${inputs.secrets}/cluster/authentik.json"; }; authentik_secret_key = { format = "json"; key = "authentik-secret-key"; sopsFile = "${inputs.secrets}/cluster/authentik.json"; }; }; services.rpcbind.enable = true; services.k3s = { enable = true; token = config.sops.secrets.k3s_token.path; clusterInit = mkIf cfg.initServer true; serverAddr = mkIf (!cfg.initServer) "https://10.5.0.103:6443"; manifests = import ./manifests.nix { inherit lib cfg inputs; }; images = import ./images args; disable = []; }; systemd.services.k3s = { restartTriggers = [ (builtins.toJSON config.services.k3s.manifests) (builtins.toJSON config.services.k3s.images) ]; stopIfChanged = false; after = [ "rpcbind.service" "network-online.target" ]; wants = [ "rpcbind.service" ]; path = with pkgs; [ openiscsi ]; }; system.activationScripts.cleanK3sManifests = { text = '' MANIFESTS_DIR="/var/lib/rancher/k3s/server/manifests" NIX_MANIFESTS="${builtins.concatStringsSep " " (builtins.attrNames config.services.k3s.manifests)}" DEFAULT_MANIFESTS="ccm coredns local-storage rolebindings runtimes traefik" echo "---" echo "Cleaning up K3s manifests" echo for file in $MANIFESTS_DIR/*.yaml; do [ -f "$file" ] || continue name=$(basename "$file" .yaml) if ! (echo "$NIX_MANIFESTS $DEFAULT_MANIFESTS" | grep -qw "$name"); then echo "Stale manifest found, deleting $name.yaml" ${pkgs.k3s}/bin/k3s kubectl delete -f "$file" --ignore-not-found=true 2>/dev/null || true rm -f "$file" fi done echo "Done" echo ''; }; }; }