{ config, lib, pkgs, inputs, ... }: with lib; let cfg = config.nyxs-nix.services.k3s; in { options.nyxs-nix.services.k3s = with types; { enable = mkEnableOption "Enable the vrrp quotes service"; initServer = mkOption { type = bool; default = false; description = "Whether or not the node is the main one or sub node"; }; }; config = mkIf cfg.enable { networking.firewall.allowedTCPPorts = [ 6443 # k3s: required so that pods can reach the API server (running on port 6443 by default) 2379 # k3s, etcd clients: required if using a "High Availability Embedded etcd" configuration 2380 # k3s, etcd peers: required if using a "High Availability Embedded etcd" configuration ]; networking.firewall.allowedUDPPorts = [ 8472 # k3s, flannel: required if using multi-node for inter-node networking ]; environment.systemPackages = with pkgs; [ nfs-utils openiscsi ]; sops.secrets.k3s_token = { format = "json"; key = "token"; sopsFile = "${inputs.secrets}/k3s.json"; }; services.rpcbind.enable = true; services.k3s = { enable = true; token = config.sops.secrets.k3s_token.path; clusterInit = mkIf cfg.initServer true; serverAddr = mkIf (!cfg.initServer) "https://10.5.0.103:6443"; manifests = import ./manifests.nix { inherit lib cfg inputs; }; disable = []; }; services.udev.packages = with pkgs; [ openiscsi ]; services.udev.rules = '' ACTION=="add", SUBSYSTEM=="scsi_host", KERNEL=="host*", RUN+="${pkgs.bash}/bin/sh -c 'echo - - - > /sys/class/scsi_host/%k/scan'" ''; systemd.services.k3s = { restartTriggers = [ (builtins.attrNames config.services.k3s.manifests) ]; stopIfChanged = false; after = [ "rpcbind.service" "network-online.target" ]; wants = [ "rpcbind.service" ]; path = with pkgs; [ openiscsi util-linux kmod systemd udev ]; }; system.activationScripts.cleanK3sManifests = { text = '' MANIFESTS_DIR="/var/lib/rancher/k3s/server/manifests" NIX_MANIFESTS="${builtins.concatStringsSep " " (builtins.attrNames config.services.k3s.manifests)}" DEFAULT_MANIFESTS="ccm coredns local-storage rolebindings runtimes traefik" echo "---" echo "Cleaning up K3s manifests" echo for file in $MANIFESTS_DIR/*.yaml; do [ -f "$file" ] || continue name=$(basename "$file" .yaml) if ! (echo "$NIX_MANIFESTS $DEFAULT_MANIFESTS" | grep -qw "$name"); then echo "Stale manifest found, deleting $name.yaml" ${pkgs.k3s}/bin/k3s kubectl delete -f "$file" --ignore-not-found=true 2>/dev/null || true rm -f "$file" fi done echo "Done" echo ''; }; }; }