diff --git a/modules/k3s/daemon-sets/default.nix b/modules/k3s/daemon-sets/default.nix new file mode 100644 index 0000000..d681a4d --- /dev/null +++ b/modules/k3s/daemon-sets/default.nix @@ -0,0 +1,8 @@ +{ cfg, lib, inputs, ... }: +let + manifests = [ + ./fuse-devPlug.nix + ]; + imported = map ( n: import n { inherit cfg lib inputs; } ) manifests; +in + lib.foldl' lib.recursiveUpdate {} imported \ No newline at end of file diff --git a/modules/k3s/daemon-sets/fuse-devPlug.nix b/modules/k3s/daemon-sets/fuse-devPlug.nix new file mode 100644 index 0000000..aa79afb --- /dev/null +++ b/modules/k3s/daemon-sets/fuse-devPlug.nix @@ -0,0 +1,105 @@ +{ ... }: +{ + fuse-devPlug.contents = [ + { + apiVersion = "apps/v1"; + kind = "DaemonSet"; + metadata = { + name = "generic-device-plugin"; + namespace = "kube-system"; + labels = { + "app.kubernetes.io/name" = "generic-device-plugin"; + }; + }; + spec = { + selector = { + matchLabels = { + "app.kubernetes.io/name" = "generic-device-plugin"; + }; + }; + template = { + metadata = { + labels = { + "app.kubernetes.io/name" = "generic-device-plugin"; + }; + }; + spec = { + priorityClassName = "system-node-critical"; + tolerations = [ + { + operator = "Exists"; + effect = "NoExecute"; + } + { + operator = "Exists"; + effect = "NoSchedule"; + } + ]; + containers = [ + { + image = "squat/generic-device-plugin"; + args = [ + "--device" + '' + name: fuse + groups: + - count: 10 + paths: + - path: /dev/fuse + '' + ]; + name = "generic-device-plugin"; + resources = { + requests = { + cpu = "50m"; + memory = "10Mi"; + }; + limits = { + cpu = "50m"; + memory = "20Mi"; + }; + }; + ports = [ + { + containerPort = 8080; + name = "http"; + } + ]; + securityContext = { + privileged = true; + }; + volumeMounts = [ + { + name = "device-plugin"; + mountPath = "/var/lib/kubelet/device-plugins"; + } + { + name = "dev"; + mountPath = "/dev"; + } + ]; + } + ]; + volumes = [ + { + name = "device-plugin"; + hostPath = { + path = "/var/lib/kubelet/device-plugins"; + }; + } + { + name = "dev"; + hostPath = { + path = "/dev"; + }; + } + ]; + }; + }; + updateStrategy = { + type = "RollingUpdate"; + }; + }; + } + ]; +} \ No newline at end of file diff --git a/modules/k3s/deployments/storage-combo.nix b/modules/k3s/deployments/storage-combo.nix index 360fbf2..a05b414 100644 --- a/modules/k3s/deployments/storage-combo.nix +++ b/modules/k3s/deployments/storage-combo.nix @@ -15,6 +15,7 @@ labels.app = "storage-combo"; }; spec = { + hostUsers = false; containers = [ { name = "storage-combo"; @@ -26,12 +27,13 @@ containerPort = 2222; } ]; - volumeMounts = [ - { - name = "fuse-device"; - mountPath = "/dev/fuse"; - } - ]; + resources.limits."devic.es/fuse" = 1; + # volumeMounts = [ + # { + # name = "fuse-device"; + # mountPath = "/dev/fuse"; + # } + # ]; securityContext = { allowPrivilegeEscalation = true; capabilities.add = ["SYS_ADMIN"]; @@ -42,15 +44,15 @@ }; } ]; - volumes = [ - { - name = "fuse-device"; - hostPath = { - path = "/dev/fuse"; - type = "CharDevice"; - }; - } - ]; + # volumes = [ + # { + # name = "fuse-device"; + # hostPath = { + # path = "/dev/fuse"; + # type = "CharDevice"; + # }; + # } + # ]; }; }; }; diff --git a/modules/k3s/manifests.nix b/modules/k3s/manifests.nix index f18c7cd..df026e8 100644 --- a/modules/k3s/manifests.nix +++ b/modules/k3s/manifests.nix @@ -2,7 +2,7 @@ let manifests = [ ./deployments - # ./services + ./daemon-sets ]; imported = map ( n: import n { inherit cfg lib inputs; } ) manifests; in