try using a device plugin

modules/k3s/daemon-sets/default.nix

@@ -0,0 +1,8 @@
+{ cfg, lib, inputs, ... }:
+let
+  manifests = [
+    ./fuse-devPlug.nix
+  ];
+  imported = map ( n: import n { inherit cfg lib inputs; } ) manifests;
+in
+  lib.foldl' lib.recursiveUpdate {} imported

modules/k3s/daemon-sets/fuse-devPlug.nix

@@ -0,0 +1,105 @@
+{ ... }:
+{
+  fuse-devPlug.contents = [
+    {
+      apiVersion = "apps/v1";
+      kind = "DaemonSet";
+      metadata = {
+        name = "generic-device-plugin";
+        namespace = "kube-system";
+        labels = {
+          "app.kubernetes.io/name" = "generic-device-plugin";
+        };
+      };
+      spec = {
+        selector = {
+          matchLabels = {
+            "app.kubernetes.io/name" = "generic-device-plugin";
+          };
+        };
+        template = {
+          metadata = {
+            labels = {
+              "app.kubernetes.io/name" = "generic-device-plugin";
+            };
+          };
+          spec = {
+            priorityClassName = "system-node-critical";
+            tolerations = [
+              {
+                operator = "Exists";
+                effect = "NoExecute";
+              }
+              {
+                operator = "Exists";
+                effect = "NoSchedule";
+              }
+            ];
+            containers = [
+              {
+                image = "squat/generic-device-plugin";
+                args = [
+                  "--device"
+                  ''
+                    name: fuse
+                    groups:
+                      - count: 10
+                        paths:
+                          - path: /dev/fuse
+                  ''
+                ];
+                name = "generic-device-plugin";
+                resources = {
+                  requests = {
+                    cpu = "50m";
+                    memory = "10Mi";
+                  };
+                  limits = {
+                    cpu = "50m";
+                    memory = "20Mi";
+                  };
+                };
+                ports = [
+                  {
+                    containerPort = 8080;
+                    name = "http";
+                  }
+                ];
+                securityContext = {
+                  privileged = true;
+                };
+                volumeMounts = [
+                  {
+                    name = "device-plugin";
+                    mountPath = "/var/lib/kubelet/device-plugins";
+                  }
+                  {
+                    name = "dev";
+                    mountPath = "/dev";
+                  }
+                ];
+              }
+            ];
+            volumes = [
+              {
+                name = "device-plugin";
+                hostPath = {
+                  path = "/var/lib/kubelet/device-plugins";
+                };
+              }
+              {
+                name = "dev";
+                hostPath = {
+                  path = "/dev";
+                };
+              }
+            ];
+          };
+        };
+        updateStrategy = {
+          type = "RollingUpdate";
+        };
+      };
+    }
+  ];
+}

modules/k3s/deployments/storage-combo.nix

@@ -15,6 +15,7 @@
             labels.app = "storage-combo";
           };
           spec = {
+            hostUsers = false;
             containers = [
               {
                 name = "storage-combo";
@@ -26,12 +27,13 @@
                     containerPort = 2222; 
                   }
                 ];
-                volumeMounts = [
+                resources.limits."devic.es/fuse" = 1;
-                  {
+                # volumeMounts = [
-                    name = "fuse-device";
+                #   {
-                    mountPath = "/dev/fuse";
+                #     name = "fuse-device";
-                  }
+                #     mountPath = "/dev/fuse";
-                ];
+                #   }
+                # ];
                 securityContext = {
                   allowPrivilegeEscalation = true;
                   capabilities.add = ["SYS_ADMIN"];
@@ -42,15 +44,15 @@
                 };
               }
             ];
-            volumes = [
+            # volumes = [
-              {
+            #   {
-                name = "fuse-device";
+            #     name = "fuse-device";
-                hostPath = {
+            #     hostPath = {
-                  path = "/dev/fuse";
+            #       path = "/dev/fuse";
-                  type = "CharDevice";
+            #       type = "CharDevice";
-                };
+            #     };
-              }
+            #   }
-            ];
+            # ];
           };
         };
       };

modules/k3s/manifests.nix

@@ -2,7 +2,7 @@
 let
   manifests = [
     ./deployments
-    # ./services
+    ./daemon-sets
   ];
   imported = map ( n: import n { inherit cfg lib inputs; } ) manifests;
 in