[docs] write guide and add examples

[j-brn]

No description provided.

[glanch]

Hey, do you know any NixOS configurations publicly hosted using the modules of nixos-vfio? This would be really helpful to bridge time until documentation is ready.

[j-brn]

Hey, do you know any NixOS configurations publicly hosted using the modules of nixos-vfio? This would be really helpful to bridge time until documentation is ready.

Sorry, I must have missed your comment. I'm not aware of any publicly accessible config, but I'm happy to provide my own

{ pkgs, ... }:
let
  hostCores = "0-7,16-23";
  totalCores = "0-31";
in
{
  environment.systemPackages = with pkgs; [ swtpm ];
  systemd.services.libvirtd.path = [ pkgs.swtpm ];

  virtualisation = {
    libvirtd = {
      onBoot = "ignore";
      onShutdown = "shutdown";

      deviceACL = [
        "/dev/vfio/vfio"
        "/dev/kvm"
        "/dev/kvmfr0"
        "/dev/null"
      ];

      qemu.networks.declarative = true;
      qemu.networks.networks = {
        default.config = {
          forward = { mode = "nat"; };

          ips = [
            {
              family = "ipv4";
              address = "192.168.100.1";
              prefix = 24;

              dhcpRanges = [{
                start = "192.168.100.128";
                end = "192.168.100.254";
              }];
            }
            {
              family = "ipv6";
              address = "2001:db8:ca2:2::1";
              prefix = 64;

              dhcpRanges = [{
                start = "2001:db8:ca2:2::100";
                end = "2001:db8:ca2:2::1ff";
              }];
            }
          ];
        };
        default.autstart = true;
      };

      qemu.domains.declarative = true;
      qemu.domains.domains = {
        win10.config = {
          memory = {
            memory = {
              value = 32;
              unit = "G";
            };

            disableBallooning = true;
            useHugepages = false;
          };

          vcpu = {
            count = 16;
            placement = "static";
          };

          cputune = {
            vcpupins = [
              { vcpu = 0; cpuset = [ 8 ]; }
              { vcpu = 1; cpuset = [ 24 ]; }
              { vcpu = 2; cpuset = [ 9 ]; }
              { vcpu = 3; cpuset = [ 25 ]; }
              { vcpu = 4; cpuset = [ 10 ]; }
              { vcpu = 5; cpuset = [ 26 ]; }
              { vcpu = 6; cpuset = [ 11 ]; }
              { vcpu = 7; cpuset = [ 27 ]; }
              { vcpu = 8; cpuset = [ 12 ]; }
              { vcpu = 9; cpuset = [ 28 ]; }
              { vcpu = 10; cpuset = [ 13 ]; }
              { vcpu = 11; cpuset = [ 29 ]; }
              { vcpu = 12; cpuset = [ 14 ]; }
              { vcpu = 13; cpuset = [ 30 ]; }
              { vcpu = 14; cpuset = [ 15 ]; }
              { vcpu = 15; cpuset = [ 31 ]; }
            ];
          };

          cpu = {
            mode = "host-passthrough";
            topology = {
              sockets = 1;
              dies = 1;
              cores = 8;
              threads = 2;
            };
          };

          input = {
            virtioMouse = true;
            virtioKeyboard = true;
          };

          spice = {
            spiceAudio = true;
            spicemvcChannel = true;
            spiceGraphics = true;
          };

          pciHostDevices = [
            # nvme
            {
              sourceAddress = {
                bus = "0x04";
                slot = "0x00";
                function = 0;
              };

              bootIndex = 1;
            }

            # Nvidia RTX3090
            {
              sourceAddress = {
                bus = "0x0b";
                slot = "0x00";
                function = 0;
              };
            }

            # Nvidia RTX3090 audio device
            {
              sourceAddress = {
                bus = "0x0b";
                slot = "0x00";
                function = 1;
              };
            }
          ];

          networkInterfaces = [{ sourceNetwork = "default"; }];

          kvmfr = {
            device = "/dev/kvmfr0";
            size = "33554432";
          };
        };
      };

      scopedHooks.qemu = {
        "10-activate-core-isolation" = {
          enable = true;
          scope = {
            objects = [ "win10" ];
            operations = [ "prepare" ];
          };
          script = ''
            systemctl set-property --runtime -- user.slice AllowedCPUs=${hostCores}
            systemctl set-property --runtime -- system.slice AllowedCPUs=${hostCores}
            systemctl set-property --runtime -- init.scope AllowedCPUs=${hostCores}
          '';
        };

        "10-deactivate-core-isolation" = {
          enable = true;
          scope = {
            objects = [ "win10" ];
            operations = [ "release" ];
          };
          script = ''
            systemctl set-property --runtime -- user.slice AllowedCPUs=${totalCores}
            systemctl set-property --runtime -- system.slice AllowedCPUs=${totalCores}
            systemctl set-property --runtime -- init.scope AllowedCPUs=${totalCores}
          '';
        };
      };
    };

    vfio = {
      enable = true;
      IOMMUType = "amd";
      devices = [
        "10de:2204"
        "10de:1aef"
      ];
      blacklistNvidia = true;
      disableEFIfb = true;
      ignoreMSRs = true;
    };

    kvmfr = {
      enable = true;

      devices = [
        {
          resolution = {
            width = 2560;
            height = 1440;
            pixelFormat = "rgba32";
          };

          permissions = {
            user = "jonas";
            group = "qemu-libvirtd";
            mode = "0660";
          };
        }
      ];
    };
  };
}

The work on the guide is currently paused because I want to make some bigger changes on the libvirtd domain/network modules first