Add the vendor VFIO vGPU device backend

integration/vgpu_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -21,21 +23,23 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestVGPU is an integration test that verifies vGPU (SR-IOV mdev) support works.
+// TestVGPU is an integration test that verifies vGPU (SR-IOV) support works
+// on the host's framework: mdev or NVIDIA's vendor-specific VFIO.
 //
 // This test automatically detects vGPU availability and skips if:
-//   - No SR-IOV VFs are found in /sys/class/mdev_bus/
+//   - No vGPU framework (mdev or vendor VFIO) is discovered
 //   - No vGPU profiles are available
-//   - Not running as root (required for mdev creation)
+//   - Not running as root (required for sysfs vGPU assignment)
 //   - KVM is not available
 //
 // To run manually:
 //
 //	sudo go test -v -run TestVGPU -timeout 5m ./integration/...
 //
-// Note: This test verifies mdev creation and PCI device visibility inside the VM.
-// It does NOT test nvidia-smi or CUDA functionality since that requires NVIDIA
-// guest drivers pre-installed in the image.
+// Note: This test verifies vGPU assignment, release on stop, reacquisition on
+// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi
+// or CUDA functionality since that requires NVIDIA guest drivers pre-installed
+// in the image.
 func TestVGPU(t *testing.T) {
 	t.Parallel()
 	if testing.Short() {
@@ -84,9 +88,15 @@ func TestVGPU(t *testing.T) {
 
 	// Cleanup any orphaned instances and mdevs
 	t.Cleanup(func() {
-		if instanceID != "" {
-			t.Log("Cleanup: Deleting instance...")
-			instanceManager.DeleteInstance(ctx, instanceID)
+		if instanceID == "" {
+			return
+		}
+		if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil {
+			t.Logf("Cleanup: stop instance: %v", err)
+		}
+		t.Log("Cleanup: Deleting instance...")
+		if err := instanceManager.DeleteInstance(ctx, instanceID); err != nil {
+			t.Errorf("cleanup: delete instance: %v", err)
 		}
 	})
 
@@ -159,9 +169,18 @@ func TestVGPU(t *testing.T) {
 	instanceID = inst.Id
 	t.Logf("Instance created: %s", inst.Id)
 
-	// Verify mdev UUID was assigned
-	require.NotEmpty(t, inst.GPUMdevUUID, "Instance should have mdev UUID assigned")
-	t.Logf("mdev UUID: %s", inst.GPUMdevUUID)
+	// Verify the assignment matches the host's framework
+	require.NotEmpty(t, inst.GPUDevicePath, "Instance should have a vGPU device path assigned")
+	switch inst.GPUFramework {
+	case devices.VGPUFrameworkMdev:
+		require.NotEmpty(t, inst.GPUMdevUUID, "mdev instance should have a UUID assigned")
+		t.Logf("mdev UUID: %s", inst.GPUMdevUUID)
+	case devices.VGPUFrameworkVendorVFIO:
+		require.Empty(t, inst.GPUMdevUUID, "vendor VFIO instance should not have an mdev UUID")
+		t.Logf("vendor VFIO VF: %s", inst.GPUDevicePath)
+	default:
+		t.Fatalf("unexpected vGPU framework %q", inst.GPUFramework)
+	}
 
 	// Step 5: Check GPU resources AFTER creating instance
 	t.Run("ResourcesDecrementedAfterCreation", func(t *testing.T) {
@@ -180,12 +199,9 @@ func TestVGPU(t *testing.T) {
 		assert.Less(t, availableAfter, availableBefore, "available instances should decrease after creating VM")
 	})
 
-	// Step 6: Verify mdev was created in sysfs
-	t.Run("MdevCreated", func(t *testing.T) {
-		mdevPath := "/sys/bus/mdev/devices/" + inst.GPUMdevUUID
-		_, err := os.Stat(mdevPath)
-		assert.NoError(t, err, "mdev device should exist at %s", mdevPath)
-		t.Logf("mdev exists at: %s", mdevPath)
+	// Step 6: Verify the assignment exists in sysfs
+	t.Run("VGPUAssignedInSysfs", func(t *testing.T) {
+		assertVGPUAssigned(t, inst.GPUFramework, inst.GPUDevicePath)
 	})
 
 	// Step 7: Wait for guest agent to be ready
@@ -225,13 +241,68 @@ func TestVGPU(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, profile, actualInst.GPUProfile, "GPU profile should match")
-		assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set")
-		t.Logf("Instance GPU: profile=%s, mdev=%s", actualInst.GPUProfile, actualInst.GPUMdevUUID)
+		assert.Equal(t, inst.GPUFramework, actualInst.GPUFramework, "framework should match")
+		assert.NotEmpty(t, actualInst.GPUDevicePath, "device path should be set")
+		if inst.GPUFramework == devices.VGPUFrameworkMdev {
+			assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set")
+		}
+		t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath)
+	})
+
+	t.Log("Step 10: Stopping instance...")
+	_, err = instanceManager.StopInstance(ctx, inst.Id)
+	require.NoError(t, err, "stop should succeed")
+
+	t.Run("VGPUReleasedOnStop", func(t *testing.T) {
+		stopped, err := instanceManager.GetInstance(ctx, inst.Id)
+		require.NoError(t, err)
+		assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop")
+		assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath)
+	})
+
+	t.Log("Step 11: Starting instance to reacquire a vGPU...")
+	started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{})
+	require.NoError(t, err, "start should succeed")
+
+	t.Run("VGPUReacquiredOnStart", func(t *testing.T) {
+		require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU")
+		assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match")
+		assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath)
 	})
 
 	t.Log("✅ vGPU test PASSED!")
 }
 
+func assertVGPUAssigned(t *testing.T, framework devices.VGPUFramework, devicePath string) {
+	t.Helper()
+	switch framework {
+	case devices.VGPUFrameworkMdev:
+		_, err := os.Stat(devicePath)
+		assert.NoError(t, err, "mdev device should exist at %s", devicePath)
+	case devices.VGPUFrameworkVendorVFIO:
+		data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type"))
+		require.NoError(t, err, "VF should expose current_vgpu_type")
+		assert.NotEqual(t, "0", strings.TrimSpace(string(data)), "VF should have a vGPU type assigned")
+	default:
+		t.Fatalf("unexpected vGPU framework %q", framework)
+	}
+}
+
+func assertVGPUReleased(t *testing.T, framework devices.VGPUFramework, devicePath string) {
+	t.Helper()
+	switch framework {
+	case devices.VGPUFrameworkMdev:
+		_, err := os.Stat(devicePath)
+		assert.True(t, os.IsNotExist(err), "mdev device should be gone from %s", devicePath)
+	case devices.VGPUFrameworkVendorVFIO:
+		data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type"))
+		require.NoError(t, err, "VF should expose current_vgpu_type")
+		assert.Equal(t, "0", strings.TrimSpace(string(data)), "VF assignment should be released")
+	default:
+		t.Fatalf("unexpected vGPU framework %q", framework)
+	}
+}
+
 // checkVGPUTestPrerequisites checks if vGPU test can run.
 // Returns (skipReason, profileName) - skipReason is empty if all prerequisites are met.
 func checkVGPUTestPrerequisites() (string, string) {
@@ -245,10 +316,18 @@ func checkVGPUTestPrerequisites() (string, string) {
 		return "vGPU test requires root (sudo) for mdev creation", ""
 	}
 
-	// Check for vGPU mode (SR-IOV VFs present)
-	mode := devices.DetectHostGPUMode()
-	if mode != devices.GPUModeVGPU {
-		return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", ""
+	// Check for a vGPU framework (mdev or vendor VFIO)
+	framework, _, err := devices.DiscoverVGPU()
+	if err != nil {
+		return "vGPU test failed to discover vGPU framework: " + err.Error(), ""
+	}
+	if framework == devices.VGPUFrameworkNone {
+		return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
+	}
+	if framework == devices.VGPUFrameworkVendorVFIO {
+		// CreateVGPU rejects vendor VFIO until the instance lifecycle
+		// integration lands.
+		return "vGPU test requires the vendor VFIO instance lifecycle integration", ""
 	}
 
 	// Check for available profiles

lib/devices/GPU.md

@@ -8,16 +8,17 @@ hypeman supports two GPU modes, automatically detected based on host configurati
 
 | Mode | Description | Use Case |
 |------|-------------|----------|
-| **vGPU (SR-IOV)** | Virtual GPUs via mdev on SR-IOV VFs | Multi-tenant, shared GPU resources |
+| **vGPU (SR-IOV)** | Virtual GPUs on SR-IOV VFs via mdev or vendor VFIO | Multi-tenant, shared GPU resources |
 | **Passthrough** | Whole GPU VFIO passthrough | Dedicated GPU per instance |
 
 The host's GPU mode is determined by the host driver configuration:
-- If `/sys/class/mdev_bus/` contains VFs → vGPU mode
-- If NVIDIA GPUs are available for VFIO → passthrough mode
+- If `/sys/class/mdev_bus/` contains VFs → mdev vGPU mode
+- If VFs expose `/sys/bus/pci/devices/<VF>/nvidia/current_vgpu_type` → vendor VFIO vGPU mode
+- If NVIDIA GPUs are available for whole-device VFIO → passthrough mode
 
 ## vGPU Mode (Recommended)
 
-vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs), each capable of hosting an mdev (mediated device) representing a vGPU.
+vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs). Hosts on older kernels represent each vGPU as an mdev. Hosts using NVIDIA's vendor VFIO framework assign the profile directly to the VF through `current_vgpu_type`.
 
 ### How It Works
 
@@ -74,7 +75,7 @@ curl -X POST http://localhost:4973/instances \
   }'
 ```
 
-The response includes the assigned mdev UUID:
+On an mdev host, the response also includes the assigned mdev UUID:
 
 ```json
 {
@@ -87,19 +88,23 @@ The response includes the assigned mdev UUID:
 }
 ```
 
-### Ephemeral mdev Lifecycle
+### Ephemeral vGPU Lifecycle
 
-mdev devices are **ephemeral**: created on instance start, destroyed on instance delete.
+vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts and writes the profile ID/`0` to `current_vgpu_type` on vendor VFIO hosts.
 
 ```
-Instance Create → Create mdev → Attach to VM → Instance Running
-Instance Delete → Stop VM → Destroy mdev → VF available again
+Instance Create → Assign profile to VF → Attach VF to VM → Instance Running
+Instance Stop/Delete → Release profile → VF available again
 ```
 
-This ensures:
-- **Security**: No VRAM data leakage between instances
-- **Clean state**: Fresh vGPU for each instance
-- **Automatic cleanup**: Orphaned mdevs cleaned up on server restart
+Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM.
+
+### Hypervisor Support
+
+Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**:
+
+- **QEMU**: fully supported and validated on both mdev and vendor VFIO hosts.
+- **Cloud Hypervisor**: vendor VFIO vGPUs are known broken upstream ([cloud-hypervisor#7572](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7572)) — the VM boots and the VF attaches, but VFIO region reads fail, the guest driver cannot initialize, and the vGPU is non-functional. Do not place vGPU instances on Cloud Hypervisor.
 
 ## Passthrough Mode
 
@@ -241,7 +246,8 @@ To upgrade the NVIDIA driver version:
 
 1. Check host GPU mode detection:
    ```bash
-   ls /sys/class/mdev_bus/  # Should show VFs for vGPU mode
+   ls /sys/class/mdev_bus/
+   find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type'
    ```
 
 2. Verify NVIDIA drivers are loaded on host:
@@ -265,17 +271,56 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles'
    curl http://localhost:4973/instances/<id>/logs?source=app
    ```
 
-### mdev creation fails
+### Guest driver init times out on one VF (vendor VFIO)
 
-1. Check if VFs are available:
-   ```bash
-   ls /sys/class/mdev_bus/
-   ```
+A VF can be wedged inside the NVIDIA stack while its sysfs interface stays
+healthy: assignment succeeds, the host plugin logs `display_init inst: 0
+successful`, but the guest driver loops on
 
-2. Verify mdev types:
-   ```bash
-   cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances
-   ```
+```
+NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)
+```
+
+(0x65 = timeout; the guest's init requests are never answered, and
+`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because
+placement is deterministic least-loaded, an idle host re-picks the same VF for
+every request, so one wedged VF presents as all vGPU instances failing while
+`/resources` reports full capacity.
+
+The wedge itself leaves no host-side log: no kernel error, no XID, no plugin
+crash. In the observed case it followed a period of heavy attach/teardown
+churn on the VF, including QEMU processes that exited within seconds of
+opening the VFIO device (failed start attempts that were then retried), so
+suspect any workload that repeatedly kills the VMM mid-device-init.
+
+Confirm by assigning the same profile on a different VF: if that guest
+initializes, the VF is wedged, not the driver stack. Remediate by cycling
+SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it
+requires no vGPU assignments on that GPU):
+
+```bash
+/usr/lib/nvidia/sriov-manage -d <parent-gpu-pci-addr>
+/usr/lib/nvidia/sriov-manage -e <parent-gpu-pci-addr>
+```
+
+Do not unbind/rebind the VF from the nvidia driver — it breaks the
+nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`)
+and the VF stops accepting assignments entirely until the SR-IOV cycle.
+Services holding the GPU (DCGM, persistenced) must be stopped for the cycle
+to obtain the unbind lock.
+
+### vGPU assignment fails
+
+Check the files for the framework detected on the host:
+
+```bash
+# mdev
+cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances
+
+# vendor VFIO
+cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types
+cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type
+```
 
 ## Performance Tuning
 

lib/devices/mdev_darwin.go

@@ -12,10 +12,9 @@ func SetGPUProfileCacheTTL(ttl string) {
 	// No-op on macOS
 }
 
-// DiscoverVFs returns an empty list on macOS.
-// SR-IOV Virtual Functions are not available on macOS.
-func DiscoverVFs() ([]VirtualFunction, error) {
-	return []VirtualFunction{}, nil
+// DiscoverVGPU reports no vGPU framework on macOS.
+func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) {
+	return VGPUFrameworkNone, nil, nil
 }
 
 // ListGPUProfiles returns an empty list on macOS.
@@ -24,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) {
 }
 
 // ListGPUProfilesWithVFs returns an empty list on macOS.
-func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) {
+func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) {
 	return []GPUProfile{}, nil
 }
 
@@ -59,6 +58,10 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error {
 	return nil
 }
 
+func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error {
+	return nil
+}
+
 // ReconcileMdevs is a no-op on macOS.
 func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error {
 	return nil