From cb3e330648a802f2e246a27aa8b1a7db13d112b2 Mon Sep 17 00:00:00 2001 From: Karen Chen Date: Thu, 23 Jul 2026 13:30:34 -0700 Subject: [PATCH 1/4] test(e2e): add multi-GPU MIG profile coverage --- e2e/scenario_gpu_managed_experience_test.go | 59 +++++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index 23305e15ddf..e2bd5ecfd0e 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -574,10 +574,10 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG(t *testing.T) { ValidateNvidiaDevicePluginServiceRunning(ctx, s) // Validate that MIG mode is enabled via nvidia-smi - ValidateMIGModeEnabled(ctx, s) + ValidateMIGModeEnabled(ctx, s, 1) // Validate that MIG instances are created - ValidateMIGInstancesCreated(ctx, s, "MIG 2g.20gb") + ValidateMIGInstancesCreated(ctx, s, "MIG 2g.20gb", 3) // Validate that GPU resources are advertised by the device plugin ValidateNodeAdvertisesGPUResources(ctx, s, 3, "nvidia.com/gpu") @@ -611,6 +611,57 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG(t *testing.T) { }) } +func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_MultiGPU(t *testing.T) { + const ( + gpuCount = 8 + migInstancesPerGPU = 3 + totalMIGInstances = gpuCount * migInstancesPerGPU + multiGPUA100VMSize = "Standard_ND96asr_v4" + ) + + RunScenario(t, &Scenario{ + Description: "Tests that a MIG profile is applied to every GPU on an Ubuntu 24.04 multi-GPU VM", + Location: "southcentralus", + K8sSystemPoolSKU: "Standard_D2s_v3", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2Containerd, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = multiGPUA100VMSize + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = true + nbc.EnableNvidia = true + nbc.GPUInstanceProfile = "MIG2g" + nbc.EnableManagedGPU = true + nbc.MigStrategy = "Single" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(multiGPUA100VMSize) + + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + versions := components.GetExpectedPackageVersions("nvidia-device-plugin", "ubuntu", "r2404") + require.Lenf(s.T, versions, 1, "Expected exactly one nvidia-device-plugin version for ubuntu r2404 but got %d", len(versions)) + ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) + + ValidateNvidiaDevicePluginServiceRunning(ctx, s) + ValidateSystemdUnitIsRunning(ctx, s, "nvidia-fabricmanager.service") + ValidateMIGModeEnabled(ctx, s, gpuCount) + ValidateMIGInstancesCreated(ctx, s, "MIG 2g.10gb", totalMIGInstances) + ValidateNodeAdvertisesGPUResources(ctx, s, totalMIGInstances, "nvidia.com/gpu") + ValidateGPUWorkloadSchedulable(ctx, s, 1, "nvidia.com/gpu") + }, + }, + }) +} + func Test_Ubuntu2204_NvidiaDevicePluginRunning_WithoutVMSSTag(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that NVIDIA device plugin and DCGM Exporter work via NBC EnableManagedGPU field without VMSS tag", @@ -766,10 +817,10 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_Mixed(t *testing.T) { ValidateNvidiaDevicePluginServiceRunning(ctx, s) // Validate that MIG mode is enabled via nvidia-smi - ValidateMIGModeEnabled(ctx, s) + ValidateMIGModeEnabled(ctx, s, 1) // Validate that MIG instances are created - ValidateMIGInstancesCreated(ctx, s, "MIG 1g.10gb") + ValidateMIGInstancesCreated(ctx, s, "MIG 1g.10gb", 7) // Validate that MIG profile-specific GPU resources are advertised by the device plugin migResourceName := "nvidia.com/mig-1g.10gb" From 5e0bfdfb6b82d13d02adb30ab8f416ec34aac18d Mon Sep 17 00:00:00 2001 From: Karen Chen Date: Thu, 23 Jul 2026 13:30:45 -0700 Subject: [PATCH 2/4] test(e2e): validate exact MIG GPU and instance counts --- e2e/validators.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/e2e/validators.go b/e2e/validators.go index abe8f2a3ab3..abe23b98a99 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -2589,26 +2589,29 @@ func ValidateNvidiaDCGMExporterScrapeCommonMetric(ctx context.Context, s *Scenar execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Nvidia DCGM Exporter is not returning "+metric) } -func ValidateMIGModeEnabled(ctx context.Context, s *Scenario) { +func ValidateMIGModeEnabled(ctx context.Context, s *Scenario, gpuCountExpected int) { s.T.Helper() - s.T.Logf("validating that MIG mode is enabled") + s.T.Logf("validating that MIG mode is enabled on %d GPUs", gpuCountExpected) command := []string{ "set -ex", - // Grep to verify it contains 'Enabled' - this will fail if MIG is disabled - "sudo nvidia-smi --query-gpu=mig.mode.current --format=csv,noheader | grep -i 'Enabled'", + "sudo nvidia-smi --query-gpu=mig.mode.current --format=csv,noheader", } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "MIG mode is not enabled") stdout := strings.TrimSpace(execResult.stdout) s.T.Logf("MIG mode status: %s", stdout) - require.Contains(s.T, stdout, "Enabled", "expected MIG mode to be enabled, but got: %s", stdout) - s.T.Logf("MIG mode is enabled") + gpuStatuses := strings.Split(stdout, "\n") + require.Len(s.T, gpuStatuses, gpuCountExpected, "expected MIG status for %d GPUs, but got: %s", gpuCountExpected, stdout) + for gpuIndex, gpuStatus := range gpuStatuses { + require.Equalf(s.T, "Enabled", strings.TrimSpace(gpuStatus), "expected MIG mode to be enabled on GPU %d", gpuIndex) + } + s.T.Logf("MIG mode is enabled on %d GPUs", gpuCountExpected) } -func ValidateMIGInstancesCreated(ctx context.Context, s *Scenario, migProfile string) { +func ValidateMIGInstancesCreated(ctx context.Context, s *Scenario, migProfile string, instanceCountExpected int) { s.T.Helper() - s.T.Logf("validating that MIG instances are created with profile %s", migProfile) + s.T.Logf("validating that %d MIG instances are created with profile %s", instanceCountExpected, migProfile) command := []string{ "set -ex", @@ -2620,9 +2623,10 @@ func ValidateMIGInstancesCreated(ctx context.Context, s *Scenario, migProfile st execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "MIG instances with profile "+migProfile+" were not found") stdout := execResult.stdout - require.Contains(s.T, stdout, migProfile, "expected to find MIG profile %s in output, but did not.\nOutput:\n%s", migProfile, stdout) + instanceCount := strings.Count(stdout, migProfile) + require.Equal(s.T, instanceCountExpected, instanceCount, "expected %d MIG instances with profile %s, but found %d.\nOutput:\n%s", instanceCountExpected, migProfile, instanceCount, stdout) require.NotContains(s.T, stdout, "No MIG-enabled devices found", "no MIG devices were created.\nOutput:\n%s", stdout) - s.T.Logf("MIG instances with profile %s are created", migProfile) + s.T.Logf("%d MIG instances with profile %s are created", instanceCountExpected, migProfile) } // ValidateIPTablesCompatibleWithCiliumEBPF validates that all iptables rules in each table match the provided patterns which are accounted for From f033980b81b2e214cba223ef1ac7a8302b213958 Mon Sep 17 00:00:00 2001 From: Karen Chen Date: Thu, 23 Jul 2026 14:30:00 -0700 Subject: [PATCH 3/4] test(e2e): use two-GPU MIG VM --- e2e/scenario_gpu_managed_experience_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index e2bd5ecfd0e..ebeafcb9d6e 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -613,15 +613,15 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG(t *testing.T) { func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_MultiGPU(t *testing.T) { const ( - gpuCount = 8 + gpuCount = 2 migInstancesPerGPU = 3 totalMIGInstances = gpuCount * migInstancesPerGPU - multiGPUA100VMSize = "Standard_ND96asr_v4" + multiGPUA100VMSize = "Standard_NC48ads_A100_v4" ) RunScenario(t, &Scenario{ Description: "Tests that a MIG profile is applied to every GPU on an Ubuntu 24.04 multi-GPU VM", - Location: "southcentralus", + Location: "westus3", K8sSystemPoolSKU: "Standard_D2s_v3", Tags: Tags{ GPU: true, @@ -652,9 +652,8 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_MultiGPU(t *testing.T) { ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) ValidateNvidiaDevicePluginServiceRunning(ctx, s) - ValidateSystemdUnitIsRunning(ctx, s, "nvidia-fabricmanager.service") ValidateMIGModeEnabled(ctx, s, gpuCount) - ValidateMIGInstancesCreated(ctx, s, "MIG 2g.10gb", totalMIGInstances) + ValidateMIGInstancesCreated(ctx, s, "MIG 2g.20gb", totalMIGInstances) ValidateNodeAdvertisesGPUResources(ctx, s, totalMIGInstances, "nvidia.com/gpu") ValidateGPUWorkloadSchedulable(ctx, s, 1, "nvidia.com/gpu") }, From e14887dd9680c902bdb86e7cdea2f8d03f2e4710 Mon Sep 17 00:00:00 2001 From: Karen Chen Date: Tue, 28 Jul 2026 11:16:44 -0700 Subject: [PATCH 4/4] test(e2e): count MIG instance lines --- e2e/scenario_gpu_managed_experience_test.go | 1 - e2e/validators.go | 13 ++++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index ebeafcb9d6e..371cd37b917 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -621,7 +621,6 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_MultiGPU(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that a MIG profile is applied to every GPU on an Ubuntu 24.04 multi-GPU VM", - Location: "westus3", K8sSystemPoolSKU: "Standard_D2s_v3", Tags: Tags{ GPU: true, diff --git a/e2e/validators.go b/e2e/validators.go index abe23b98a99..d61feca8ba2 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -2617,15 +2617,18 @@ func ValidateMIGInstancesCreated(ctx context.Context, s *Scenario, migProfile st "set -ex", // List MIG devices using nvidia-smi "sudo nvidia-smi mig -lgi", - // Ensure the output contains the expected MIG profile (will fail if "No MIG-enabled devices found") - "sudo nvidia-smi mig -lgi | grep -v 'No MIG-enabled devices found' | grep -q '" + migProfile + "'", } - execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "MIG instances with profile "+migProfile+" were not found") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to list MIG instances") stdout := execResult.stdout - instanceCount := strings.Count(stdout, migProfile) - require.Equal(s.T, instanceCountExpected, instanceCount, "expected %d MIG instances with profile %s, but found %d.\nOutput:\n%s", instanceCountExpected, migProfile, instanceCount, stdout) require.NotContains(s.T, stdout, "No MIG-enabled devices found", "no MIG devices were created.\nOutput:\n%s", stdout) + instanceCount := 0 + for _, line := range strings.Split(stdout, "\n") { + if strings.Contains(line, migProfile) { + instanceCount++ + } + } + require.Equal(s.T, instanceCountExpected, instanceCount, "expected %d MIG instances with profile %s, but found %d.\nOutput:\n%s", instanceCountExpected, migProfile, instanceCount, stdout) s.T.Logf("%d MIG instances with profile %s are created", instanceCountExpected, migProfile) }