diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index 89c29c12..7cc7bb39 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -11,7 +11,7 @@ project configuration. Test suites are defined in the [test-suites] section of azldev.toml and referenced by images via the [images.NAME.tests] subtable. Each test suite specifies a type -and framework-specific configuration in a matching subtable. +(pytest or lisa) and framework-specific configuration in a matching subtable. By default, all test suites associated with the named image are run. Use --test-suite to select specific suites (may be repeated). @@ -25,6 +25,9 @@ with the configured test paths and extra arguments. Use {image-path} in extra-args to insert the image path. Glob patterns (including **) in test-paths are expanded automatically. +For LISA tests, the test runner executes on the host and boots the image in a +QEMU VM. + ``` azldev image test IMAGE_NAME [flags] ``` diff --git a/internal/app/azldev/cmds/image/lisarunner.go b/internal/app/azldev/cmds/image/lisarunner.go new file mode 100644 index 00000000..59a910eb --- /dev/null +++ b/internal/app/azldev/cmds/image/lisarunner.go @@ -0,0 +1,343 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package image + +import ( + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +const ( + // lisaDirName is the parent directory under the project work dir for LISA-related state. + lisaDirName = "lisa" + // lisaVenvDirName is the name of the venv subdirectory for LISA. + lisaVenvDirName = "venv" + // lisaFrameworkDirName is the subdirectory for cloned LISA framework repos. + lisaFrameworkDirName = "framework" + // lisaRunbookDirName is the subdirectory for cloned runbook repos. + lisaRunbookDirName = "runbook" + // lisaKeysDirName is the subdirectory for auto-generated SSH key pairs. + lisaKeysDirName = "keys" + // lisaProgram is the LISA executable name inside the venv. + lisaProgram = "lisa" + // shortSHALength is the number of characters to use from a SHA for directory names. + shortSHALength = 12 + // adminKeyPlaceholder is the placeholder for the auto-generated admin SSH private key path. + adminKeyPlaceholder = "{admin-key-path}" +) + +// RunLisaSuite runs a LISA-based test suite by cloning the framework and runbook repos, +// setting up a venv, and invoking LISA. +func RunLisaSuite( + env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, + imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, +) error { + lisaConfig := suiteConfig.Lisa + if lisaConfig == nil { + return fmt.Errorf("test suite %#q is missing lisa configuration", suiteConfig.Name) + } + + slog.Info("Running LISA test suite", + slog.String("name", suiteConfig.Name), + slog.String("framework-ref", lisaConfig.Framework.Ref), + slog.String("runbook-ref", lisaConfig.Runbook.Ref), + slog.String("image-path", options.ImagePath), + ) + + // Ensure python3 and git are available. + if err := prereqs.RequireExecutable(env, pythonProgram, nil); err != nil { + return fmt.Errorf("python3 is required to run LISA tests:\n%w", err) + } + + if err := prereqs.RequireExecutable(env, "git", nil); err != nil { + return fmt.Errorf("git is required to clone LISA repos:\n%w", err) + } + + lisaBaseDir := filepath.Join(env.WorkDir(), lisaDirName) + + // Clone/update the LISA framework. + frameworkDir, err := ensureGitRepo( + env, lisaBaseDir, lisaFrameworkDirName, &lisaConfig.Framework, + ) + if err != nil { + return fmt.Errorf("failed to set up LISA framework:\n%w", err) + } + + // Set up or reuse the LISA venv and install the framework. + venvDir, err := ensureLisaVenv(env, suiteConfig.Name, frameworkDir, lisaConfig.PipPreInstall, lisaConfig.PipExtras) + if err != nil { + return err + } + + // Clone/update the runbook repo and resolve the runbook path. + runbookPath, err := resolveRunbookPath(env, lisaBaseDir, lisaConfig, frameworkDir) + if err != nil { + return err + } + + // Generate (or reuse) an SSH key pair for LISA to use for VM access. + adminKeyPath, err := ensureAdminKeyPair(env, lisaBaseDir) + if err != nil { + return err + } + + // Build LISA arguments with placeholder expansion. + lisaArgs := buildLisaArgs(runbookPath, lisaConfig.ExtraArgs, imageConfig, options, adminKeyPath) + + slog.Info("Running LISA", slog.Any("args", lisaArgs)) + + lisaBin := filepath.Join(venvDir, "bin", lisaProgram) + + // Run LISA from the framework's lisa/ directory so that relative extension paths + // (e.g., microsoft/testsuites) resolve correctly. + lisaWorkDir := filepath.Join(frameworkDir, "lisa") + + lisaCmd := exec.CommandContext(env, lisaBin, lisaArgs...) + lisaCmd.Dir = lisaWorkDir + lisaCmd.Stdout = os.Stdout + lisaCmd.Stderr = os.Stderr + + cmd, err := env.Command(lisaCmd) + if err != nil { + return fmt.Errorf("failed to create LISA command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return fmt.Errorf("LISA test run failed:\n%w", err) + } + + return nil +} + +// resolveRunbookPath clones the runbook repo (or reuses the framework clone if they match) +// and returns the absolute path to the runbook YAML file. +func resolveRunbookPath( + env *azldev.Env, lisaBaseDir string, lisaConfig *projectconfig.LisaConfig, frameworkDir string, +) (string, error) { + // Reuse the framework clone if the runbook is in the same repo and at the same ref. + runbookRepoDir := frameworkDir + + if lisaConfig.Runbook.GitURL != lisaConfig.Framework.GitURL || + lisaConfig.Runbook.Ref != lisaConfig.Framework.Ref { + var err error + + runbookRepoDir, err = ensureGitRepo( + env, lisaBaseDir, lisaRunbookDirName, &lisaConfig.Runbook.GitSourceConfig, + ) + if err != nil { + return "", fmt.Errorf("failed to set up LISA runbook repo:\n%w", err) + } + } else { + slog.Info("Runbook is in the same repo as framework; reusing clone") + } + + runbookPath := filepath.Join(runbookRepoDir, lisaConfig.Runbook.Path) + + runbookExists, err := fileutils.Exists(env.FS(), runbookPath) + if err != nil { + return "", fmt.Errorf("cannot check runbook at %#q:\n%w", runbookPath, err) + } + + if !runbookExists { + return "", fmt.Errorf("runbook not found at %#q in cloned repo", runbookPath) + } + + return runbookPath, nil +} + +// ensureGitRepo clones a git repo (if not already present) and checks out the specified +// commit SHA. Returns the path to the cloned repo directory. +func ensureGitRepo( + env *azldev.Env, baseDir string, category string, source *projectconfig.GitSourceConfig, +) (string, error) { + shortSHA := source.Ref[:shortSHALength] + repoDir := filepath.Join(baseDir, category, shortSHA) + + repoExists, err := fileutils.DirExists(env.FS(), repoDir) + if err != nil { + return "", fmt.Errorf("cannot check repo dir at %#q:\n%w", repoDir, err) + } + + gitProvider, err := git.NewGitProviderImpl(env, env) + if err != nil { + return "", fmt.Errorf("failed to create git provider:\n%w", err) + } + + if !repoExists { + slog.Info("Cloning git repo", + slog.String("url", source.GitURL), + slog.String("ref", source.Ref), + slog.String("dest", repoDir), + ) + + if err := gitProvider.Clone(env, source.GitURL, repoDir); err != nil { + return "", fmt.Errorf("failed to clone %#q:\n%w", source.GitURL, err) + } + + if err := gitProvider.Checkout(env, repoDir, source.Ref); err != nil { + return "", fmt.Errorf("failed to checkout %#q:\n%w", source.Ref, err) + } + } else { + slog.Info("Reusing existing git repo", + slog.String("path", repoDir), + slog.String("ref", source.Ref), + ) + } + + return repoDir, nil +} + +// ensureLisaVenv creates or reuses a Python venv for LISA and installs the framework via +// pip install -e. If pipExtras are specified, they are appended as pip extras +// (e.g., pip install -e ".[azure,legacy]"). +func ensureLisaVenv( + env *azldev.Env, suiteName string, frameworkDir string, + pipPreInstall []string, pipExtras []string, +) (string, error) { + venvDir := filepath.Join(env.WorkDir(), lisaDirName, lisaVenvDirName, suiteName) + + venvPython := filepath.Join(venvDir, "bin", pythonProgram) + + venvExists, err := fileutils.Exists(env.FS(), venvPython) + if err != nil { + return "", fmt.Errorf("cannot check LISA venv at %#q:\n%w", venvDir, err) + } + + if !venvExists { + slog.Info("Creating LISA Python venv", slog.String("path", venvDir)) + + if err := createPythonVenv(env, venvDir); err != nil { + return "", err + } + } else { + slog.Info("Reusing existing LISA venv", slog.String("path", venvDir)) + } + + // Install pre-install packages before the framework (to override version pins). + if len(pipPreInstall) > 0 { + slog.Info("Installing pre-install packages", slog.Any("packages", pipPreInstall)) + + preInstallArgs := append([]string{"-m", "pip", "install", "--quiet"}, pipPreInstall...) + + preInstallCmd := exec.CommandContext(env, venvPython, preInstallArgs...) + preInstallCmd.Stdout = os.Stdout + preInstallCmd.Stderr = os.Stderr + + cmd, err := env.Command(preInstallCmd) + if err != nil { + return "", fmt.Errorf("failed to create pip pre-install command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return "", fmt.Errorf("failed to install pre-install packages:\n%w", err) + } + } + + // Always refresh LISA installation from the framework directory. + slog.Info("Installing LISA framework", + slog.String("framework", frameworkDir), + ) + + // Build the pip install target, appending extras if specified. + pipTarget := frameworkDir + if len(pipExtras) > 0 { + pipTarget = frameworkDir + "[" + strings.Join(pipExtras, ",") + "]" + } + + pipCmd := exec.CommandContext( + env, venvPython, "-m", "pip", "install", "--quiet", "-e", pipTarget, + ) + pipCmd.Stdout = os.Stdout + pipCmd.Stderr = os.Stderr + + cmd, err := env.Command(pipCmd) + if err != nil { + return "", fmt.Errorf("failed to create pip install command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return "", fmt.Errorf("failed to install LISA framework:\n%w", err) + } + + return venvDir, nil +} + +// ensureAdminKeyPair generates (or reuses) an RSA SSH key pair for LISA VM access. +// The key pair is stored in the LISA work directory and reused across runs. +func ensureAdminKeyPair(env *azldev.Env, lisaBaseDir string) (string, error) { + keysDir := filepath.Join(lisaBaseDir, lisaKeysDirName) + privateKeyPath := filepath.Join(keysDir, "id_rsa") + + keyExists, err := fileutils.Exists(env.FS(), privateKeyPath) + if err != nil { + return "", fmt.Errorf("cannot check admin key at %#q:\n%w", privateKeyPath, err) + } + + if keyExists { + slog.Info("Reusing existing admin SSH key pair", slog.String("path", privateKeyPath)) + + return privateKeyPath, nil + } + + slog.Info("Generating admin SSH key pair", slog.String("path", privateKeyPath)) + + if err := fileutils.MkdirAll(env.FS(), keysDir); err != nil { + return "", fmt.Errorf("failed to create keys directory %#q:\n%w", keysDir, err) + } + + keygenCmd := exec.CommandContext( + env, "ssh-keygen", "-t", "rsa", "-b", "4096", "-f", privateKeyPath, "-N", "", "-q", + ) + + cmd, err := env.Command(keygenCmd) + if err != nil { + return "", fmt.Errorf("failed to create ssh-keygen command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return "", fmt.Errorf("failed to generate admin SSH key pair:\n%w", err) + } + + return privateKeyPath, nil +} + +// buildLisaArgs constructs the LISA command-line arguments. The runbook path is passed +// via -r, and extra-args are appended after placeholder expansion. +func buildLisaArgs( + runbookPath string, + extraArgs []string, + imageConfig *projectconfig.ImageConfig, + options *ImageTestOptions, + adminKeyPath string, +) []string { + absImagePath, err := filepath.Abs(options.ImagePath) + if err != nil { + absImagePath = options.ImagePath + } + + replacer := strings.NewReplacer( + imagePlaceholder, absImagePath, + imageNamePlaceholder, options.ImageName, + capabilitiesPlaceholder, strings.Join(imageConfig.Capabilities.EnabledNames(), ","), + adminKeyPlaceholder, adminKeyPath, + ) + + args := []string{"-r", runbookPath} + + for _, arg := range extraArgs { + args = append(args, replacer.Replace(arg)) + } + + return args +} diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index 77a6215a..5741d187 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -54,7 +54,7 @@ project configuration. Test suites are defined in the [test-suites] section of azldev.toml and referenced by images via the [images.NAME.tests] subtable. Each test suite specifies a type -and framework-specific configuration in a matching subtable. +(pytest or lisa) and framework-specific configuration in a matching subtable. By default, all test suites associated with the named image are run. Use --test-suite to select specific suites (may be repeated). @@ -66,7 +66,10 @@ For pytest tests, azldev creates a Python virtual environment, installs dependencies from pyproject.toml in the working directory, and runs pytest with the configured test paths and extra arguments. Use {image-path} in extra-args to insert the image path. Glob patterns (including **) in -test-paths are expanded automatically.`, +test-paths are expanded automatically. + +For LISA tests, the test runner executes on the host and boots the image in a +QEMU VM.`, Example: ` # Run all test suites for an image (artifact auto-resolved from output dir) azldev image test vm-base @@ -257,6 +260,9 @@ func runTestSuite( case projectconfig.TestTypePytest: return RunPytestSuite(env, suiteConfig, imageConfig, options) + case projectconfig.TestTypeLisa: + return RunLisaSuite(env, suiteConfig, imageConfig, options) + default: return fmt.Errorf("unsupported test type %#q for test suite %#q", suiteConfig.Type, suiteConfig.Name) } diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index 44e7d6a3..3e521e95 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -805,6 +805,22 @@ description = "Smoke tests for images" working-dir = "tests" test-paths = ["cases/test_*.py"] extra-args = ["--image-path", "{image-path}"] + +[test-suites.integration] +type = "lisa" +description = "LISA integration tests" + +[test-suites.integration.lisa] +extra-args = ["-v", "qcow2:{image-path}"] + +[test-suites.integration.lisa.framework] +git-url = "https://github.com/microsoft/lisa.git" +ref = "abcdef0123456789abcdef0123456789abcdef01" + +[test-suites.integration.lisa.runbook] +git-url = "https://github.com/microsoft/azurelinux.git" +ref = "abcdef0123456789abcdef0123456789abcdef01" +path = "tests/lisa/runbooks/azl-qemu.yml" ` configDir := filepath.Dir(testConfigPath) @@ -815,7 +831,7 @@ extra-args = ["--image-path", "{image-path}"] config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) require.NoError(t, err) - require.Len(t, config.TestSuites, 1) + require.Len(t, config.TestSuites, 2) // Check pytest test. if assert.Contains(t, config.TestSuites, "smoke") { @@ -828,6 +844,20 @@ extra-args = ["--image-path", "{image-path}"] assert.Equal(t, []string{"cases/test_*.py"}, smokeTest.Pytest.TestPaths) assert.Equal(t, []string{"--image-path", "{image-path}"}, smokeTest.Pytest.ExtraArgs) } + + // Check LISA test. + if assert.Contains(t, config.TestSuites, "integration") { + lisaTest := config.TestSuites["integration"] + assert.Equal(t, "integration", lisaTest.Name) + assert.Equal(t, TestTypeLisa, lisaTest.Type) + assert.Equal(t, "LISA integration tests", lisaTest.Description) + require.NotNil(t, lisaTest.Lisa) + assert.Equal(t, "https://github.com/microsoft/lisa.git", lisaTest.Lisa.Framework.GitURL) + assert.Equal(t, "abcdef0123456789abcdef0123456789abcdef01", lisaTest.Lisa.Framework.Ref) + assert.Equal(t, "https://github.com/microsoft/azurelinux.git", lisaTest.Lisa.Runbook.GitURL) + assert.Equal(t, "tests/lisa/runbooks/azl-qemu.yml", lisaTest.Lisa.Runbook.Path) + assert.Equal(t, []string{"-v", "qcow2:{image-path}"}, lisaTest.Lisa.ExtraArgs) + } } func TestLoadAndResolveProjectConfig_DuplicateTests(t *testing.T) { diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index a1cc917c..cdde588c 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -4,6 +4,7 @@ package projectconfig import ( + "encoding/hex" "errors" "fmt" @@ -16,6 +17,8 @@ type TestType string const ( // TestTypePytest uses pytest to run static/offline validation checks. TestTypePytest TestType = "pytest" + // TestTypeLisa uses the LISA framework to run live VM tests. + TestTypeLisa TestType = "lisa" ) var ( @@ -28,12 +31,12 @@ var ( // ErrUndefinedTestSuite is returned when an image references a test suite name that is not defined. ErrUndefinedTestSuite = errors.New("undefined test suite reference") // ErrMismatchedTestSubtable is returned when a test config has a subtable that does not - // match its declared type. Currently only one test type (pytest) exists, so this cannot - // trigger yet. When adding a new test type with its own subtable field, add cross-checks - // in [TestSuiteConfig.Validate] to ensure only the matching subtable is populated. + // match its declared type. ErrMismatchedTestSubtable = errors.New("mismatched test subtable") // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. ErrInvalidInstallMode = errors.New("invalid install mode") + // ErrInvalidGitRef is returned when a git ref is not a valid hex commit SHA. + ErrInvalidGitRef = errors.New("invalid git ref") ) // TestSuiteConfig defines a named test suite. @@ -45,11 +48,14 @@ type TestSuiteConfig struct { Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test suite"` // Type indicates the test framework to use. - Type TestType `toml:"type" json:"type" jsonschema:"required,enum=pytest,title=Type,description=Type of test framework (pytest)"` + Type TestType `toml:"type" json:"type" jsonschema:"required,enum=pytest lisa,title=Type,description=Type of test framework (pytest or lisa)"` // Pytest holds pytest-specific configuration. Required when Type is "pytest". Pytest *PytestConfig `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=Pytest-specific configuration (required when type is pytest)"` + // Lisa holds LISA-specific configuration. Required when Type is "lisa". + Lisa *LisaConfig `toml:"lisa,omitempty" json:"lisa,omitempty" jsonschema:"title=LISA config,description=LISA-specific configuration (required when type is lisa)"` + // Reference to the source config file that this definition came from; not present // in serialized files. SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-"` @@ -91,6 +97,92 @@ type PytestConfig struct { Install PytestInstallMode `toml:"install,omitempty" json:"install,omitempty" jsonschema:"enum=pyproject,enum=requirements,enum=none,title=Install mode,description=How to install Python dependencies: pyproject\\, requirements\\, or none (default)"` } +// LisaConfig holds configuration specific to LISA-based test suites. +type LisaConfig struct { + // Framework identifies the git source for the LISA framework itself. + Framework GitSourceConfig `toml:"framework" json:"framework" jsonschema:"required,title=Framework,description=Git source for the LISA framework"` + + // Runbook identifies the git source and path for the LISA runbook to run. + Runbook LisaRunbookConfig `toml:"runbook" json:"runbook" jsonschema:"required,title=Runbook,description=Git source and path for the LISA runbook"` + + // PipPreInstall lists pip packages to install before the LISA framework itself. + // This can be used to override framework version pins that conflict with the local + // environment (e.g., installing a system-matching libvirt-python version). + PipPreInstall []string `toml:"pip-pre-install,omitempty" json:"pipPreInstall,omitempty" jsonschema:"title=Pip pre-install,description=Pip packages to install before the framework (for overriding version pins)"` + + // PipExtras lists pip extras to install from the LISA framework package (e.g., "azure", + // "legacy"). These are appended to the pip install command as pip install -e ".[extra1,extra2]". + PipExtras []string `toml:"pip-extras,omitempty" json:"pipExtras,omitempty" jsonschema:"title=Pip extras,description=Pip extras to install from the LISA framework package"` + + // ExtraArgs is the list of additional arguments to pass to LISA. These are passed + // verbatim after placeholder substitution. Supports {image-path}, {image-name}, + // and {capabilities} placeholders. + ExtraArgs []string `toml:"extra-args,omitempty" json:"extraArgs,omitempty" jsonschema:"title=Extra arguments,description=Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders."` +} + +// GitSourceConfig identifies a git repository at a specific commit. +type GitSourceConfig struct { + // GitURL is the URL of the git repository. + GitURL string `toml:"git-url" json:"gitUrl" jsonschema:"required,title=Git URL,description=URL of the git repository"` + + // Ref is the commit SHA to check out. Must be a full hex commit hash. + Ref string `toml:"ref" json:"ref" jsonschema:"required,title=Ref,description=Commit SHA to check out (full hex hash)"` +} + +// Validate checks that the [GitSourceConfig] has required fields and a valid ref. +func (g *GitSourceConfig) Validate(context string) error { + if g.GitURL == "" { + return fmt.Errorf("%w: %s requires 'git-url'", ErrMissingTestField, context) + } + + if g.Ref == "" { + return fmt.Errorf("%w: %s requires 'ref'", ErrMissingTestField, context) + } + + if err := validateCommitSHA(g.Ref); err != nil { + return fmt.Errorf("%s: %w", context, err) + } + + return nil +} + +// LisaRunbookConfig identifies a LISA runbook within a git repository. +type LisaRunbookConfig struct { + GitSourceConfig `toml:",inline"` + + // Path is the path to the runbook YAML file within the repository. + Path string `toml:"path" json:"path" jsonschema:"required,title=Path,description=Path to the runbook YAML file within the repository"` +} + +// Validate checks that the [LisaRunbookConfig] has required fields. +func (r *LisaRunbookConfig) Validate(context string) error { + if err := r.GitSourceConfig.Validate(context); err != nil { + return err + } + + if r.Path == "" { + return fmt.Errorf("%w: %s requires 'path'", ErrMissingTestField, context) + } + + return nil +} + +// validateCommitSHA checks that s is a valid full-length hex commit SHA (40 characters). +func validateCommitSHA(ref string) error { + const commitSHALength = 40 + + if len(ref) != commitSHALength { + return fmt.Errorf("%w: expected %d hex characters, got %d: %#q", + ErrInvalidGitRef, commitSHALength, len(ref), ref) + } + + if _, err := hex.DecodeString(ref); err != nil { + return fmt.Errorf("%w: not a valid hex string: %#q", ErrInvalidGitRef, ref) + } + + return nil +} + // Validate checks that the test suite config has valid type-specific required fields and that // only the matching subtable is present. func (t *TestSuiteConfig) Validate() error { @@ -110,10 +202,31 @@ func (t *TestSuiteConfig) Validate() error { return fmt.Errorf("test suite %#q: %w", t.Name, err) } - // NOTE: When adding a new test type with its own subtable field (e.g., Lisa *LisaConfig), - // add a mismatch check here: - // if t.Lisa != nil { return fmt.Errorf("%w: ...", ErrMismatchedTestSubtable) } - // and add the symmetric check in the new type's case branch. + if t.Lisa != nil { + return fmt.Errorf("%w: test suite %#q of type %#q must not have a [lisa] subtable", + ErrMismatchedTestSubtable, t.Name, t.Type) + } + + case TestTypeLisa: + if t.Lisa == nil { + return fmt.Errorf("%w: test suite %#q of type %#q requires a [lisa] subtable", + ErrMissingTestField, t.Name, t.Type) + } + + frameworkContext := fmt.Sprintf("test suite %#q lisa.framework", t.Name) + if err := t.Lisa.Framework.Validate(frameworkContext); err != nil { + return err + } + + runbookContext := fmt.Sprintf("test suite %#q lisa.runbook", t.Name) + if err := t.Lisa.Runbook.Validate(runbookContext); err != nil { + return err + } + + if t.Pytest != nil { + return fmt.Errorf("%w: test suite %#q of type %#q must not have a [pytest] subtable", + ErrMismatchedTestSubtable, t.Name, t.Type) + } default: return fmt.Errorf("%w: %#q (test suite: %#q)", ErrUnknownTestType, t.Type, t.Name) @@ -195,5 +308,15 @@ func (t *TestSuiteConfig) WithAbsolutePaths(referenceDir string) *TestSuiteConfi } } + if t.Lisa != nil { + result.Lisa = &LisaConfig{ + Framework: t.Lisa.Framework, + Runbook: t.Lisa.Runbook, + PipPreInstall: t.Lisa.PipPreInstall, + PipExtras: t.Lisa.PipExtras, + ExtraArgs: t.Lisa.ExtraArgs, + } + } + return result } diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index 6d5e96e3..db0dcba5 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -12,6 +12,26 @@ import ( "github.com/stretchr/testify/require" ) +// validTestSHA is a 40-character hex string for use in tests. +const validTestSHA = "abcdef0123456789abcdef0123456789abcdef01" + +// validLisaConfig returns a valid [projectconfig.LisaConfig] for use in tests. +func validLisaConfig() *projectconfig.LisaConfig { + return &projectconfig.LisaConfig{ + Framework: projectconfig.GitSourceConfig{ + GitURL: "https://github.com/microsoft/lisa.git", + Ref: validTestSHA, + }, + Runbook: projectconfig.LisaRunbookConfig{ + GitSourceConfig: projectconfig.GitSourceConfig{ + GitURL: "https://github.com/microsoft/azurelinux.git", + Ref: validTestSHA, + }, + Path: "tests/lisa/runbooks/azl-qemu.yml", + }, + } +} + func TestImageCapabilities_EnabledNames(t *testing.T) { t.Run("all enabled", func(t *testing.T) { caps := projectconfig.ImageCapabilities{ @@ -170,6 +190,15 @@ func TestTestSuiteConfig_Validate(t *testing.T) { assert.NoError(t, testConfig.Validate()) }) + t.Run("valid lisa config", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: validLisaConfig(), + } + assert.NoError(t, testConfig.Validate()) + }) + t.Run("pytest missing subtable", func(t *testing.T) { testConfig := projectconfig.TestSuiteConfig{ Name: "smoke", @@ -181,6 +210,82 @@ func TestTestSuiteConfig_Validate(t *testing.T) { assert.Contains(t, err.Error(), "[pytest]") }) + t.Run("pytest with lisa subtable", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{}, + Lisa: validLisaConfig(), + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + }) + + t.Run("lisa missing subtable", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "[lisa]") + }) + + t.Run("lisa missing framework git-url", func(t *testing.T) { + cfg := validLisaConfig() + cfg.Framework.GitURL = "" + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: cfg, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "git-url") + }) + + t.Run("lisa invalid framework ref", func(t *testing.T) { + cfg := validLisaConfig() + cfg.Framework.Ref = "not-a-sha" + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: cfg, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidGitRef) + }) + + t.Run("lisa missing runbook path", func(t *testing.T) { + cfg := validLisaConfig() + cfg.Runbook.Path = "" + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: cfg, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "path") + }) + + t.Run("lisa with pytest subtable", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: validLisaConfig(), + Pytest: &projectconfig.PytestConfig{}, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + }) + t.Run("unknown test type", func(t *testing.T) { testConfig := projectconfig.TestSuiteConfig{ Name: "bad", diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 076dc148..cc40d3a2 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -520,6 +520,26 @@ "release-ver" ] }, + "GitSourceConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref" + ] + }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -637,6 +657,76 @@ "additionalProperties": false, "type": "object" }, + "LisaConfig": { + "properties": { + "framework": { + "$ref": "#/$defs/GitSourceConfig", + "title": "Framework", + "description": "Git source for the LISA framework" + }, + "runbook": { + "$ref": "#/$defs/LisaRunbookConfig", + "title": "Runbook", + "description": "Git source and path for the LISA runbook" + }, + "pip-pre-install": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip pre-install", + "description": "Pip packages to install before the framework (for overriding version pins)" + }, + "pip-extras": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip extras", + "description": "Pip extras to install from the LISA framework package" + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "framework", + "runbook" + ] + }, + "LisaRunbookConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + }, + "path": { + "type": "string", + "title": "Path", + "description": "Path to the runbook YAML file within the repository" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref", + "path" + ] + }, "Origin": { "properties": { "type": { @@ -931,15 +1021,20 @@ "type": { "type": "string", "enum": [ - "pytest" + "pytest lisa" ], "title": "Type", - "description": "Type of test framework (pytest)" + "description": "Type of test framework (pytest or lisa)" }, "pytest": { "$ref": "#/$defs/PytestConfig", "title": "Pytest config", "description": "Pytest-specific configuration (required when type is pytest)" + }, + "lisa": { + "$ref": "#/$defs/LisaConfig", + "title": "LISA config", + "description": "LISA-specific configuration (required when type is lisa)" } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 076dc148..cc40d3a2 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -520,6 +520,26 @@ "release-ver" ] }, + "GitSourceConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref" + ] + }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -637,6 +657,76 @@ "additionalProperties": false, "type": "object" }, + "LisaConfig": { + "properties": { + "framework": { + "$ref": "#/$defs/GitSourceConfig", + "title": "Framework", + "description": "Git source for the LISA framework" + }, + "runbook": { + "$ref": "#/$defs/LisaRunbookConfig", + "title": "Runbook", + "description": "Git source and path for the LISA runbook" + }, + "pip-pre-install": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip pre-install", + "description": "Pip packages to install before the framework (for overriding version pins)" + }, + "pip-extras": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip extras", + "description": "Pip extras to install from the LISA framework package" + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "framework", + "runbook" + ] + }, + "LisaRunbookConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + }, + "path": { + "type": "string", + "title": "Path", + "description": "Path to the runbook YAML file within the repository" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref", + "path" + ] + }, "Origin": { "properties": { "type": { @@ -931,15 +1021,20 @@ "type": { "type": "string", "enum": [ - "pytest" + "pytest lisa" ], "title": "Type", - "description": "Type of test framework (pytest)" + "description": "Type of test framework (pytest or lisa)" }, "pytest": { "$ref": "#/$defs/PytestConfig", "title": "Pytest config", "description": "Pytest-specific configuration (required when type is pytest)" + }, + "lisa": { + "$ref": "#/$defs/LisaConfig", + "title": "LISA config", + "description": "LISA-specific configuration (required when type is lisa)" } }, "additionalProperties": false, diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 076dc148..cc40d3a2 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -520,6 +520,26 @@ "release-ver" ] }, + "GitSourceConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref" + ] + }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -637,6 +657,76 @@ "additionalProperties": false, "type": "object" }, + "LisaConfig": { + "properties": { + "framework": { + "$ref": "#/$defs/GitSourceConfig", + "title": "Framework", + "description": "Git source for the LISA framework" + }, + "runbook": { + "$ref": "#/$defs/LisaRunbookConfig", + "title": "Runbook", + "description": "Git source and path for the LISA runbook" + }, + "pip-pre-install": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip pre-install", + "description": "Pip packages to install before the framework (for overriding version pins)" + }, + "pip-extras": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip extras", + "description": "Pip extras to install from the LISA framework package" + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "framework", + "runbook" + ] + }, + "LisaRunbookConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + }, + "path": { + "type": "string", + "title": "Path", + "description": "Path to the runbook YAML file within the repository" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref", + "path" + ] + }, "Origin": { "properties": { "type": { @@ -931,15 +1021,20 @@ "type": { "type": "string", "enum": [ - "pytest" + "pytest lisa" ], "title": "Type", - "description": "Type of test framework (pytest)" + "description": "Type of test framework (pytest or lisa)" }, "pytest": { "$ref": "#/$defs/PytestConfig", "title": "Pytest config", "description": "Pytest-specific configuration (required when type is pytest)" + }, + "lisa": { + "$ref": "#/$defs/LisaConfig", + "title": "LISA config", + "description": "LISA-specific configuration (required when type is lisa)" } }, "additionalProperties": false,