From 0bd5fe463750ab4c7816f30abf8bf5a8546cc0a5 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Fri, 24 Apr 2026 11:45:01 +0100 Subject: [PATCH 1/9] feat(cli): added Nexus package template and cli to validate the nexus package configuration Signed-off-by: Christian Pinto --- .copywrite.hcl | 2 +- .pre-commit-config.yaml | 9 +- README.md | 56 +++ ruff.toml | 23 +- src/algorithm_nexus/__init__.py | 4 + src/algorithm_nexus/cli.py | 94 +++++ src/algorithm_nexus/models.py | 182 +++++++++ src/algorithm_nexus/validation.py | 219 ++++++++++ templates/nexus-package-template/README.md | 98 +++++ .../models/your-model-name/model.yaml | 56 +++ .../your-model-name/tests/test_inference.py | 25 ++ .../models/your-model-name/tests/test_vllm.py | 26 ++ templates/nexus-package-template/nexus.yaml | 13 + tests/__init__.py | 4 + tests/fixtures/packages/.gitignore | 2 + tests/fixtures/packages/README.md | 74 ++++ .../models/broken-model/model.yaml | 19 + .../models/undeclared-model/model.yaml | 10 + .../packages/invalid-package/nexus.yaml | 7 + .../fixtures/packages/valid-package/AGENTS.md | 9 + .../models/example-model/benchmarks/custom.py | 10 + .../models/example-model/model.yaml | 36 ++ .../example-model/tests/test_inference.py | 16 + .../models/example-model/usage.md | 24 ++ .../packages/valid-package/nexus.yaml | 8 + tests/test_cli_validation.py | 279 +++++++++++++ tests/test_models.py | 377 ++++++++++++++++++ 27 files changed, 1654 insertions(+), 28 deletions(-) create mode 100644 src/algorithm_nexus/__init__.py create mode 100644 src/algorithm_nexus/cli.py create mode 100644 src/algorithm_nexus/models.py create mode 100644 src/algorithm_nexus/validation.py create mode 100644 templates/nexus-package-template/README.md create mode 100644 templates/nexus-package-template/models/your-model-name/model.yaml create mode 100644 templates/nexus-package-template/models/your-model-name/tests/test_inference.py create mode 100644 templates/nexus-package-template/models/your-model-name/tests/test_vllm.py create mode 100644 templates/nexus-package-template/nexus.yaml create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/packages/.gitignore create mode 100644 tests/fixtures/packages/README.md create mode 100644 tests/fixtures/packages/invalid-package/models/broken-model/model.yaml create mode 100644 tests/fixtures/packages/invalid-package/models/undeclared-model/model.yaml create mode 100644 tests/fixtures/packages/invalid-package/nexus.yaml create mode 100644 tests/fixtures/packages/valid-package/AGENTS.md create mode 100644 tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py create mode 100644 tests/fixtures/packages/valid-package/models/example-model/model.yaml create mode 100644 tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py create mode 100644 tests/fixtures/packages/valid-package/models/example-model/usage.md create mode 100644 tests/fixtures/packages/valid-package/nexus.yaml create mode 100644 tests/test_cli_validation.py create mode 100644 tests/test_models.py diff --git a/.copywrite.hcl b/.copywrite.hcl index 7820588..c90ecd2 100644 --- a/.copywrite.hcl +++ b/.copywrite.hcl @@ -2,7 +2,7 @@ schema_version = 1 project { license = "Apache-2.0" - copyright_year = 2024 + copyright_year = 2026 header_ignore = [ # Ignore generated files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa7178b..177d054 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,14 +15,7 @@ repos: - id: detect-secrets args: [--baseline, .secrets.baseline, --use-all-plugins, --fail-on-unaudited] - # Black - Python code formatter - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 26.3.1 - hooks: - - id: black-jupyter - language_version: python3 - - # Ruff - Python linter + # Ruff - Python linter and formatter - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.6 hooks: diff --git a/README.md b/README.md index a349231..4717a55 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,62 @@ of multiple models with different dependencies. - **Models scoreboard implemented** to track the performance of the integrated models +## Algorithm Nexus CLI + +Algorithm Nexus provides the `algorithm-nexus` CLI tool for managing Nexus +packages. This tool allows the validation of the structure of a Nexus Package. + +### Installation + +To use the CLI for package configuration validation, clone the repository and +install with uv: + +```bash +git clone https://github.com/IBM/algorithm-nexus.git +cd algorithm-nexus +uv sync --extra cli +``` + +### Available Tools + +#### Nexus Package Validation + +The validation tool checks: + +- **Package structure**: Verifies required files (`nexus.yaml`, `model.yaml`) + and directories (`tests/`) exist +- **YAML syntax**: Ensures all configuration files are valid YAML +- **Schema validation**: Validates configuration against Pydantic models for + correct field types and required fields +- **Cross-validation**: Checks dependencies between configurations (e.g., vLLM + enabled requires vLLM testing) +- **Model declarations**: Ensures all models in `nexus.yaml` have corresponding + directories + +Example usage: + +```bash +algorithm-nexus validate /path/to/package +``` + +In case of validation errors a detailed report guides the user to fix the +issues. + +## Getting Started + +### Creating a New Nexus Package + +Use the provided template to create a new Nexus package: + +```bash +cp -r templates/nexus-package-template /path/to/your-package +cd /path/to/your-package +``` + +Follow the instructions in the template's +[README](templates/nexus-package-template/README.md) to customize it for your +model. + ## Contributing This project is currently in closed beta. We are not accepting external diff --git a/ruff.toml b/ruff.toml index 7231958..ed1c3de 100644 --- a/ruff.toml +++ b/ruff.toml @@ -28,9 +28,6 @@ exclude = [ "venv", ] -# Same as Black. -line-length = 88 -indent-width = 4 # Assume Python 3.10 target-version = "py310" @@ -40,7 +37,7 @@ target-version = "py310" # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or # McCabe complexity (`C901`) by default. select = [ - "D", # pydocstyle + # "D", "E", # pycodestyle errors "F", # Pyflakes "FA", # flake8-future-annotations @@ -83,30 +80,18 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" # Ignore `S106` (Possible hardcoded password assigned to argument) # Ignore `S108` (Probable insecure usage of temporary file or directory) # Ignore `S311` (Standard pseudo-random generators are not suitable for cryptographic purposes) - -[format] -# Like Black, use double quotes for strings. -quote-style = "double" - -# Like Black, indent with spaces, rather than tabs. -indent-style = "space" - -# Like Black, respect magic trailing commas. -skip-magic-trailing-comma = false - -# Like Black, automatically detect the appropriate line ending. -line-ending = "auto" +"**/tests/**" = ["S101", "S105", "S106", "S108", "S311"] # Enable auto-formatting of code examples in docstrings. Markdown, # reStructuredText code/literal blocks and doctests are all supported. # # This is currently disabled by default, but it is planned for this # to be opt-out in the future. -docstring-code-format = false +# docstring-code-format = false # Set the line length limit used when formatting code snippets in # docstrings. # # This only has an effect when the `docstring-code-format` setting is # enabled. -docstring-code-line-length = "dynamic" +# docstring-code-line-length = "dynamic" diff --git a/src/algorithm_nexus/__init__.py b/src/algorithm_nexus/__init__.py new file mode 100644 index 0000000..e92c9f0 --- /dev/null +++ b/src/algorithm_nexus/__init__.py @@ -0,0 +1,4 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Algorithm Nexus package.""" diff --git a/src/algorithm_nexus/cli.py b/src/algorithm_nexus/cli.py new file mode 100644 index 0000000..bc7be02 --- /dev/null +++ b/src/algorithm_nexus/cli.py @@ -0,0 +1,94 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Command-line interface for Algorithm Nexus package validation.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +try: + import typer + from rich.console import Console + from rich.panel import Panel +except ImportError: + print( + "Error: CLI dependencies are not installed.\n" + "Please install them with: pip install algorithm-nexus[cli]", + file=sys.stderr, + ) + sys.exit(1) + +from algorithm_nexus.validation import ( + ValidationErrorCollector, + validate_package_directory, +) + +console = Console() + +app = typer.Typer( + help="Algorithm Nexus CLI - Tools for managing and validating Nexus packages.", + add_completion=False, +) + + +@app.callback(invoke_without_command=True) +def main_callback(ctx: typer.Context) -> None: + """Algorithm Nexus CLI - Tools for managing and validating Nexus packages.""" + if ctx.invoked_subcommand is None: + console.print(ctx.get_help()) + raise typer.Exit() + + +@app.command(name="validate") +def validate( + package_path: Path = typer.Argument( + ..., + help="Path to a Nexus package directory.", + ), +) -> None: + """Validate Nexus package structure and YAML configuration files.""" + collector = ValidationErrorCollector() + + resolved_path = package_path.resolve() + + if not resolved_path.exists(): + collector.add(f"Package path does not exist: {resolved_path}") + console.print( + Panel( + str(collector), + title="[bold red]Validation Failed[/bold red]", + border_style="red", + ) + ) + raise typer.Exit(code=1) + + validate_package_directory(resolved_path, collector) + + if collector.has_errors: + console.print( + Panel( + str(collector), + title="[bold red]Validation Failed[/bold red]", + border_style="red", + ) + ) + raise typer.Exit(code=1) + + console.print( + Panel( + "[green]✓[/green] All validation checks passed", + title="[bold green]Validation Successful[/bold green]", + border_style="green", + ) + ) + + +def main() -> None: + """Entry point for the CLI application.""" + app() + + +if __name__ == "__main__": + main() diff --git a/src/algorithm_nexus/models.py b/src/algorithm_nexus/models.py new file mode 100644 index 0000000..b1d5216 --- /dev/null +++ b/src/algorithm_nexus/models.py @@ -0,0 +1,182 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Pydantic models for Nexus package YAML validation.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field, model_validator + + +class AgentSkills(BaseModel): + """Agent skills configuration.""" + + embedded: bool | None = Field( + None, description="Whether agent skills are embedded in the package" + ) + external: str | None = Field( + None, description="External agent skills reference or URL" + ) + + @model_validator(mode="after") + def validate_mutually_exclusive(self) -> AgentSkills: + """Validate that if both embedded and external are defined, embedded must be False or None.""" + if self.external is not None and self.embedded is True: + raise ValueError( + "embedded must be False or None when external is defined. " + "Agent skills cannot be both embedded and external." + ) + return self + + +class PackageConfig(BaseModel): + """Package-level configuration.""" + + name: str = Field(..., min_length=1, description="Python package name") + agent_skills: AgentSkills | None = Field( + None, description="Agent skills configuration for this package" + ) + + +class VLLMPlugins(BaseModel): + """vLLM plugins configuration.""" + + general: str | None = Field(None, description="General vLLM plugin configuration") + io_processors: list[str] | None = Field( + None, description="List of I/O processor plugins for vLLM" + ) + + +class VLLMConfig(BaseModel): + """vLLM serving configuration. + + Should only be defined for models that require additional vLLM plugins + and belong to a Nexus Package targeting the product or candidate distribution variants. + """ + + enabled: Literal[True] = Field( + ..., description="Whether vLLM serving is enabled for this model" + ) + plugins: VLLMPlugins | None = Field(None, description="vLLM plugins configuration") + + +class GPUConfig(BaseModel): + """GPU hardware configuration.""" + + type: str | None = Field( + None, description="GPU type (e.g., 'NVIDIA A100', 'NVIDIA H100')" + ) + count: int | None = Field(None, description="Number of GPUs required") + cpu_fallback: bool | None = Field( + None, description="Whether CPU fallback is allowed if GPU is unavailable" + ) + + +class CPUConfig(BaseModel): + """CPU hardware configuration.""" + + cores: int | None = Field(None, description="Number of CPU cores required") + ram: str | None = Field(None, description="RAM requirement (e.g., '32GB', '64GB')") + + +class HardwareConfig(BaseModel): + """Hardware requirements configuration.""" + + gpu: GPUConfig | None = Field(None, description="GPU hardware requirements") + cpu: CPUConfig | None = Field(None, description="CPU hardware requirements") + + +class VLLMTestingConfig(BaseModel): + """vLLM-specific testing configuration.""" + + commands: list[str] = Field( + ..., min_length=1, description="Shell commands to run vLLM-specific tests" + ) + + +class ModelTestingConfig(BaseModel): + """Model testing configuration.""" + + hardware: HardwareConfig = Field( + ..., description="Hardware requirements for testing" + ) + commands: list[str] = Field( + ..., min_length=1, description="Shell commands to run tests" + ) + vllm: VLLMTestingConfig | None = Field( + None, + description="vLLM-specific testing configuration. Should only be defined for models that should be tested with vLLM and belong to a Nexus Package targeting the product or candidate distribution variants.", + ) + + +class BenchmarkExperiment(BaseModel): + """Benchmark experiment from catalog.""" + + name: str = Field( + ..., description="Name of the benchmark experiment from the catalog" + ) + args: str = Field(..., description="Arguments to pass to the benchmark experiment") + + +class CustomBenchmarkExperiment(BaseModel): + """Custom benchmark experiment.""" + + name: str = Field(..., description="Name of the custom benchmark experiment") + python_module: str = Field( + ..., description="Python module path for the custom benchmark" + ) + args: str = Field(..., description="Arguments to pass to the custom benchmark") + + +class BenchmarkingConfig(BaseModel): + """Model benchmarking configuration.""" + + experiments: list[BenchmarkExperiment] | None = Field( + None, description="List of benchmark experiments from the catalog" + ) + custom_experiments: list[CustomBenchmarkExperiment] | None = Field( + None, description="List of custom benchmark experiments" + ) + + +class ModelConfig(BaseModel): + """Model-level configuration.""" + + id: str = Field(..., min_length=1, description="Hugging Face model repository ID") + owner: str | None = Field(None, description="Owner or maintainer of the model") + vllm: VLLMConfig | None = Field( + None, + description="vLLM serving configuration. Only required for models that need additional vLLM plugins and belong to a Nexus Package targeting the product or candidate distribution variants.", + ) + testing: ModelTestingConfig = Field( + ..., description="Testing configuration for the model" + ) + benchmarking: BenchmarkingConfig | None = Field( + None, description="Benchmarking configuration for the model" + ) + + @model_validator(mode="after") + def validate_vllm_testing(self) -> ModelConfig: + """Validate that vllm testing is present when vllm is enabled.""" + if self.vllm is not None and self.vllm.enabled and self.testing.vllm is None: + raise ValueError( + "model.testing.vllm is required when model.vllm.enabled is true" + ) + return self + + +class ModelYAML(BaseModel): + """Root model.yaml structure.""" + + model: ModelConfig = Field(..., description="Model configuration") + + +class NexusYAML(BaseModel): + """Root nexus.yaml structure.""" + + package: PackageConfig = Field(..., description="Package-level configuration") + models: list[str] | None = Field( + default_factory=list, description="List of model directory names under models/" + ) diff --git a/src/algorithm_nexus/validation.py b/src/algorithm_nexus/validation.py new file mode 100644 index 0000000..867ae05 --- /dev/null +++ b/src/algorithm_nexus/validation.py @@ -0,0 +1,219 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Validation utilities for Nexus packages.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml +from pydantic import ValidationError + +from algorithm_nexus.models import ModelYAML, NexusYAML + + +class ValidationErrorCollector: + """Collects validation errors during package validation.""" + + def __init__(self) -> None: + """Initialize the error collector.""" + self.errors: list[str] = [] + + def add(self, message: str) -> None: + """Add a single error message.""" + self.errors.append(message) + + def extend(self, messages: list[str]) -> None: + """Add multiple error messages.""" + self.errors.extend(messages) + + @property + def has_errors(self) -> bool: + """Check if any errors have been collected.""" + return bool(self.errors) + + def __str__(self) -> str: + """Format errors as a bulleted list.""" + return "\n".join(f"[red]✗[/red] {error}" for error in self.errors) + + +def load_yaml_file( + path: Path, collector: ValidationErrorCollector +) -> dict[str, Any] | list[Any] | None: + """Load and parse a YAML file, collecting any errors.""" + try: + with path.open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) + except FileNotFoundError: + collector.add(f"Missing YAML file: {path}") + return None + except yaml.YAMLError as exc: + collector.add(f"Invalid YAML syntax in {path}: {exc}") + return None + + if data is None: + collector.add(f"YAML file is empty: {path}") + return None + + return data + + +def format_pydantic_error(error: dict[str, Any], file_path: Path) -> str: + """Format a Pydantic validation error into a readable message.""" + loc = ".".join(str(x) for x in error["loc"]) + msg = error["msg"] + + # Extract more context from the error if available + error_type = error.get("type", "") + + # For value_error types, the message usually contains the full explanation + if error_type.startswith("value_error"): + # The message already contains the full context + return f"[bold]{file_path}[/bold]\n Field: [cyan]{loc}[/cyan]\n Error: {msg}" + + # For missing field errors + if error_type == "missing": + return f"[bold]{file_path}[/bold]\n Field: [cyan]{loc}[/cyan]\n Error: This required field is missing" + + # Default format for other errors + return f"[bold]{file_path}[/bold]\n Field: [cyan]{loc}[/cyan]\n Error: {msg}" + + +def validate_nexus_yaml( + package_dir: Path, + collector: ValidationErrorCollector, +) -> tuple[NexusYAML | None, list[str]]: + """Validate nexus.yaml and return the config and model names.""" + nexus_yaml_path = package_dir / "nexus.yaml" + data = load_yaml_file(nexus_yaml_path, collector) + if data is None: + return None, [] + + if not isinstance(data, dict): + collector.add( + f"{nexus_yaml_path} must contain a YAML mapping at the top level." + ) + return None, [] + + try: + nexus_config = NexusYAML.model_validate(data) + model_names = nexus_config.models or [] + return nexus_config, model_names + except ValidationError as exc: + for error in exc.errors(): + collector.add(format_pydantic_error(error, nexus_yaml_path)) + return None, [] + + +def validate_model_yaml( + model_dir: Path, collector: ValidationErrorCollector, nexus_config: NexusYAML | None +) -> None: + """Validate a model's model.yaml file.""" + model_yaml_path = model_dir / "model.yaml" + data = load_yaml_file(model_yaml_path, collector) + if data is None: + return + + if not isinstance(data, dict): + collector.add( + f"{model_yaml_path} must contain a YAML mapping at the top level." + ) + return + + try: + ModelYAML.model_validate(data) + except ValidationError as exc: + for error in exc.errors(): + collector.add(format_pydantic_error(error, model_yaml_path)) + + +def validate_path_is_file( + path: Path, collector: ValidationErrorCollector, context: str +) -> bool: + """Validate that path is a file when it exists. Returns True if valid or doesn't exist.""" + if path.exists() and not path.is_file(): + collector.add(f"{context} must be a file when present: {path}") + return False + return True + + +def validate_path_is_dir( + path: Path, collector: ValidationErrorCollector, context: str +) -> bool: + """Validate that path is a directory when it exists. Returns True if valid or doesn't exist.""" + if path.exists() and not path.is_dir(): + collector.add(f"{context} must be a directory when present: {path}") + return False + return True + + +def validate_model_directory( + model_dir: Path, + collector: ValidationErrorCollector, + nexus_config: NexusYAML | None, +) -> None: + """Validate a single model directory structure and contents.""" + if not model_dir.is_dir(): + collector.add(f"Declared model directory is missing: {model_dir}") + return + + # Validate required tests directory + tests_dir = model_dir / "tests" + if not tests_dir.is_dir(): + collector.add(f"Missing required tests directory: {tests_dir}") + + # Validate optional files/directories + usage_md = model_dir / "usage.md" + validate_path_is_file(usage_md, collector, "usage.md") + + benchmarks_dir = model_dir / "benchmarks" + validate_path_is_dir(benchmarks_dir, collector, "benchmarks") + + # Validate model.yaml + validate_model_yaml(model_dir, collector, nexus_config) + + +def check_undeclared_models( + models_dir: Path, + declared_model_names: list[str], + collector: ValidationErrorCollector, +) -> None: + """Check for model directories that aren't declared in nexus.yaml.""" + for child in models_dir.iterdir(): + if child.is_dir() and child.name not in declared_model_names: + collector.add(f"Undeclared model directory found under models/: {child}") + + +def validate_package_directory( + package_dir: Path, collector: ValidationErrorCollector +) -> None: + """Validate the structure and contents of a Nexus package directory.""" + if not package_dir.is_dir(): + collector.add(f"Package path is not a directory: {package_dir}") + return + + nexus_config, model_names = validate_nexus_yaml(package_dir, collector) + + # Validate optional AGENTS.md + agents_md = package_dir / "AGENTS.md" + validate_path_is_file(agents_md, collector, "AGENTS.md") + + # Handle case where no models are declared + # Only validate models directory if there are models declared + if not model_names: + return + + models_dir = package_dir / "models" + if not models_dir.is_dir(): + collector.add(f"Missing required models directory: {models_dir}") + return + + # Validate each declared model + for model_name in model_names: + model_dir = models_dir / model_name + validate_model_directory(model_dir, collector, nexus_config) + + # Check for undeclared models + check_undeclared_models(models_dir, model_names, collector) diff --git a/templates/nexus-package-template/README.md b/templates/nexus-package-template/README.md new file mode 100644 index 0000000..66b4a08 --- /dev/null +++ b/templates/nexus-package-template/README.md @@ -0,0 +1,98 @@ +# Nexus Package Template + +This is a template for creating a new Nexus package. Follow the instructions +below to customize it for your model. + +## Quick Start + +1. **Copy this template** to create your package: + + ```bash + cp -r templates/nexus-package-template /path/to/your-package + cd /path/to/your-package + ``` + +2. **Rename the model directory**: + + ```bash + mv models/your-model-name models/your-actual-model-name + ``` + +3. **Update `nexus.yaml`**: + - Replace `your-package-name` with your package name + - Replace `your-model-name` with your model directory name + - Update version and agent_skills as needed + +4. **Update `models/your-model-name/model.yaml`**: + - Replace `your-org/your-model-name` with your model ID + - Replace `your-gh-id` with the GitHub ID of the model owner if explicitly + set. + - Configure hardware requirements + - Add vLLM configuration if needed (or remove the vllm section) + - Update test commands + +5. **Implement tests**: + - Add actual inference tests in `tests/test_inference.py` + - Add vLLM tests in `tests/test_vllm.py` (if using vLLM) + +6. **Validate your package**: + + ```bash + algorithm-nexus validate /path/to/your-package + ``` + +## Package Structure + +```text +your-package/ +├── nexus.yaml # Package configuration +├── models/ +│ └── your-model-name/ +│ ├── model.yaml # Model configuration +│ └── tests/ +│ ├── test_inference.py # Inference tests +│ └── test_vllm.py # vLLM tests (if applicable) +└── README.md +``` + +## Configuration Guidelines + +### vLLM Configuration + +- If your model uses vLLM, set `vllm.enabled: true` and provide vLLM testing +- If not using vLLM, remove the entire `vllm` section from `model.yaml` +- The `enabled` field must be `true` if the vllm section is present + +### Testing Requirements + +- All models must have a `tests/` directory with at least one test file +- Test commands must be specified in `model.testing.commands` +- If `vllm.enabled: true`, you must provide `model.testing.vllm.commands` + +### Hardware Requirements + +- Specify CPU requirements (cores and RAM) +- Optionally specify GPU requirements (type, count, cpu_fallback) + +## Documentation + +For detailed documentation on Nexus package requirements, see: + +- [Nexus Package Requirements](../../docs/requirements/nexus_package.md) +- [Contributing Guide](../../CONTRIBUTING.md) + +## Validation + +Before submitting your package, ensure it passes validation: + +```bash +algorithm-nexus validate /path/to/your-package +``` + +The validator checks: + +- Package structure (required files and directories) +- YAML syntax +- Schema validation (field types and required fields) +- Cross-validation (e.g., vLLM enabled requires vLLM testing) +- Model declarations match directories diff --git a/templates/nexus-package-template/models/your-model-name/model.yaml b/templates/nexus-package-template/models/your-model-name/model.yaml new file mode 100644 index 0000000..ad26a2c --- /dev/null +++ b/templates/nexus-package-template/models/your-model-name/model.yaml @@ -0,0 +1,56 @@ +# Model Configuration Template +# Replace placeholders with your actual model information + +model: + id: "your-org/your-model-name" # Replace with your model ID (e.g., "ibm/granite-3.0-8b") + # (optional) Model owner. + # owner: "your-gh-id" + + # vLLM Configuration (optional - remove if not using vLLM) + vllm: + enabled: true # Must be true if vLLM section is present + plugins: + # General plugin (optional) + # general: "your-vllm-plugin" + + # IO processors (optional) + io_processors: + - "your-io-processor" + + # Testing Configuration (required) + testing: + hardware: + # GPU configuration (optional) + gpu: + type: "NVIDIA A100" # Replace with your GPU type + count: 1 # Number of GPUs required + cpu_fallback: false # Whether CPU fallback is supported + + # CPU configuration (required) + cpu: + cores: 8 # Number of CPU cores required + ram: "32GB" # Amount of RAM required + + # Test commands (required) + commands: + - "pytest tests/test_inference.py -v" + + # vLLM testing (required if vllm.enabled is true) + vllm: + commands: + - "pytest tests/test_vllm.py -v" + + # Benchmarking Configuration (optional) + # If present, benchmarks should be defined in the benchmark + # folder inside the model folder. + # benchmarking: + # Standard experiments + # experiments: + # - name: "standard-benchmark" + # args: "--batch-size 8" + +# Custom experiments (optional) +# custom_experiments: +# - name: "custom-benchmark" +# python_module: "benchmarks/custom.py" +# args: "--mode advanced" diff --git a/templates/nexus-package-template/models/your-model-name/tests/test_inference.py b/templates/nexus-package-template/models/your-model-name/tests/test_inference.py new file mode 100644 index 0000000..3d16c65 --- /dev/null +++ b/templates/nexus-package-template/models/your-model-name/tests/test_inference.py @@ -0,0 +1,25 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Model Inference Tests. + +Add your model inference tests here. +""" + + +def test_basic_inference(): + """Test basic model inference. + + Replace this with actual inference tests for your model. + + Example: + model = load_model() + result = model.predict(input_data) + assert result is not None + + """ + # TODO: Implement actual inference test + assert True + + +# Add more tests as needed diff --git a/templates/nexus-package-template/models/your-model-name/tests/test_vllm.py b/templates/nexus-package-template/models/your-model-name/tests/test_vllm.py new file mode 100644 index 0000000..97f4e36 --- /dev/null +++ b/templates/nexus-package-template/models/your-model-name/tests/test_vllm.py @@ -0,0 +1,26 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""vLLM Integration Tests. + +Add your vLLM-specific tests here (only if vllm.enabled is true). +""" + + +def test_vllm_inference(): + """Test inference using vLLM. + + Replace this with actual vLLM inference tests. + + Example: + from vllm import LLM, SamplingParams + llm = LLM(model="your-org/your-model-name") + outputs = llm.generate(["Test prompt"], SamplingParams()) + assert len(outputs) > 0 + + """ + # TODO: Implement actual vLLM inference test + assert True + + +# Add more vLLM-specific tests as needed diff --git a/templates/nexus-package-template/nexus.yaml b/templates/nexus-package-template/nexus.yaml new file mode 100644 index 0000000..4faecab --- /dev/null +++ b/templates/nexus-package-template/nexus.yaml @@ -0,0 +1,13 @@ +# Nexus Package Configuration Template +# Replace placeholders with your actual package information + +package: + name: "your-package-name" # Replace with your package name (e.g., "terratorch", "tokamind") + version: "0.1.0" # Replace with your package version + agent_skills: + embedded: true # Set to true if agent skills are embedded in the package + +models: + - your-model-name # Replace with your model directory name(s) + # Add more models as needed: + # - another-model-name diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..7c4fa8c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Algorithm Nexus.""" diff --git a/tests/fixtures/packages/.gitignore b/tests/fixtures/packages/.gitignore new file mode 100644 index 0000000..d84cb91 --- /dev/null +++ b/tests/fixtures/packages/.gitignore @@ -0,0 +1,2 @@ +# Keep the fixture packages in git +!*/ diff --git a/tests/fixtures/packages/README.md b/tests/fixtures/packages/README.md new file mode 100644 index 0000000..f3120b9 --- /dev/null +++ b/tests/fixtures/packages/README.md @@ -0,0 +1,74 @@ +# Test Package Fixtures + +This directory contains sample Nexus packages for testing validation. + +## Valid Package (`valid-package/`) + +A complete, valid Nexus package that passes all validation checks. + +**Structure:** + +- ✅ Valid `nexus.yaml` with package metadata +- ✅ Optional `AGENTS.md` for embedded agent skills +- ✅ One declared model: `example-model` +- ✅ Complete model configuration with vLLM support +- ✅ Required `usage.md` documentation +- ✅ Required `tests/` directory with test files +- ✅ Optional `benchmarks/` directory with custom benchmark + +**Usage:** + +```bash +algorithm-nexus validate tests/fixtures/packages/valid-package +``` + +Expected result: ✅ Validation successful + +--- + +## Invalid Package (`invalid-package/`) + +A Nexus package with multiple validation errors for testing error detection. + +**Issues:** + +1. ❌ **Missing model ID**: `broken-model/model.yaml` is missing the required + `id` field +2. ❌ **Missing vLLM testing**: `broken-model` defines `vllm` but doesn't + provide `testing.vllm` +3. ❌ **Missing tests directory**: `broken-model` doesn't have required tests + directory +4. ❌ **Undeclared model**: `undeclared-model` exists but is not declared in + `nexus.yaml` + +**Usage:** + +```bash +algorithm-nexus validate tests/fixtures/packages/invalid-package +``` + +Expected result: ❌ Validation failed with multiple errors + +--- + +## Testing with Fixtures + +These fixtures can be used in automated tests: + +```python +from pathlib import Path +from algorithm_nexus.cli import _validate_package_directory, ValidationErrorCollector + +def test_valid_package(): + collector = ValidationErrorCollector() + package_dir = Path("tests/fixtures/packages/valid-package") + _validate_package_directory(package_dir, collector) + assert not collector.has_errors + +def test_invalid_package(): + collector = ValidationErrorCollector() + package_dir = Path("tests/fixtures/packages/invalid-package") + _validate_package_directory(package_dir, collector) + assert collector.has_errors + assert len(collector.errors) >= 5 # Multiple errors expected +``` diff --git a/tests/fixtures/packages/invalid-package/models/broken-model/model.yaml b/tests/fixtures/packages/invalid-package/models/broken-model/model.yaml new file mode 100644 index 0000000..f0f308c --- /dev/null +++ b/tests/fixtures/packages/invalid-package/models/broken-model/model.yaml @@ -0,0 +1,19 @@ +model: + # Missing required 'id' field - VALIDATION ERROR + owner: "broken-team" + + vllm: + enabled: true + plugins: + io_processors: + - "broken-processor" + # Missing vllm testing when vllm is enabled - VALIDATION ERROR + + testing: + hardware: + cpu: + cores: 4 + + commands: + - "pytest tests/" + # vllm testing is missing but vllm is defined above diff --git a/tests/fixtures/packages/invalid-package/models/undeclared-model/model.yaml b/tests/fixtures/packages/invalid-package/models/undeclared-model/model.yaml new file mode 100644 index 0000000..ee59ea3 --- /dev/null +++ b/tests/fixtures/packages/invalid-package/models/undeclared-model/model.yaml @@ -0,0 +1,10 @@ +model: + id: "broken-org/undeclared-model" + + testing: + hardware: + cpu: + cores: 2 + + commands: + - "pytest tests/" diff --git a/tests/fixtures/packages/invalid-package/nexus.yaml b/tests/fixtures/packages/invalid-package/nexus.yaml new file mode 100644 index 0000000..fff4993 --- /dev/null +++ b/tests/fixtures/packages/invalid-package/nexus.yaml @@ -0,0 +1,7 @@ +package: + name: "broken-package" + # Missing version is OK, but we'll have other issues + +models: + - broken-model + # Note: undeclared-model exists but is not listed here diff --git a/tests/fixtures/packages/valid-package/AGENTS.md b/tests/fixtures/packages/valid-package/AGENTS.md new file mode 100644 index 0000000..d9492b4 --- /dev/null +++ b/tests/fixtures/packages/valid-package/AGENTS.md @@ -0,0 +1,9 @@ +# Agent Skills + +This package provides embedded agent skills for geospatial analysis. + +## Available Skills + +- Flood detection and analysis +- Fire risk assessment +- Terrain analysis diff --git a/tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py b/tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py new file mode 100644 index 0000000..31df688 --- /dev/null +++ b/tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py @@ -0,0 +1,10 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Custom benchmark implementation.""" + + +def custom_benchmark(args): + """Run custom benchmark with given arguments.""" + print(f"Running custom benchmark with args: {args}") + return {"accuracy": 0.95, "latency": 100} diff --git a/tests/fixtures/packages/valid-package/models/example-model/model.yaml b/tests/fixtures/packages/valid-package/models/example-model/model.yaml new file mode 100644 index 0000000..7558ef7 --- /dev/null +++ b/tests/fixtures/packages/valid-package/models/example-model/model.yaml @@ -0,0 +1,36 @@ +model: + id: "example-org/example-model" + owner: "example-team" + + vllm: + enabled: true + plugins: + io_processors: + - "example-processor" + + testing: + hardware: + gpu: + type: "NVIDIA A100" + count: 1 + cpu_fallback: false + cpu: + cores: 8 + ram: "32GB" + + commands: + - "pytest tests/test_inference.py -v" + + vllm: + commands: + - "pytest tests/test_vllm.py -v" + + benchmarking: + experiments: + - name: "standard-benchmark" + args: "--batch-size 8" + + custom_experiments: + - name: "custom-benchmark" + python_module: "benchmarks/custom.py" + args: "--mode advanced" diff --git a/tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py b/tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py new file mode 100644 index 0000000..7dc753d --- /dev/null +++ b/tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py @@ -0,0 +1,16 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +# Test file for model inference + +"""Model inference tests.""" + + +def test_basic_inference(): + """Test basic model inference.""" + assert True + + +def test_batch_inference(): + """Test batch inference.""" + assert True diff --git a/tests/fixtures/packages/valid-package/models/example-model/usage.md b/tests/fixtures/packages/valid-package/models/example-model/usage.md new file mode 100644 index 0000000..2d0948e --- /dev/null +++ b/tests/fixtures/packages/valid-package/models/example-model/usage.md @@ -0,0 +1,24 @@ +# Example Model Usage + +This document provides usage instructions for the example model. + +## Installation + +```bash +pip install example-package +``` + +## Basic Usage + +```python +from example_package import ExampleModel + +model = ExampleModel.from_pretrained("example-org/example-model") +result = model.predict(input_data) +``` + +## Advanced Features + +- Batch processing +- GPU acceleration +- Custom preprocessing diff --git a/tests/fixtures/packages/valid-package/nexus.yaml b/tests/fixtures/packages/valid-package/nexus.yaml new file mode 100644 index 0000000..b96aecf --- /dev/null +++ b/tests/fixtures/packages/valid-package/nexus.yaml @@ -0,0 +1,8 @@ +package: + name: "example-package" + version: "1.0.0" + agent_skills: + embedded: true + +models: + - example-model diff --git a/tests/test_cli_validation.py b/tests/test_cli_validation.py new file mode 100644 index 0000000..294c7d3 --- /dev/null +++ b/tests/test_cli_validation.py @@ -0,0 +1,279 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for CLI validation command.""" + +from pathlib import Path +from textwrap import dedent + +import pytest +from typer.testing import CliRunner + +from algorithm_nexus.cli import app + +runner = CliRunner() + + +@pytest.fixture +def temp_package_dir(tmp_path: Path) -> Path: + """Create a temporary package directory structure.""" + package_dir = tmp_path / "packages" / "test-package" + package_dir.mkdir(parents=True) + return package_dir + + +def create_valid_nexus_yaml(package_dir: Path) -> None: + """Create a valid nexus.yaml file.""" + nexus_yaml = package_dir / "nexus.yaml" + nexus_yaml.write_text( + dedent(""" + package: + name: "test-package" + version: "1.0.0" + agent_skills: + embedded: true + + models: + - test-model + """) + ) + + +def create_valid_model_structure( + package_dir: Path, model_name: str = "test-model" +) -> None: + """Create a valid model directory structure.""" + model_dir = package_dir / "models" / model_name + model_dir.mkdir(parents=True) + + # Create model.yaml + model_yaml = model_dir / "model.yaml" + model_yaml.write_text( + dedent(""" + model: + id: "org/test-model" + owner: "test-team" + + vllm: + enabled: true + plugins: + io_processors: + - "test-processor" + + testing: + hardware: + gpu: + type: "NVIDIA A100" + count: 1 + cpu_fallback: false + cpu: + cores: 8 + ram: "32GB" + + commands: + - "pytest tests/test_inference.py -v" + + vllm: + commands: + - "pytest tests/test_vllm.py -v" + + benchmarking: + experiments: + - name: "test-experiment" + args: "--batch-size 8" + """) + ) + + # Create required files and directories + (model_dir / "usage.md").write_text("# Usage\n\nTest model usage.") + (model_dir / "tests").mkdir() + (model_dir / "tests" / "test_inference.py").write_text("# Test file") + + +class TestValidPackageStructure: + """Tests for fully correct package structure.""" + + def test_valid_package_passes_validation(self, temp_package_dir: Path) -> None: + """Test that a fully valid package passes validation.""" + create_valid_nexus_yaml(temp_package_dir) + create_valid_model_structure(temp_package_dir) + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 0 + assert "Validation Successful" in result.stdout + + def test_valid_package_with_multiple_models(self, temp_package_dir: Path) -> None: + """Test validation with multiple models.""" + nexus_yaml = temp_package_dir / "nexus.yaml" + nexus_yaml.write_text( + dedent(""" + package: + name: "test-package" + + models: + - model-1 + - model-2 + """) + ) + + # Create model structures without vLLM (ecosystem only) + for model_name in ["model-1", "model-2"]: + model_dir = temp_package_dir / "models" / model_name + model_dir.mkdir(parents=True) + + model_yaml = model_dir / "model.yaml" + model_yaml.write_text( + dedent(""" + model: + id: "org/model" + + testing: + hardware: + cpu: + cores: 4 + + commands: + - "pytest tests/" + """) + ) + + (model_dir / "usage.md").write_text("# Usage") + (model_dir / "tests").mkdir() + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 0 + assert "Validation Successful" in result.stdout + + +class TestMissingModelConfig: + """Tests for missing model configuration files.""" + + def test_missing_model_yaml(self, temp_package_dir: Path) -> None: + """Test that missing model.yaml is detected.""" + create_valid_nexus_yaml(temp_package_dir) + + model_dir = temp_package_dir / "models" / "test-model" + model_dir.mkdir(parents=True) + (model_dir / "usage.md").write_text("# Usage") + (model_dir / "tests").mkdir() + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 1 + assert "model.yaml" in result.stdout + + def test_missing_tests_directory(self, temp_package_dir: Path) -> None: + """Test that missing tests directory is detected.""" + create_valid_nexus_yaml(temp_package_dir) + create_valid_model_structure(temp_package_dir) + + # Remove tests directory + import shutil + + shutil.rmtree(temp_package_dir / "models" / "test-model" / "tests") + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 1 + assert "tests" in result.stdout + + +class TestMissingPackageConfig: + """Tests for missing package configuration.""" + + def test_missing_nexus_yaml(self, temp_package_dir: Path) -> None: + """Test that missing nexus.yaml is detected.""" + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 1 + assert "nexus.yaml" in result.stdout + + def test_empty_nexus_yaml(self, temp_package_dir: Path) -> None: + """Test that empty nexus.yaml is detected.""" + nexus_yaml = temp_package_dir / "nexus.yaml" + nexus_yaml.write_text("") + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 1 + assert "empty" in result.stdout.lower() + + +class TestMalformedPackageConfig: + """Tests for malformed package configuration.""" + + def test_invalid_yaml_syntax_in_nexus(self, temp_package_dir: Path) -> None: + """Test that invalid YAML syntax in nexus.yaml is detected.""" + nexus_yaml = temp_package_dir / "nexus.yaml" + nexus_yaml.write_text("package:\n name: [invalid yaml") + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 1 + assert "yaml" in result.stdout.lower() + + def test_undeclared_model_directory(self, temp_package_dir: Path) -> None: + """Test that undeclared model directories are detected.""" + create_valid_nexus_yaml(temp_package_dir) + create_valid_model_structure(temp_package_dir) + + # Create an undeclared model directory + undeclared_model = temp_package_dir / "models" / "undeclared-model" + undeclared_model.mkdir() + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 1 + assert "undeclared" in result.stdout.lower() + + +class TestCrossValidation: + """Tests for cross-validation between package and model configs.""" + + def test_vllm_requires_vllm_testing(self, temp_package_dir: Path) -> None: + """Test that vLLM configuration requires vLLM testing.""" + nexus_yaml = temp_package_dir / "nexus.yaml" + nexus_yaml.write_text( + dedent(""" + package: + name: "test-package" + + models: + - test-model + """) + ) + + model_dir = temp_package_dir / "models" / "test-model" + model_dir.mkdir(parents=True) + + model_yaml = model_dir / "model.yaml" + model_yaml.write_text( + dedent(""" + model: + id: "org/test-model" + + vllm: + enabled: true + plugins: + io_processors: + - "test-processor" + + testing: + hardware: + cpu: + cores: 4 + commands: + - "pytest tests/" + """) + ) + + (model_dir / "usage.md").write_text("# Usage") + (model_dir / "tests").mkdir() + + result = runner.invoke(app, ["validate", str(temp_package_dir)]) + + assert result.exit_code == 1 + assert "vllm" in result.stdout.lower() + assert "testing" in result.stdout.lower() diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..f8680fd --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,377 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for Pydantic models.""" + +from textwrap import dedent + +import pytest +import yaml +from pydantic import ValidationError + +from algorithm_nexus.models import ( + ModelConfig, + ModelTestingConfig, + ModelYAML, + NexusYAML, + PackageConfig, + VLLMConfig, +) + + +class TestPackageConfig: + """Tests for PackageConfig model.""" + + def test_valid_package_config(self) -> None: + """Test valid package configuration.""" + data = { + "name": "test-package", + } + config = PackageConfig(**data) + assert config.name == "test-package" + + def test_missing_name(self) -> None: + """Test that missing name is detected.""" + data = {} + with pytest.raises(ValidationError) as exc_info: + PackageConfig(**data) + assert "name" in str(exc_info.value) + + def test_with_agent_skills(self) -> None: + """Test package config with agent skills.""" + data = { + "name": "test-package", + "agent_skills": { + "embedded": True, + }, + } + config = PackageConfig(**data) + assert config.agent_skills is not None + assert config.agent_skills.embedded is True + + def test_agent_skills_embedded_false_and_external(self) -> None: + """Test that embedded=False and external can coexist.""" + data = { + "name": "test-package", + "agent_skills": { + "embedded": False, + "external": "https://example.com/skills", + }, + } + config = PackageConfig(**data) + assert config.agent_skills is not None + assert config.agent_skills.embedded is False + assert config.agent_skills.external == "https://example.com/skills" + + def test_agent_skills_embedded_none_and_external(self) -> None: + """Test that embedded=None and external can coexist.""" + data = { + "name": "test-package", + "agent_skills": { + "external": "https://example.com/skills", + }, + } + config = PackageConfig(**data) + assert config.agent_skills is not None + assert config.agent_skills.embedded is None + assert config.agent_skills.external == "https://example.com/skills" + + def test_agent_skills_embedded_true_and_external_fails(self) -> None: + """Test that embedded=True and external cannot coexist.""" + data = { + "name": "test-package", + "agent_skills": { + "embedded": True, + "external": "https://example.com/skills", + }, + } + with pytest.raises(ValidationError) as exc_info: + PackageConfig(**data) + assert "embedded must be False or None when external is defined" in str( + exc_info.value + ) + + +class TestVLLMConfig: + """Tests for VLLMConfig model.""" + + def test_vllm_with_plugins(self) -> None: + """Test vLLM with plugins.""" + data = { + "enabled": True, + "plugins": {"io_processors": ["processor1", "processor2"]}, + } + config = VLLMConfig(**data) + assert config.enabled is True + assert config.plugins is not None + assert len(config.plugins.io_processors) == 2 + + def test_vllm_with_general_plugin(self) -> None: + """Test vLLM with general plugin.""" + data = { + "enabled": True, + "plugins": {"general": "my-vllm-plugin"}, + } + config = VLLMConfig(**data) + assert config.enabled is True + assert config.plugins is not None + assert config.plugins.general == "my-vllm-plugin" + + def test_vllm_disabled_fails(self) -> None: + """Test that enabled=False is rejected.""" + data = {"enabled": False} + with pytest.raises(ValidationError) as exc_info: + VLLMConfig(**data) + assert "enabled" in str(exc_info.value).lower() + + def test_vllm_missing_enabled_fails(self) -> None: + """Test that missing enabled field is detected.""" + data = {"plugins": {"general": "my-plugin"}} + with pytest.raises(ValidationError) as exc_info: + VLLMConfig(**data) + assert "enabled" in str(exc_info.value) + + +class TestTestingConfig: + """Tests for TestingConfig model.""" + + def test_valid_testing_config(self) -> None: + """Test valid testing configuration.""" + data = { + "hardware": { + "cpu": { + "cores": 4, + } + }, + "commands": ["pytest tests/"], + } + config = ModelTestingConfig(**data) + assert config.commands == ["pytest tests/"] + assert config.hardware.cpu.cores == 4 + + def test_missing_commands(self) -> None: + """Test that missing commands is detected.""" + data = { + "hardware": { + "cpu": { + "cores": 4, + } + }, + } + with pytest.raises(ValidationError) as exc_info: + ModelTestingConfig(**data) + assert "commands" in str(exc_info.value) + + def test_vllm_testing_required_when_vllm_enabled(self) -> None: + """Test that vLLM testing is required when vLLM is enabled.""" + # This validation happens at the ModelConfig level + + +class TestModelConfig: + """Tests for ModelConfig model.""" + + def test_valid_model_config(self) -> None: + """Test valid model configuration.""" + data = { + "id": "org/model", + "testing": { + "hardware": { + "cpu": { + "cores": 4, + } + }, + "commands": ["pytest tests/"], + }, + } + config = ModelConfig(**data) + assert config.id == "org/model" + assert config.testing.commands == ["pytest tests/"] + + def test_model_without_vllm(self) -> None: + """Test that models without vLLM configuration are valid.""" + data = { + "id": "org/model", + "testing": { + "hardware": { + "cpu": { + "cores": 4, + } + }, + "commands": ["pytest tests/"], + }, + } + config = ModelConfig(**data) + assert config.id == "org/model" + assert config.vllm is None + assert config.testing.vllm is None + + def test_missing_model_id(self) -> None: + """Test that missing model.id is detected.""" + data = { + "testing": { + "hardware": { + "cpu": { + "cores": 4, + } + }, + "commands": ["pytest tests/"], + } + } + with pytest.raises(ValidationError) as exc_info: + ModelConfig(**data) + assert "id" in str(exc_info.value) + + def test_vllm_enabled_requires_vllm_testing(self) -> None: + """Test that vLLM enabled requires vLLM testing.""" + data = { + "id": "org/model", + "vllm": { + "enabled": True, + "plugins": {"io_processors": ["processor1"]}, + }, + "testing": { + "hardware": { + "cpu": { + "cores": 4, + } + }, + "commands": ["pytest tests/"], + }, + } + with pytest.raises(ValidationError) as exc_info: + ModelConfig(**data) + assert "vllm" in str(exc_info.value).lower() + assert "testing" in str(exc_info.value).lower() + + def test_vllm_disabled_fails(self) -> None: + """Test that vLLM enabled=False is rejected.""" + data = { + "id": "org/model", + "vllm": { + "enabled": False, + }, + "testing": { + "hardware": { + "cpu": { + "cores": 4, + } + }, + "commands": ["pytest tests/"], + }, + } + with pytest.raises(ValidationError) as exc_info: + ModelConfig(**data) + assert "enabled" in str(exc_info.value).lower() + + def test_vllm_with_vllm_testing(self) -> None: + """Test valid vLLM configuration with testing.""" + data = { + "id": "org/model", + "vllm": { + "enabled": True, + "plugins": {"io_processors": ["processor1"]}, + }, + "testing": { + "hardware": { + "cpu": { + "cores": 4, + } + }, + "commands": ["pytest tests/"], + "vllm": { + "commands": ["pytest tests/test_vllm.py"], + }, + }, + } + config = ModelConfig(**data) + assert config.vllm is not None + assert config.testing.vllm is not None + + +class TestModelYAML: + """Tests for ModelYAML model.""" + + def test_valid_model_yaml(self) -> None: + """Test valid model.yaml structure.""" + yaml_content = dedent(""" + model: + id: "org/test-model" + testing: + hardware: + cpu: + cores: 4 + commands: + - "pytest tests/" + """) + data = yaml.safe_load(yaml_content) + model_yaml = ModelYAML(**data) + assert model_yaml.model.id == "org/test-model" + + def test_model_yaml_with_vllm(self) -> None: + """Test model.yaml with vLLM configuration.""" + yaml_content = dedent(""" + model: + id: "org/test-model" + vllm: + enabled: true + plugins: + io_processors: + - "processor1" + testing: + hardware: + cpu: + cores: 4 + commands: + - "pytest tests/" + vllm: + commands: + - "pytest tests/test_vllm.py" + """) + data = yaml.safe_load(yaml_content) + model_yaml = ModelYAML(**data) + assert model_yaml.model.vllm is not None + assert model_yaml.model.vllm.plugins is not None + + +class TestNexusYAML: + """Tests for NexusYAML model.""" + + def test_valid_nexus_yaml(self) -> None: + """Test valid nexus.yaml structure.""" + yaml_content = dedent(""" + package: + name: "test-package" + + models: + - test-model + """) + data = yaml.safe_load(yaml_content) + nexus_yaml = NexusYAML(**data) + assert nexus_yaml.package.name == "test-package" + assert nexus_yaml.models == ["test-model"] + + def test_nexus_yaml_with_multiple_models(self) -> None: + """Test nexus.yaml with multiple models.""" + yaml_content = dedent(""" + package: + name: "test-package" + + models: + - model-1 + - model-2 + - model-3 + """) + data = yaml.safe_load(yaml_content) + nexus_yaml = NexusYAML(**data) + assert nexus_yaml.models is not None + assert len(nexus_yaml.models) == 3 + + def test_nexus_yaml_without_models(self) -> None: + """Test nexus.yaml without models list.""" + yaml_content = dedent(""" + package: + name: "test-package" + """) + data = yaml.safe_load(yaml_content) + nexus_yaml = NexusYAML(**data) + assert nexus_yaml.models is None or nexus_yaml.models == [] From da9482f2fbdd46232780c4c19c2b18770dba326c Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Tue, 28 Apr 2026 09:06:57 +0100 Subject: [PATCH 2/9] feat(cli): Updated the cli validation command to verify the approved nexus package structure Signed-off-by: Christian Pinto --- src/algorithm_nexus/cli.py | 12 +- src/algorithm_nexus/models.py | 146 ++---------- src/algorithm_nexus/validation.py | 152 ++++++------- .../models/your-model-name/model.yaml | 69 ++---- .../your-model-name/tests/test_inference.py | 25 -- .../models/your-model-name/tests/test_vllm.py | 26 --- templates/nexus-package-template/nexus.yaml | 10 +- .../models/example-model/benchmarks/custom.py | 10 - .../models/example-model/model.yaml | 27 --- .../example-model/tests/test_inference.py | 16 -- .../models/minimal-model/model.yaml | 2 + .../packages/valid-package/nexus.yaml | 6 - tests/test_cli_validation.py | 139 ++---------- tests/test_models.py | 213 ++---------------- 14 files changed, 159 insertions(+), 694 deletions(-) delete mode 100644 templates/nexus-package-template/models/your-model-name/tests/test_inference.py delete mode 100644 templates/nexus-package-template/models/your-model-name/tests/test_vllm.py delete mode 100644 tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py delete mode 100644 tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py create mode 100644 tests/fixtures/packages/valid-package/models/minimal-model/model.yaml diff --git a/src/algorithm_nexus/cli.py b/src/algorithm_nexus/cli.py index bc7be02..f83e067 100644 --- a/src/algorithm_nexus/cli.py +++ b/src/algorithm_nexus/cli.py @@ -69,16 +69,24 @@ def validate( if collector.has_errors: console.print( Panel( - str(collector), + collector.format_errors(), title="[bold red]Validation Failed[/bold red]", border_style="red", ) ) raise typer.Exit(code=1) + # Build success message + success_message = "[green]✓[/green] All validation checks passed" + + if collector.has_info: + success_message += ( + "\n\n[bold]Optional files/directories:[/bold]\n" + collector.format_info() + ) + console.print( Panel( - "[green]✓[/green] All validation checks passed", + success_message, title="[bold green]Validation Successful[/bold green]", border_style="green", ) diff --git a/src/algorithm_nexus/models.py b/src/algorithm_nexus/models.py index b1d5216..d84882e 100644 --- a/src/algorithm_nexus/models.py +++ b/src/algorithm_nexus/models.py @@ -7,45 +7,27 @@ from typing import Literal -from pydantic import BaseModel, Field, model_validator - - -class AgentSkills(BaseModel): - """Agent skills configuration.""" - - embedded: bool | None = Field( - None, description="Whether agent skills are embedded in the package" - ) - external: str | None = Field( - None, description="External agent skills reference or URL" - ) - - @model_validator(mode="after") - def validate_mutually_exclusive(self) -> AgentSkills: - """Validate that if both embedded and external are defined, embedded must be False or None.""" - if self.external is not None and self.embedded is True: - raise ValueError( - "embedded must be False or None when external is defined. " - "Agent skills cannot be both embedded and external." - ) - return self +from pydantic import BaseModel, Field class PackageConfig(BaseModel): """Package-level configuration.""" + model_config = {"extra": "forbid"} + name: str = Field(..., min_length=1, description="Python package name") - agent_skills: AgentSkills | None = Field( - None, description="Agent skills configuration for this package" - ) class VLLMPlugins(BaseModel): """vLLM plugins configuration.""" - general: str | None = Field(None, description="General vLLM plugin configuration") + model_config = {"extra": "forbid"} + + general: str | None = Field( + None, description="General vLLM plugin that loads the model class" + ) io_processors: list[str] | None = Field( - None, description="List of I/O processor plugins for vLLM" + None, description="List of vLLM IO processor plugins supported by this model" ) @@ -56,127 +38,43 @@ class VLLMConfig(BaseModel): and belong to a Nexus Package targeting the product or candidate distribution variants. """ + model_config = {"extra": "forbid"} + enabled: Literal[True] = Field( ..., description="Whether vLLM serving is enabled for this model" ) plugins: VLLMPlugins | None = Field(None, description="vLLM plugins configuration") -class GPUConfig(BaseModel): - """GPU hardware configuration.""" - - type: str | None = Field( - None, description="GPU type (e.g., 'NVIDIA A100', 'NVIDIA H100')" - ) - count: int | None = Field(None, description="Number of GPUs required") - cpu_fallback: bool | None = Field( - None, description="Whether CPU fallback is allowed if GPU is unavailable" - ) - - -class CPUConfig(BaseModel): - """CPU hardware configuration.""" - - cores: int | None = Field(None, description="Number of CPU cores required") - ram: str | None = Field(None, description="RAM requirement (e.g., '32GB', '64GB')") - - -class HardwareConfig(BaseModel): - """Hardware requirements configuration.""" - - gpu: GPUConfig | None = Field(None, description="GPU hardware requirements") - cpu: CPUConfig | None = Field(None, description="CPU hardware requirements") - - -class VLLMTestingConfig(BaseModel): - """vLLM-specific testing configuration.""" - - commands: list[str] = Field( - ..., min_length=1, description="Shell commands to run vLLM-specific tests" - ) - +class ModelConfig(BaseModel): + """Model-level configuration.""" -class ModelTestingConfig(BaseModel): - """Model testing configuration.""" + model_config = {"extra": "forbid"} - hardware: HardwareConfig = Field( - ..., description="Hardware requirements for testing" - ) - commands: list[str] = Field( - ..., min_length=1, description="Shell commands to run tests" + id: str = Field( + ..., min_length=1, description="Hugging Face model repository identifier" ) - vllm: VLLMTestingConfig | None = Field( + owner: str | None = Field( None, - description="vLLM-specific testing configuration. Should only be defined for models that should be tested with vLLM and belong to a Nexus Package targeting the product or candidate distribution variants.", - ) - - -class BenchmarkExperiment(BaseModel): - """Benchmark experiment from catalog.""" - - name: str = Field( - ..., description="Name of the benchmark experiment from the catalog" - ) - args: str = Field(..., description="Arguments to pass to the benchmark experiment") - - -class CustomBenchmarkExperiment(BaseModel): - """Custom benchmark experiment.""" - - name: str = Field(..., description="Name of the custom benchmark experiment") - python_module: str = Field( - ..., description="Python module path for the custom benchmark" - ) - args: str = Field(..., description="Arguments to pass to the custom benchmark") - - -class BenchmarkingConfig(BaseModel): - """Model benchmarking configuration.""" - - experiments: list[BenchmarkExperiment] | None = Field( - None, description="List of benchmark experiments from the catalog" - ) - custom_experiments: list[CustomBenchmarkExperiment] | None = Field( - None, description="List of custom benchmark experiments" + description="Model owner GitHub identifier. If omitted, ownership defaults to the Nexus package owner.", ) - - -class ModelConfig(BaseModel): - """Model-level configuration.""" - - id: str = Field(..., min_length=1, description="Hugging Face model repository ID") - owner: str | None = Field(None, description="Owner or maintainer of the model") vllm: VLLMConfig | None = Field( None, description="vLLM serving configuration. Only required for models that need additional vLLM plugins and belong to a Nexus Package targeting the product or candidate distribution variants.", ) - testing: ModelTestingConfig = Field( - ..., description="Testing configuration for the model" - ) - benchmarking: BenchmarkingConfig | None = Field( - None, description="Benchmarking configuration for the model" - ) - - @model_validator(mode="after") - def validate_vllm_testing(self) -> ModelConfig: - """Validate that vllm testing is present when vllm is enabled.""" - if self.vllm is not None and self.vllm.enabled and self.testing.vllm is None: - raise ValueError( - "model.testing.vllm is required when model.vllm.enabled is true" - ) - return self class ModelYAML(BaseModel): """Root model.yaml structure.""" + model_config = {"extra": "forbid"} + model: ModelConfig = Field(..., description="Model configuration") class NexusYAML(BaseModel): """Root nexus.yaml structure.""" + model_config = {"extra": "forbid"} + package: PackageConfig = Field(..., description="Package-level configuration") - models: list[str] | None = Field( - default_factory=list, description="List of model directory names under models/" - ) diff --git a/src/algorithm_nexus/validation.py b/src/algorithm_nexus/validation.py index 867ae05..a3e674d 100644 --- a/src/algorithm_nexus/validation.py +++ b/src/algorithm_nexus/validation.py @@ -15,16 +15,21 @@ class ValidationErrorCollector: - """Collects validation errors during package validation.""" + """Collects validation errors and info messages during package validation.""" def __init__(self) -> None: """Initialize the error collector.""" self.errors: list[str] = [] + self.info: list[str] = [] def add(self, message: str) -> None: """Add a single error message.""" self.errors.append(message) + def add_info(self, message: str) -> None: + """Add a single info message.""" + self.info.append(message) + def extend(self, messages: list[str]) -> None: """Add multiple error messages.""" self.errors.extend(messages) @@ -34,15 +39,32 @@ def has_errors(self) -> bool: """Check if any errors have been collected.""" return bool(self.errors) - def __str__(self) -> str: + @property + def has_info(self) -> bool: + """Check if any info messages have been collected.""" + return bool(self.info) + + def format_errors(self) -> str: """Format errors as a bulleted list.""" return "\n".join(f"[red]✗[/red] {error}" for error in self.errors) + def format_info(self) -> str: + """Format info messages as a bulleted list.""" + return "\n".join(f"[yellow]i[/yellow] {msg}" for msg in self.info) + + def __str__(self) -> str: + """Format errors as a bulleted list (for backward compatibility).""" + return self.format_errors() + def load_yaml_file( path: Path, collector: ValidationErrorCollector -) -> dict[str, Any] | list[Any] | None: - """Load and parse a YAML file, collecting any errors.""" +) -> dict[str, Any] | None: + """Load and parse a YAML file, collecting any errors. + + Returns a dict if successful, None otherwise. + Validates that the YAML contains a mapping (dict) at the top level. + """ try: with path.open("r", encoding="utf-8") as handle: data = yaml.safe_load(handle) @@ -57,6 +79,12 @@ def load_yaml_file( collector.add(f"YAML file is empty: {path}") return None + if not isinstance(data, dict): + collector.add( + f"{path} must contain a YAML mapping at the top level, got {type(data).__name__}" + ) + return None + return data @@ -84,44 +112,27 @@ def format_pydantic_error(error: dict[str, Any], file_path: Path) -> str: def validate_nexus_yaml( package_dir: Path, collector: ValidationErrorCollector, -) -> tuple[NexusYAML | None, list[str]]: - """Validate nexus.yaml and return the config and model names.""" +) -> None: + """Validate nexus.yaml.""" nexus_yaml_path = package_dir / "nexus.yaml" data = load_yaml_file(nexus_yaml_path, collector) if data is None: - return None, [] - - if not isinstance(data, dict): - collector.add( - f"{nexus_yaml_path} must contain a YAML mapping at the top level." - ) - return None, [] + return try: - nexus_config = NexusYAML.model_validate(data) - model_names = nexus_config.models or [] - return nexus_config, model_names + NexusYAML.model_validate(data) except ValidationError as exc: for error in exc.errors(): collector.add(format_pydantic_error(error, nexus_yaml_path)) - return None, [] -def validate_model_yaml( - model_dir: Path, collector: ValidationErrorCollector, nexus_config: NexusYAML | None -) -> None: +def validate_model_yaml(model_dir: Path, collector: ValidationErrorCollector) -> None: """Validate a model's model.yaml file.""" model_yaml_path = model_dir / "model.yaml" data = load_yaml_file(model_yaml_path, collector) if data is None: return - if not isinstance(data, dict): - collector.add( - f"{model_yaml_path} must contain a YAML mapping at the top level." - ) - return - try: ModelYAML.model_validate(data) except ValidationError as exc: @@ -129,61 +140,51 @@ def validate_model_yaml( collector.add(format_pydantic_error(error, model_yaml_path)) -def validate_path_is_file( +def validate_optional_file( path: Path, collector: ValidationErrorCollector, context: str ) -> bool: - """Validate that path is a file when it exists. Returns True if valid or doesn't exist.""" - if path.exists() and not path.is_file(): - collector.add(f"{context} must be a file when present: {path}") - return False + """Validate optional file and add info if missing. Returns True if valid or doesn't exist.""" + if path.exists(): + if not path.is_file(): + collector.add(f"{context} must be a file when present: {path}") + return False + else: + collector.add_info(f"{context}") return True -def validate_path_is_dir( +def validate_optional_dir( path: Path, collector: ValidationErrorCollector, context: str ) -> bool: - """Validate that path is a directory when it exists. Returns True if valid or doesn't exist.""" - if path.exists() and not path.is_dir(): - collector.add(f"{context} must be a directory when present: {path}") - return False + """Validate optional directory and add info if missing. Returns True if valid or doesn't exist.""" + if path.exists(): + if not path.is_dir(): + collector.add(f"{context} must be a directory when present: {path}") + return False + else: + collector.add_info(f"{context}") return True def validate_model_directory( model_dir: Path, collector: ValidationErrorCollector, - nexus_config: NexusYAML | None, ) -> None: """Validate a single model directory structure and contents.""" if not model_dir.is_dir(): - collector.add(f"Declared model directory is missing: {model_dir}") + collector.add(f"Model path is not a directory: {model_dir}") return - # Validate required tests directory - tests_dir = model_dir / "tests" - if not tests_dir.is_dir(): - collector.add(f"Missing required tests directory: {tests_dir}") - - # Validate optional files/directories + # Validate optional usage.md usage_md = model_dir / "usage.md" - validate_path_is_file(usage_md, collector, "usage.md") - - benchmarks_dir = model_dir / "benchmarks" - validate_path_is_dir(benchmarks_dir, collector, "benchmarks") + validate_optional_file( + usage_md, + collector, + f"Optional model file missing for '{model_dir.name}': usage.md", + ) # Validate model.yaml - validate_model_yaml(model_dir, collector, nexus_config) - - -def check_undeclared_models( - models_dir: Path, - declared_model_names: list[str], - collector: ValidationErrorCollector, -) -> None: - """Check for model directories that aren't declared in nexus.yaml.""" - for child in models_dir.iterdir(): - if child.is_dir() and child.name not in declared_model_names: - collector.add(f"Undeclared model directory found under models/: {child}") + validate_model_yaml(model_dir, collector) def validate_package_directory( @@ -194,26 +195,25 @@ def validate_package_directory( collector.add(f"Package path is not a directory: {package_dir}") return - nexus_config, model_names = validate_nexus_yaml(package_dir, collector) + validate_nexus_yaml(package_dir, collector) - # Validate optional AGENTS.md - agents_md = package_dir / "AGENTS.md" - validate_path_is_file(agents_md, collector, "AGENTS.md") + # Validate optional skills directory + skills_dir = package_dir / "skills" + validate_optional_dir( + skills_dir, collector, "Optional package directory missing: skills" + ) - # Handle case where no models are declared - # Only validate models directory if there are models declared - if not model_names: + # Check if models directory exists + models_dir = package_dir / "models" + if not models_dir.exists(): + # No models directory is valid - package may not have models + collector.add_info("Optional package directory missing: models") return - models_dir = package_dir / "models" if not models_dir.is_dir(): - collector.add(f"Missing required models directory: {models_dir}") + collector.add(f"models/ must be a directory when present: {models_dir}") return - # Validate each declared model - for model_name in model_names: - model_dir = models_dir / model_name - validate_model_directory(model_dir, collector, nexus_config) - - # Check for undeclared models - check_undeclared_models(models_dir, model_names, collector) + # Validate each model directory found + for model_dir in models_dir.iterdir(): + validate_model_directory(model_dir, collector) diff --git a/templates/nexus-package-template/models/your-model-name/model.yaml b/templates/nexus-package-template/models/your-model-name/model.yaml index ad26a2c..2b6570a 100644 --- a/templates/nexus-package-template/models/your-model-name/model.yaml +++ b/templates/nexus-package-template/models/your-model-name/model.yaml @@ -2,55 +2,20 @@ # Replace placeholders with your actual model information model: - id: "your-org/your-model-name" # Replace with your model ID (e.g., "ibm/granite-3.0-8b") - # (optional) Model owner. - # owner: "your-gh-id" - - # vLLM Configuration (optional - remove if not using vLLM) - vllm: - enabled: true # Must be true if vLLM section is present - plugins: - # General plugin (optional) - # general: "your-vllm-plugin" - - # IO processors (optional) - io_processors: - - "your-io-processor" - - # Testing Configuration (required) - testing: - hardware: - # GPU configuration (optional) - gpu: - type: "NVIDIA A100" # Replace with your GPU type - count: 1 # Number of GPUs required - cpu_fallback: false # Whether CPU fallback is supported - - # CPU configuration (required) - cpu: - cores: 8 # Number of CPU cores required - ram: "32GB" # Amount of RAM required - - # Test commands (required) - commands: - - "pytest tests/test_inference.py -v" - - # vLLM testing (required if vllm.enabled is true) - vllm: - commands: - - "pytest tests/test_vllm.py -v" - - # Benchmarking Configuration (optional) - # If present, benchmarks should be defined in the benchmark - # folder inside the model folder. - # benchmarking: - # Standard experiments - # experiments: - # - name: "standard-benchmark" - # args: "--batch-size 8" - -# Custom experiments (optional) -# custom_experiments: -# - name: "custom-benchmark" -# python_module: "benchmarks/custom.py" -# args: "--mode advanced" + id: "your-org/your-model-name" # Replace with Hugging Face model repository identifier (e.g., "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL") + + # (Optional) Model owner GitHub identifier + # If omitted, ownership defaults to the Nexus package owner + # owner: "your-github-username" + + # vLLM Configuration (optional - only include if model needs additional vLLM plugins) + # Only required for models targeting the product or candidate distribution variants + # vllm: + # enabled: true # Must be true if vLLM section is present + # plugins: + # # General plugin (optional) - loads the model class required in runtime + # # general: "your-vllm-plugin" + # + # # IO processors (optional) - list of vLLM IO processor plugins + # io_processors: + # - "your-io-processor-plugin" diff --git a/templates/nexus-package-template/models/your-model-name/tests/test_inference.py b/templates/nexus-package-template/models/your-model-name/tests/test_inference.py deleted file mode 100644 index 3d16c65..0000000 --- a/templates/nexus-package-template/models/your-model-name/tests/test_inference.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright IBM Corp. 2026 -# SPDX-License-Identifier: Apache-2.0 - -"""Model Inference Tests. - -Add your model inference tests here. -""" - - -def test_basic_inference(): - """Test basic model inference. - - Replace this with actual inference tests for your model. - - Example: - model = load_model() - result = model.predict(input_data) - assert result is not None - - """ - # TODO: Implement actual inference test - assert True - - -# Add more tests as needed diff --git a/templates/nexus-package-template/models/your-model-name/tests/test_vllm.py b/templates/nexus-package-template/models/your-model-name/tests/test_vllm.py deleted file mode 100644 index 97f4e36..0000000 --- a/templates/nexus-package-template/models/your-model-name/tests/test_vllm.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright IBM Corp. 2026 -# SPDX-License-Identifier: Apache-2.0 - -"""vLLM Integration Tests. - -Add your vLLM-specific tests here (only if vllm.enabled is true). -""" - - -def test_vllm_inference(): - """Test inference using vLLM. - - Replace this with actual vLLM inference tests. - - Example: - from vllm import LLM, SamplingParams - llm = LLM(model="your-org/your-model-name") - outputs = llm.generate(["Test prompt"], SamplingParams()) - assert len(outputs) > 0 - - """ - # TODO: Implement actual vLLM inference test - assert True - - -# Add more vLLM-specific tests as needed diff --git a/templates/nexus-package-template/nexus.yaml b/templates/nexus-package-template/nexus.yaml index 4faecab..c583652 100644 --- a/templates/nexus-package-template/nexus.yaml +++ b/templates/nexus-package-template/nexus.yaml @@ -2,12 +2,4 @@ # Replace placeholders with your actual package information package: - name: "your-package-name" # Replace with your package name (e.g., "terratorch", "tokamind") - version: "0.1.0" # Replace with your package version - agent_skills: - embedded: true # Set to true if agent skills are embedded in the package - -models: - - your-model-name # Replace with your model directory name(s) - # Add more models as needed: - # - another-model-name + name: "your-package-name" # Replace with your Python package name (e.g., "terratorch", "tokamind") diff --git a/tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py b/tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py deleted file mode 100644 index 31df688..0000000 --- a/tests/fixtures/packages/valid-package/models/example-model/benchmarks/custom.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright IBM Corp. 2026 -# SPDX-License-Identifier: Apache-2.0 - -"""Custom benchmark implementation.""" - - -def custom_benchmark(args): - """Run custom benchmark with given arguments.""" - print(f"Running custom benchmark with args: {args}") - return {"accuracy": 0.95, "latency": 100} diff --git a/tests/fixtures/packages/valid-package/models/example-model/model.yaml b/tests/fixtures/packages/valid-package/models/example-model/model.yaml index 7558ef7..79e3ea2 100644 --- a/tests/fixtures/packages/valid-package/models/example-model/model.yaml +++ b/tests/fixtures/packages/valid-package/models/example-model/model.yaml @@ -7,30 +7,3 @@ model: plugins: io_processors: - "example-processor" - - testing: - hardware: - gpu: - type: "NVIDIA A100" - count: 1 - cpu_fallback: false - cpu: - cores: 8 - ram: "32GB" - - commands: - - "pytest tests/test_inference.py -v" - - vllm: - commands: - - "pytest tests/test_vllm.py -v" - - benchmarking: - experiments: - - name: "standard-benchmark" - args: "--batch-size 8" - - custom_experiments: - - name: "custom-benchmark" - python_module: "benchmarks/custom.py" - args: "--mode advanced" diff --git a/tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py b/tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py deleted file mode 100644 index 7dc753d..0000000 --- a/tests/fixtures/packages/valid-package/models/example-model/tests/test_inference.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright IBM Corp. 2026 -# SPDX-License-Identifier: Apache-2.0 - -# Test file for model inference - -"""Model inference tests.""" - - -def test_basic_inference(): - """Test basic model inference.""" - assert True - - -def test_batch_inference(): - """Test batch inference.""" - assert True diff --git a/tests/fixtures/packages/valid-package/models/minimal-model/model.yaml b/tests/fixtures/packages/valid-package/models/minimal-model/model.yaml new file mode 100644 index 0000000..187914a --- /dev/null +++ b/tests/fixtures/packages/valid-package/models/minimal-model/model.yaml @@ -0,0 +1,2 @@ +model: + id: "example-org/minimal-model" diff --git a/tests/fixtures/packages/valid-package/nexus.yaml b/tests/fixtures/packages/valid-package/nexus.yaml index b96aecf..dc7c95a 100644 --- a/tests/fixtures/packages/valid-package/nexus.yaml +++ b/tests/fixtures/packages/valid-package/nexus.yaml @@ -1,8 +1,2 @@ package: name: "example-package" - version: "1.0.0" - agent_skills: - embedded: true - -models: - - example-model diff --git a/tests/test_cli_validation.py b/tests/test_cli_validation.py index 294c7d3..852f5e7 100644 --- a/tests/test_cli_validation.py +++ b/tests/test_cli_validation.py @@ -29,12 +29,6 @@ def create_valid_nexus_yaml(package_dir: Path) -> None: dedent(""" package: name: "test-package" - version: "1.0.0" - agent_skills: - embedded: true - - models: - - test-model """) ) @@ -59,35 +53,11 @@ def create_valid_model_structure( plugins: io_processors: - "test-processor" - - testing: - hardware: - gpu: - type: "NVIDIA A100" - count: 1 - cpu_fallback: false - cpu: - cores: 8 - ram: "32GB" - - commands: - - "pytest tests/test_inference.py -v" - - vllm: - commands: - - "pytest tests/test_vllm.py -v" - - benchmarking: - experiments: - - name: "test-experiment" - args: "--batch-size 8" """) ) - # Create required files and directories + # Create optional usage.md (model_dir / "usage.md").write_text("# Usage\n\nTest model usage.") - (model_dir / "tests").mkdir() - (model_dir / "tests" / "test_inference.py").write_text("# Test file") class TestValidPackageStructure: @@ -105,19 +75,9 @@ def test_valid_package_passes_validation(self, temp_package_dir: Path) -> None: def test_valid_package_with_multiple_models(self, temp_package_dir: Path) -> None: """Test validation with multiple models.""" - nexus_yaml = temp_package_dir / "nexus.yaml" - nexus_yaml.write_text( - dedent(""" - package: - name: "test-package" - - models: - - model-1 - - model-2 - """) - ) - - # Create model structures without vLLM (ecosystem only) + create_valid_nexus_yaml(temp_package_dir) + + # Create model structures for model_name in ["model-1", "model-2"]: model_dir = temp_package_dir / "models" / model_name model_dir.mkdir(parents=True) @@ -127,19 +87,10 @@ def test_valid_package_with_multiple_models(self, temp_package_dir: Path) -> Non dedent(""" model: id: "org/model" - - testing: - hardware: - cpu: - cores: 4 - - commands: - - "pytest tests/" """) ) (model_dir / "usage.md").write_text("# Usage") - (model_dir / "tests").mkdir() result = runner.invoke(app, ["validate", str(temp_package_dir)]) @@ -157,27 +108,11 @@ def test_missing_model_yaml(self, temp_package_dir: Path) -> None: model_dir = temp_package_dir / "models" / "test-model" model_dir.mkdir(parents=True) (model_dir / "usage.md").write_text("# Usage") - (model_dir / "tests").mkdir() - - result = runner.invoke(app, ["validate", str(temp_package_dir)]) - - assert result.exit_code == 1 - assert "model.yaml" in result.stdout - - def test_missing_tests_directory(self, temp_package_dir: Path) -> None: - """Test that missing tests directory is detected.""" - create_valid_nexus_yaml(temp_package_dir) - create_valid_model_structure(temp_package_dir) - - # Remove tests directory - import shutil - - shutil.rmtree(temp_package_dir / "models" / "test-model" / "tests") result = runner.invoke(app, ["validate", str(temp_package_dir)]) assert result.exit_code == 1 - assert "tests" in result.stdout + assert "Missing YAML file" in result.stdout class TestMissingPackageConfig: @@ -212,68 +147,24 @@ def test_invalid_yaml_syntax_in_nexus(self, temp_package_dir: Path) -> None: result = runner.invoke(app, ["validate", str(temp_package_dir)]) assert result.exit_code == 1 - assert "yaml" in result.stdout.lower() - - def test_undeclared_model_directory(self, temp_package_dir: Path) -> None: - """Test that undeclared model directories are detected.""" - create_valid_nexus_yaml(temp_package_dir) - create_valid_model_structure(temp_package_dir) - # Create an undeclared model directory - undeclared_model = temp_package_dir / "models" / "undeclared-model" - undeclared_model.mkdir() + def test_yaml_list_instead_of_dict(self, temp_package_dir: Path) -> None: + """Test that YAML containing a list instead of dict is rejected.""" + nexus_yaml = temp_package_dir / "nexus.yaml" + nexus_yaml.write_text("- item1\n- item2\n") result = runner.invoke(app, ["validate", str(temp_package_dir)]) assert result.exit_code == 1 - assert "undeclared" in result.stdout.lower() - + assert "must contain a YAML mapping at the top level" in result.stdout -class TestCrossValidation: - """Tests for cross-validation between package and model configs.""" - - def test_vllm_requires_vllm_testing(self, temp_package_dir: Path) -> None: - """Test that vLLM configuration requires vLLM testing.""" + def test_yaml_string_instead_of_dict(self, temp_package_dir: Path) -> None: + """Test that YAML containing a string instead of dict is rejected.""" nexus_yaml = temp_package_dir / "nexus.yaml" - nexus_yaml.write_text( - dedent(""" - package: - name: "test-package" - - models: - - test-model - """) - ) - - model_dir = temp_package_dir / "models" / "test-model" - model_dir.mkdir(parents=True) - - model_yaml = model_dir / "model.yaml" - model_yaml.write_text( - dedent(""" - model: - id: "org/test-model" - - vllm: - enabled: true - plugins: - io_processors: - - "test-processor" - - testing: - hardware: - cpu: - cores: 4 - commands: - - "pytest tests/" - """) - ) - - (model_dir / "usage.md").write_text("# Usage") - (model_dir / "tests").mkdir() + nexus_yaml.write_text("just a string\n") result = runner.invoke(app, ["validate", str(temp_package_dir)]) assert result.exit_code == 1 - assert "vllm" in result.stdout.lower() - assert "testing" in result.stdout.lower() + assert "must contain a YAML mapping at the top level" in result.stdout + assert "yaml" in result.stdout.lower() diff --git a/tests/test_models.py b/tests/test_models.py index f8680fd..d53dd41 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -11,7 +11,6 @@ from algorithm_nexus.models import ( ModelConfig, - ModelTestingConfig, ModelYAML, NexusYAML, PackageConfig, @@ -37,60 +36,6 @@ def test_missing_name(self) -> None: PackageConfig(**data) assert "name" in str(exc_info.value) - def test_with_agent_skills(self) -> None: - """Test package config with agent skills.""" - data = { - "name": "test-package", - "agent_skills": { - "embedded": True, - }, - } - config = PackageConfig(**data) - assert config.agent_skills is not None - assert config.agent_skills.embedded is True - - def test_agent_skills_embedded_false_and_external(self) -> None: - """Test that embedded=False and external can coexist.""" - data = { - "name": "test-package", - "agent_skills": { - "embedded": False, - "external": "https://example.com/skills", - }, - } - config = PackageConfig(**data) - assert config.agent_skills is not None - assert config.agent_skills.embedded is False - assert config.agent_skills.external == "https://example.com/skills" - - def test_agent_skills_embedded_none_and_external(self) -> None: - """Test that embedded=None and external can coexist.""" - data = { - "name": "test-package", - "agent_skills": { - "external": "https://example.com/skills", - }, - } - config = PackageConfig(**data) - assert config.agent_skills is not None - assert config.agent_skills.embedded is None - assert config.agent_skills.external == "https://example.com/skills" - - def test_agent_skills_embedded_true_and_external_fails(self) -> None: - """Test that embedded=True and external cannot coexist.""" - data = { - "name": "test-package", - "agent_skills": { - "embedded": True, - "external": "https://example.com/skills", - }, - } - with pytest.raises(ValidationError) as exc_info: - PackageConfig(**data) - assert "embedded must be False or None when external is defined" in str( - exc_info.value - ) - class TestVLLMConfig: """Tests for VLLMConfig model.""" @@ -132,41 +77,6 @@ def test_vllm_missing_enabled_fails(self) -> None: assert "enabled" in str(exc_info.value) -class TestTestingConfig: - """Tests for TestingConfig model.""" - - def test_valid_testing_config(self) -> None: - """Test valid testing configuration.""" - data = { - "hardware": { - "cpu": { - "cores": 4, - } - }, - "commands": ["pytest tests/"], - } - config = ModelTestingConfig(**data) - assert config.commands == ["pytest tests/"] - assert config.hardware.cpu.cores == 4 - - def test_missing_commands(self) -> None: - """Test that missing commands is detected.""" - data = { - "hardware": { - "cpu": { - "cores": 4, - } - }, - } - with pytest.raises(ValidationError) as exc_info: - ModelTestingConfig(**data) - assert "commands" in str(exc_info.value) - - def test_vllm_testing_required_when_vllm_enabled(self) -> None: - """Test that vLLM testing is required when vLLM is enabled.""" - # This validation happens at the ModelConfig level - - class TestModelConfig: """Tests for ModelConfig model.""" @@ -174,74 +84,35 @@ def test_valid_model_config(self) -> None: """Test valid model configuration.""" data = { "id": "org/model", - "testing": { - "hardware": { - "cpu": { - "cores": 4, - } - }, - "commands": ["pytest tests/"], - }, } config = ModelConfig(**data) assert config.id == "org/model" - assert config.testing.commands == ["pytest tests/"] def test_model_without_vllm(self) -> None: """Test that models without vLLM configuration are valid.""" data = { "id": "org/model", - "testing": { - "hardware": { - "cpu": { - "cores": 4, - } - }, - "commands": ["pytest tests/"], - }, } config = ModelConfig(**data) assert config.id == "org/model" assert config.vllm is None - assert config.testing.vllm is None def test_missing_model_id(self) -> None: """Test that missing model.id is detected.""" - data = { - "testing": { - "hardware": { - "cpu": { - "cores": 4, - } - }, - "commands": ["pytest tests/"], - } - } + data = {} with pytest.raises(ValidationError) as exc_info: ModelConfig(**data) assert "id" in str(exc_info.value) - def test_vllm_enabled_requires_vllm_testing(self) -> None: - """Test that vLLM enabled requires vLLM testing.""" + def test_model_with_owner(self) -> None: + """Test model configuration with owner.""" data = { "id": "org/model", - "vllm": { - "enabled": True, - "plugins": {"io_processors": ["processor1"]}, - }, - "testing": { - "hardware": { - "cpu": { - "cores": 4, - } - }, - "commands": ["pytest tests/"], - }, + "owner": "github-username", } - with pytest.raises(ValidationError) as exc_info: - ModelConfig(**data) - assert "vllm" in str(exc_info.value).lower() - assert "testing" in str(exc_info.value).lower() + config = ModelConfig(**data) + assert config.id == "org/model" + assert config.owner == "github-username" def test_vllm_disabled_fails(self) -> None: """Test that vLLM enabled=False is rejected.""" @@ -250,42 +121,24 @@ def test_vllm_disabled_fails(self) -> None: "vllm": { "enabled": False, }, - "testing": { - "hardware": { - "cpu": { - "cores": 4, - } - }, - "commands": ["pytest tests/"], - }, } with pytest.raises(ValidationError) as exc_info: ModelConfig(**data) assert "enabled" in str(exc_info.value).lower() - def test_vllm_with_vllm_testing(self) -> None: - """Test valid vLLM configuration with testing.""" + def test_vllm_with_plugins(self) -> None: + """Test valid vLLM configuration with plugins.""" data = { "id": "org/model", "vllm": { "enabled": True, "plugins": {"io_processors": ["processor1"]}, }, - "testing": { - "hardware": { - "cpu": { - "cores": 4, - } - }, - "commands": ["pytest tests/"], - "vllm": { - "commands": ["pytest tests/test_vllm.py"], - }, - }, } config = ModelConfig(**data) assert config.vllm is not None - assert config.testing.vllm is not None + assert config.vllm.plugins is not None + assert config.vllm.plugins.io_processors == ["processor1"] class TestModelYAML: @@ -296,12 +149,6 @@ def test_valid_model_yaml(self) -> None: yaml_content = dedent(""" model: id: "org/test-model" - testing: - hardware: - cpu: - cores: 4 - commands: - - "pytest tests/" """) data = yaml.safe_load(yaml_content) model_yaml = ModelYAML(**data) @@ -317,20 +164,12 @@ def test_model_yaml_with_vllm(self) -> None: plugins: io_processors: - "processor1" - testing: - hardware: - cpu: - cores: 4 - commands: - - "pytest tests/" - vllm: - commands: - - "pytest tests/test_vllm.py" """) data = yaml.safe_load(yaml_content) model_yaml = ModelYAML(**data) assert model_yaml.model.vllm is not None assert model_yaml.model.vllm.plugins is not None + assert model_yaml.model.vllm.plugins.io_processors == ["processor1"] class TestNexusYAML: @@ -341,37 +180,17 @@ def test_valid_nexus_yaml(self) -> None: yaml_content = dedent(""" package: name: "test-package" - - models: - - test-model """) data = yaml.safe_load(yaml_content) nexus_yaml = NexusYAML(**data) assert nexus_yaml.package.name == "test-package" - assert nexus_yaml.models == ["test-model"] - - def test_nexus_yaml_with_multiple_models(self) -> None: - """Test nexus.yaml with multiple models.""" - yaml_content = dedent(""" - package: - name: "test-package" - - models: - - model-1 - - model-2 - - model-3 - """) - data = yaml.safe_load(yaml_content) - nexus_yaml = NexusYAML(**data) - assert nexus_yaml.models is not None - assert len(nexus_yaml.models) == 3 - def test_nexus_yaml_without_models(self) -> None: - """Test nexus.yaml without models list.""" + def test_nexus_yaml_minimal(self) -> None: + """Test minimal nexus.yaml structure.""" yaml_content = dedent(""" package: - name: "test-package" + name: "minimal-package" """) data = yaml.safe_load(yaml_content) nexus_yaml = NexusYAML(**data) - assert nexus_yaml.models is None or nexus_yaml.models == [] + assert nexus_yaml.package.name == "minimal-package" From cd19464d3a7abd252dba9eb5fc070fbbb6f49b8f Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Tue, 28 Apr 2026 09:30:13 +0100 Subject: [PATCH 3/9] feat(cli): Updated the pyproject to install cli and test dependencies Signed-off-by: Christian Pinto --- pyproject.toml | 13 ++++++++++++ uv.lock | 56 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d290717..ec826d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,19 @@ Homepage = "https://github.com/IBM/algorithm-nexus" Issues = "https://github.com/IBM/algorithm-nexus/issues" Repository = "https://github.com/IBM/algorithm-nexus" +[project.scripts] +algorithm-nexus = "algorithm_nexus.cli:main" + [project.optional-dependencies] candidate = [ "vllm==0.19.1", ] +cli = [ + "pydantic>=2.13.3", + "pyyaml>=6.0.3", + "rich>=15.0.0", + "typer>=0.25.0", +] product = [ "vllm==0.18.0", ] @@ -29,8 +38,12 @@ dev = [ "pre-commit>=4.2.0", "ruff>=0.12.0", ] +test = [ + "pytest>=9.0.3", +] [tool.uv] +package = true conflicts = [ [ { extra = "ecosystem" }, diff --git a/uv.lock b/uv.lock index e12fea9..f4aefa4 100644 --- a/uv.lock +++ b/uv.lock @@ -130,12 +130,18 @@ wheels = [ [[package]] name = "algorithm-nexus" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } [package.optional-dependencies] candidate = [ { name = "vllm", version = "0.19.1", source = { registry = "https://pypi.org/simple" } }, ] +cli = [ + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "typer" }, +] product = [ { name = "vllm", version = "0.18.0", source = { registry = "https://pypi.org/simple" } }, ] @@ -148,13 +154,20 @@ dev = [ { name = "pre-commit" }, { name = "ruff" }, ] +test = [ + { name = "pytest" }, +] [package.metadata] requires-dist = [ + { name = "pydantic", marker = "extra == 'cli'", specifier = ">=2.13.3" }, + { name = "pyyaml", marker = "extra == 'cli'", specifier = ">=6.0.3" }, + { name = "rich", marker = "extra == 'cli'", specifier = ">=15.0.0" }, + { name = "typer", marker = "extra == 'cli'", specifier = ">=0.25.0" }, { name = "vllm", marker = "extra == 'candidate'", specifier = "==0.19.1" }, { name = "vllm", marker = "extra == 'product'", specifier = "==0.18.0" }, ] -provides-extras = ["candidate", "product"] +provides-extras = ["candidate", "product", "cli"] [package.metadata.requires-dev] dev = [ @@ -164,6 +177,7 @@ dev = [ { name = "pre-commit", specifier = ">=4.2.0" }, { name = "ruff", specifier = ">=0.12.0" }, ] +test = [{ name = "pytest", specifier = ">=9.0.3" }] [[package]] name = "annotated-doc" @@ -899,7 +913,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (python_full_version < '3.13' and extra == 'extra-15-algorithm-nexus-ecosystem') or (python_full_version < '3.13' and extra != 'extra-15-algorithm-nexus-candidate' and extra != 'extra-15-algorithm-nexus-product') or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-ecosystem') or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-product') or (extra == 'extra-15-algorithm-nexus-ecosystem' and extra == 'extra-15-algorithm-nexus-product')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1500,6 +1514,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "interegular" version = "0.3.3" @@ -2734,6 +2757,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "pre-commit" version = "4.5.1" @@ -3214,6 +3246,24 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-ecosystem') or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-product') or (extra == 'extra-15-algorithm-nexus-ecosystem' and extra == 'extra-15-algorithm-nexus-product')" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-ecosystem') or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-product') or (extra == 'extra-15-algorithm-nexus-ecosystem' and extra == 'extra-15-algorithm-nexus-product')" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-ecosystem') or (extra == 'extra-15-algorithm-nexus-candidate' and extra == 'extra-15-algorithm-nexus-product') or (extra == 'extra-15-algorithm-nexus-ecosystem' and extra == 'extra-15-algorithm-nexus-product')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From 8af4686abc7d60153e3a90304bf3887b9c1b7da3 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Tue, 28 Apr 2026 11:25:55 +0100 Subject: [PATCH 4/9] feat(cli): code changess to address review Co-authored-by: Alessandro Pomponio <10339005+AlessandroPomponio@users.noreply.github.com> Signed-off-by: Christian Pinto --- src/algorithm_nexus/cli.py | 31 ++++++--------- src/algorithm_nexus/models.py | 66 +++++++++++++++++++------------ src/algorithm_nexus/validation.py | 44 ++++++++++----------- 3 files changed, 74 insertions(+), 67 deletions(-) diff --git a/src/algorithm_nexus/cli.py b/src/algorithm_nexus/cli.py index f83e067..bc6e2a6 100644 --- a/src/algorithm_nexus/cli.py +++ b/src/algorithm_nexus/cli.py @@ -7,6 +7,7 @@ import sys from pathlib import Path +from typing import Annotated try: import typer @@ -43,28 +44,20 @@ def main_callback(ctx: typer.Context) -> None: @app.command(name="validate") def validate( - package_path: Path = typer.Argument( - ..., - help="Path to a Nexus package directory.", - ), + package_path: Annotated[ + Path, + typer.Argument( + help="Path to a Nexus package directory.", + dir_okay=True, + file_okay=False, + readable=True, + resolve_path=True, + ), + ], ) -> None: """Validate Nexus package structure and YAML configuration files.""" collector = ValidationErrorCollector() - - resolved_path = package_path.resolve() - - if not resolved_path.exists(): - collector.add(f"Package path does not exist: {resolved_path}") - console.print( - Panel( - str(collector), - title="[bold red]Validation Failed[/bold red]", - border_style="red", - ) - ) - raise typer.Exit(code=1) - - validate_package_directory(resolved_path, collector) + validate_package_directory(package_path, collector) if collector.has_errors: console.print( diff --git a/src/algorithm_nexus/models.py b/src/algorithm_nexus/models.py index d84882e..6889fee 100644 --- a/src/algorithm_nexus/models.py +++ b/src/algorithm_nexus/models.py @@ -5,7 +5,7 @@ from __future__ import annotations -from typing import Literal +from typing import Annotated, Literal from pydantic import BaseModel, Field @@ -15,7 +15,7 @@ class PackageConfig(BaseModel): model_config = {"extra": "forbid"} - name: str = Field(..., min_length=1, description="Python package name") + name: Annotated[str, Field(min_length=1, description="Python package name")] class VLLMPlugins(BaseModel): @@ -23,12 +23,17 @@ class VLLMPlugins(BaseModel): model_config = {"extra": "forbid"} - general: str | None = Field( - None, description="General vLLM plugin that loads the model class" - ) - io_processors: list[str] | None = Field( - None, description="List of vLLM IO processor plugins supported by this model" - ) + general: Annotated[ + str | None, + Field(None, description="General vLLM plugin that loads the model class"), + ] = None + io_processors: Annotated[ + list[str] | None, + Field( + None, + description="List of vLLM IO processor plugins supported by this model", + ), + ] = None class VLLMConfig(BaseModel): @@ -40,10 +45,14 @@ class VLLMConfig(BaseModel): model_config = {"extra": "forbid"} - enabled: Literal[True] = Field( - ..., description="Whether vLLM serving is enabled for this model" - ) - plugins: VLLMPlugins | None = Field(None, description="vLLM plugins configuration") + enabled: Annotated[ + Literal[True], + Field(description="Whether vLLM serving is enabled for this model"), + ] + plugins: Annotated[ + VLLMPlugins | None, + Field(None, description="vLLM plugins configuration"), + ] = None class ModelConfig(BaseModel): @@ -51,17 +60,24 @@ class ModelConfig(BaseModel): model_config = {"extra": "forbid"} - id: str = Field( - ..., min_length=1, description="Hugging Face model repository identifier" - ) - owner: str | None = Field( - None, - description="Model owner GitHub identifier. If omitted, ownership defaults to the Nexus package owner.", - ) - vllm: VLLMConfig | None = Field( - None, - description="vLLM serving configuration. Only required for models that need additional vLLM plugins and belong to a Nexus Package targeting the product or candidate distribution variants.", - ) + id: Annotated[ + str, + Field(min_length=1, description="Hugging Face model repository identifier"), + ] + owner: Annotated[ + str | None, + Field( + None, + description="Model owner GitHub identifier. If omitted, ownership defaults to the Nexus package owner.", + ), + ] = None + vllm: Annotated[ + VLLMConfig | None, + Field( + None, + description="vLLM serving configuration. Only required for models that need additional vLLM plugins and belong to a Nexus Package targeting the product or candidate distribution variants.", + ), + ] = None class ModelYAML(BaseModel): @@ -69,7 +85,7 @@ class ModelYAML(BaseModel): model_config = {"extra": "forbid"} - model: ModelConfig = Field(..., description="Model configuration") + model: Annotated[ModelConfig, Field(description="Model configuration")] class NexusYAML(BaseModel): @@ -77,4 +93,4 @@ class NexusYAML(BaseModel): model_config = {"extra": "forbid"} - package: PackageConfig = Field(..., description="Package-level configuration") + package: Annotated[PackageConfig, Field(description="Package-level configuration")] diff --git a/src/algorithm_nexus/validation.py b/src/algorithm_nexus/validation.py index a3e674d..91247af 100644 --- a/src/algorithm_nexus/validation.py +++ b/src/algorithm_nexus/validation.py @@ -53,7 +53,7 @@ def format_info(self) -> str: return "\n".join(f"[yellow]i[/yellow] {msg}" for msg in self.info) def __str__(self) -> str: - """Format errors as a bulleted list (for backward compatibility).""" + """Format errors as a bulleted list.""" return self.format_errors() @@ -65,12 +65,12 @@ def load_yaml_file( Returns a dict if successful, None otherwise. Validates that the YAML contains a mapping (dict) at the top level. """ - try: - with path.open("r", encoding="utf-8") as handle: - data = yaml.safe_load(handle) - except FileNotFoundError: + if not path.is_file(): collector.add(f"Missing YAML file: {path}") return None + + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) except yaml.YAMLError as exc: collector.add(f"Invalid YAML syntax in {path}: {exc}") return None @@ -144,12 +144,13 @@ def validate_optional_file( path: Path, collector: ValidationErrorCollector, context: str ) -> bool: """Validate optional file and add info if missing. Returns True if valid or doesn't exist.""" - if path.exists(): - if not path.is_file(): - collector.add(f"{context} must be a file when present: {path}") - return False - else: + if not path.exists(): collector.add_info(f"{context}") + return False + elif not path.is_file(): + collector.add(f"{context} must be a file when present: {path}") + return False + return True @@ -157,12 +158,13 @@ def validate_optional_dir( path: Path, collector: ValidationErrorCollector, context: str ) -> bool: """Validate optional directory and add info if missing. Returns True if valid or doesn't exist.""" - if path.exists(): - if not path.is_dir(): - collector.add(f"{context} must be a directory when present: {path}") - return False - else: + if not path.exists(): collector.add_info(f"{context}") + return False + elif not path.is_dir(): + collector.add(f"{context} must be a directory when present: {path}") + return False + return True @@ -205,15 +207,11 @@ def validate_package_directory( # Check if models directory exists models_dir = package_dir / "models" - if not models_dir.exists(): - # No models directory is valid - package may not have models - collector.add_info("Optional package directory missing: models") - return - - if not models_dir.is_dir(): - collector.add(f"models/ must be a directory when present: {models_dir}") + if not validate_optional_dir( + models_dir, collector, "Optional package directory missing: models" + ): return - # Validate each model directory found + # Only validate model directories if models_dir exists for model_dir in models_dir.iterdir(): validate_model_directory(model_dir, collector) From 1ef926b5d146de2754eab285c20866cfce6bedf3 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Tue, 28 Apr 2026 11:30:31 +0100 Subject: [PATCH 5/9] feat(cli): One last change after the review Signed-off-by: Christian Pinto --- src/algorithm_nexus/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/algorithm_nexus/cli.py b/src/algorithm_nexus/cli.py index bc6e2a6..7d3f82a 100644 --- a/src/algorithm_nexus/cli.py +++ b/src/algorithm_nexus/cli.py @@ -31,6 +31,7 @@ app = typer.Typer( help="Algorithm Nexus CLI - Tools for managing and validating Nexus packages.", add_completion=False, + no_args_is_help=True, ) From 5d7aed692f26e4cf10bd80ba3f75cb497d1a5c38 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Tue, 28 Apr 2026 13:49:02 +0100 Subject: [PATCH 6/9] feat(cli): Second review round Signed-off-by: Christian Pinto --- README.md | 6 +- pyproject.toml | 2 +- src/algorithm_nexus/cli.py | 5 +- src/algorithm_nexus/models.py | 29 +++++---- templates/nexus-package-template/README.md | 62 ++++++++---------- tests/fixtures/packages/README.md | 74 ---------------------- tests/test_models.py | 31 +++++++++ 7 files changed, 80 insertions(+), 129 deletions(-) delete mode 100644 tests/fixtures/packages/README.md diff --git a/README.md b/README.md index 4717a55..7667c56 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ of multiple models with different dependencies. ## Algorithm Nexus CLI -Algorithm Nexus provides the `algorithm-nexus` CLI tool for managing Nexus -packages. This tool allows the validation of the structure of a Nexus Package. +Algorithm Nexus provides the `an` CLI tool for managing Nexus packages. This +tool allows the validation of the structure of a Nexus Package. ### Installation @@ -79,7 +79,7 @@ The validation tool checks: Example usage: ```bash -algorithm-nexus validate /path/to/package +an validate /path/to/package ``` In case of validation errors a detailed report guides the user to fix the diff --git a/pyproject.toml b/pyproject.toml index ec826d9..b9e6c72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ Issues = "https://github.com/IBM/algorithm-nexus/issues" Repository = "https://github.com/IBM/algorithm-nexus" [project.scripts] -algorithm-nexus = "algorithm_nexus.cli:main" +an = "algorithm_nexus.cli:main" [project.optional-dependencies] candidate = [ diff --git a/src/algorithm_nexus/cli.py b/src/algorithm_nexus/cli.py index 7d3f82a..c2b4328 100644 --- a/src/algorithm_nexus/cli.py +++ b/src/algorithm_nexus/cli.py @@ -37,10 +37,7 @@ @app.callback(invoke_without_command=True) def main_callback(ctx: typer.Context) -> None: - """Algorithm Nexus CLI - Tools for managing and validating Nexus packages.""" - if ctx.invoked_subcommand is None: - console.print(ctx.get_help()) - raise typer.Exit() + pass @app.command(name="validate") diff --git a/src/algorithm_nexus/models.py b/src/algorithm_nexus/models.py index 6889fee..adebfba 100644 --- a/src/algorithm_nexus/models.py +++ b/src/algorithm_nexus/models.py @@ -7,13 +7,13 @@ from typing import Annotated, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class PackageConfig(BaseModel): """Package-level configuration.""" - model_config = {"extra": "forbid"} + model_config = ConfigDict(extra="forbid") name: Annotated[str, Field(min_length=1, description="Python package name")] @@ -21,16 +21,15 @@ class PackageConfig(BaseModel): class VLLMPlugins(BaseModel): """vLLM plugins configuration.""" - model_config = {"extra": "forbid"} + model_config = ConfigDict(extra="forbid") general: Annotated[ str | None, - Field(None, description="General vLLM plugin that loads the model class"), + Field(description="General vLLM plugin that loads the model class"), ] = None io_processors: Annotated[ list[str] | None, Field( - None, description="List of vLLM IO processor plugins supported by this model", ), ] = None @@ -43,7 +42,7 @@ class VLLMConfig(BaseModel): and belong to a Nexus Package targeting the product or candidate distribution variants. """ - model_config = {"extra": "forbid"} + model_config = ConfigDict(extra="forbid") enabled: Annotated[ Literal[True], @@ -51,14 +50,14 @@ class VLLMConfig(BaseModel): ] plugins: Annotated[ VLLMPlugins | None, - Field(None, description="vLLM plugins configuration"), + Field(description="vLLM plugins configuration"), ] = None class ModelConfig(BaseModel): """Model-level configuration.""" - model_config = {"extra": "forbid"} + model_config = ConfigDict(extra="forbid") id: Annotated[ str, @@ -67,14 +66,20 @@ class ModelConfig(BaseModel): owner: Annotated[ str | None, Field( - None, + # Validats the owner field against the GitHub username rules: + # https://docs.github.com/en/enterprise-cloud@latest/admin/managing-iam/iam-configuration-reference/username-considerations-for-external-authentication + # - Only contains dashes and alphanumeric characters + # - Does not start or end with a dash + # - Does not contain consecutive dashes + # - Has a maximum length of 39 characters + pattern=r"^[a-zA-Z0-9]([a-zA-Z0-9]|-[a-zA-Z0-9]){0,38}$", description="Model owner GitHub identifier. If omitted, ownership defaults to the Nexus package owner.", ), ] = None + vllm: Annotated[ VLLMConfig | None, Field( - None, description="vLLM serving configuration. Only required for models that need additional vLLM plugins and belong to a Nexus Package targeting the product or candidate distribution variants.", ), ] = None @@ -83,7 +88,7 @@ class ModelConfig(BaseModel): class ModelYAML(BaseModel): """Root model.yaml structure.""" - model_config = {"extra": "forbid"} + model_config = ConfigDict(extra="forbid") model: Annotated[ModelConfig, Field(description="Model configuration")] @@ -91,6 +96,6 @@ class ModelYAML(BaseModel): class NexusYAML(BaseModel): """Root nexus.yaml structure.""" - model_config = {"extra": "forbid"} + model_config = ConfigDict(extra="forbid") package: Annotated[PackageConfig, Field(description="Package-level configuration")] diff --git a/templates/nexus-package-template/README.md b/templates/nexus-package-template/README.md index 66b4a08..2ec148f 100644 --- a/templates/nexus-package-template/README.md +++ b/templates/nexus-package-template/README.md @@ -19,60 +19,52 @@ below to customize it for your model. ``` 3. **Update `nexus.yaml`**: - - Replace `your-package-name` with your package name - - Replace `your-model-name` with your model directory name - - Update version and agent_skills as needed + - Replace `your-package-name` with your Python package name 4. **Update `models/your-model-name/model.yaml`**: - - Replace `your-org/your-model-name` with your model ID - - Replace `your-gh-id` with the GitHub ID of the model owner if explicitly - set. - - Configure hardware requirements - - Add vLLM configuration if needed (or remove the vllm section) - - Update test commands + - Replace `your-org/your-model-name` with your Hugging Face model repository + ID + - Optionally set `owner` with the GitHub username of the model owner + - Add vLLM configuration if needed (or remove the commented vllm section) -5. **Implement tests**: - - Add actual inference tests in `tests/test_inference.py` - - Add vLLM tests in `tests/test_vllm.py` (if using vLLM) +5. **Add optional documentation**: + - Create `models/your-model-name/usage.md` with usage examples (optional) 6. **Validate your package**: ```bash - algorithm-nexus validate /path/to/your-package + an validate /path/to/your-package ``` ## Package Structure ```text your-package/ -├── nexus.yaml # Package configuration -├── models/ -│ └── your-model-name/ -│ ├── model.yaml # Model configuration -│ └── tests/ -│ ├── test_inference.py # Inference tests -│ └── test_vllm.py # vLLM tests (if applicable) -└── README.md +├── nexus.yaml # Required: Package configuration +├── skills/ # Optional: Agent skills resources +└── models/ + └── your-model-name/ + ├── model.yaml # Required: Model configuration + └── usage.md # Optional: Usage documentation ``` ## Configuration Guidelines -### vLLM Configuration +### nexus.yaml -- If your model uses vLLM, set `vllm.enabled: true` and provide vLLM testing -- If not using vLLM, remove the entire `vllm` section from `model.yaml` -- The `enabled` field must be `true` if the vllm section is present +- `package.name`: Must match your Python package name (e.g., "terratorch") -### Testing Requirements +### model.yaml -- All models must have a `tests/` directory with at least one test file -- Test commands must be specified in `model.testing.commands` -- If `vllm.enabled: true`, you must provide `model.testing.vllm.commands` - -### Hardware Requirements - -- Specify CPU requirements (cores and RAM) -- Optionally specify GPU requirements (type, count, cpu_fallback) +- `model.id`: Hugging Face model repository identifier (e.g., + "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL") +- `model.owner`: (Optional) GitHub username of the model owner. If omitted, + defaults to the Nexus package owner +- `model.vllm`: (Optional) Only include if your model requires additional vLLM + plugins for the candidate or product variants + - `enabled`: Must be `true` if the vllm section is present + - `plugins.general`: (Optional) General vLLM plugin that loads the model class + - `plugins.io_processors`: (Optional) List of vLLM IO processor plugins ## Documentation @@ -86,7 +78,7 @@ For detailed documentation on Nexus package requirements, see: Before submitting your package, ensure it passes validation: ```bash -algorithm-nexus validate /path/to/your-package +an validate /path/to/your-package ``` The validator checks: diff --git a/tests/fixtures/packages/README.md b/tests/fixtures/packages/README.md deleted file mode 100644 index f3120b9..0000000 --- a/tests/fixtures/packages/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Test Package Fixtures - -This directory contains sample Nexus packages for testing validation. - -## Valid Package (`valid-package/`) - -A complete, valid Nexus package that passes all validation checks. - -**Structure:** - -- ✅ Valid `nexus.yaml` with package metadata -- ✅ Optional `AGENTS.md` for embedded agent skills -- ✅ One declared model: `example-model` -- ✅ Complete model configuration with vLLM support -- ✅ Required `usage.md` documentation -- ✅ Required `tests/` directory with test files -- ✅ Optional `benchmarks/` directory with custom benchmark - -**Usage:** - -```bash -algorithm-nexus validate tests/fixtures/packages/valid-package -``` - -Expected result: ✅ Validation successful - ---- - -## Invalid Package (`invalid-package/`) - -A Nexus package with multiple validation errors for testing error detection. - -**Issues:** - -1. ❌ **Missing model ID**: `broken-model/model.yaml` is missing the required - `id` field -2. ❌ **Missing vLLM testing**: `broken-model` defines `vllm` but doesn't - provide `testing.vllm` -3. ❌ **Missing tests directory**: `broken-model` doesn't have required tests - directory -4. ❌ **Undeclared model**: `undeclared-model` exists but is not declared in - `nexus.yaml` - -**Usage:** - -```bash -algorithm-nexus validate tests/fixtures/packages/invalid-package -``` - -Expected result: ❌ Validation failed with multiple errors - ---- - -## Testing with Fixtures - -These fixtures can be used in automated tests: - -```python -from pathlib import Path -from algorithm_nexus.cli import _validate_package_directory, ValidationErrorCollector - -def test_valid_package(): - collector = ValidationErrorCollector() - package_dir = Path("tests/fixtures/packages/valid-package") - _validate_package_directory(package_dir, collector) - assert not collector.has_errors - -def test_invalid_package(): - collector = ValidationErrorCollector() - package_dir = Path("tests/fixtures/packages/invalid-package") - _validate_package_directory(package_dir, collector) - assert collector.has_errors - assert len(collector.errors) >= 5 # Multiple errors expected -``` diff --git a/tests/test_models.py b/tests/test_models.py index d53dd41..c8d27a7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -114,6 +114,37 @@ def test_model_with_owner(self) -> None: assert config.id == "org/model" assert config.owner == "github-username" + def test_model_with_invalid_owner_id(self) -> None: + """Test model configuration with owner.""" + data = { + "id": "org/model", + } + + # starts with a dash + data["owner"] = "-github-username" + with pytest.raises(ValidationError): + ModelConfig(**data) + + # ends with a dash + data["owner"] = "github-username-" + with pytest.raises(ValidationError): + ModelConfig(**data) + + # scontaines consecutive dashes + data["owner"] = "github--username" + with pytest.raises(ValidationError): + ModelConfig(**data) + + # contains an illegal character + data["owner"] = "github-usern@me" + with pytest.raises(ValidationError): + ModelConfig(**data) + + # longer than 39 characters + data["owner"] = "ThisGitHubUsernameIsDefinitelyTooLongToBeValid" + with pytest.raises(ValidationError): + ModelConfig(**data) + def test_vllm_disabled_fails(self) -> None: """Test that vLLM enabled=False is rejected.""" data = { From b222473087e24fcdee2e01373c72b8d921ac7688 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Tue, 28 Apr 2026 14:57:17 +0100 Subject: [PATCH 7/9] docs(PR): Added instructions for contributing a Nexus package Signed-off-by: Christian Pinto --- .../new_nexus_package.md | 64 ++++ README.md | 8 +- docs/contributing/add_new_nexus_package.md | 279 ++++++++++++++++++ .../{contributing => design}/nexus_package.md | 14 +- 4 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE/new_nexus_package.md create mode 100644 docs/contributing/add_new_nexus_package.md rename docs/{contributing => design}/nexus_package.md (91%) diff --git a/.github/PULL_REQUEST_TEMPLATE/new_nexus_package.md b/.github/PULL_REQUEST_TEMPLATE/new_nexus_package.md new file mode 100644 index 0000000..ea85fe4 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/new_nexus_package.md @@ -0,0 +1,64 @@ +--- +name: New Nexus Package +about: Submit a new Nexus package to Algorithm Nexus +title: "feat(package): Add [package-name] Nexus package" +labels: "nexus-package" +--- + +## Package Information + +**Package Name:** + +**Python Package Source:** + +**Brief Description of the Package**: + + + +**Distribution Variant(s):** + +- [ ] Ecosystem (no vLLM dependency) +- [ ] Candidate (with vLLM dependency) +- [ ] Product (to be added by maintainers) + +## Models Included + + + +| Model Name | Hugging Face ID | Requires vLLM | Owner | +| ---------------------- | --------------------------------------------------------- | --------------- | ----------------------------------------------------- | +| | | | | + +## Checklist + +Please ensure you have completed all the following steps before submitting this +PR: + +### Step 1: Create Your Nexus Package + +- [ ] Copied the Nexus package template to `packages//` +- [ ] Updated the nexus package folder with your models and configuration + +### Step 2: Validate Your Nexus Package + +- [ ] Ran `uv run an validate packages/` successfully + +### Step 3: Add Your Package to Algorithm Nexus + +- [ ] Added package to appropriate variant(s) using `uv add` +- [ ] Verified lockfile updated with `uv lock --check` + +## Additional Information + + + +## I need help with this PR + + + +## Related Issues + + diff --git a/README.md b/README.md index 7667c56..8abfff7 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,12 @@ model. ## Contributing This project is currently in closed beta. We are not accepting external -contributions at this time. For IBM contributors, please see our -[Contributing Guide](CONTRIBUTING.md) for development setup and guidelines. +contributions at this time. For IBM contributors: + +- Please, see our [Contributing Guide](CONTRIBUTING.md) for development setup + and guidelines. +- Read the [guide](./docs/contributing/add_new_nexus_package.md) for + step-by-step instructions for contributing a Nexus Package. ## License diff --git a/docs/contributing/add_new_nexus_package.md b/docs/contributing/add_new_nexus_package.md new file mode 100644 index 0000000..011597d --- /dev/null +++ b/docs/contributing/add_new_nexus_package.md @@ -0,0 +1,279 @@ + + +# Contributing a Nexus Package to Algorithm Nexus + +This guide walks you through the complete process of contributing a Nexus +package to Algorithm Nexus, from creating your package to opening a pull +request. + +## Prerequisites + +Before you begin, ensure you have: + +- A Python package published on GitHub or PyPI +- Models that your package supports +- `uv` installed on your system +- A + [fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo) + of the Algorithm Nexus repository checked out locally + +## Step 1: Create Your Nexus Package + +A Nexus package is a metadata and configuration package that references your +Python package and defines the models it supports. + +### 1.1 Use the Template + +Start by copying the Nexus package template: + +```bash +cp -r templates/nexus-package-template packages/ +``` + +Replace `` with your actual package name (e.g., +`terratorch`). + +### 1.2 Configure `nexus.yaml` + +Edit `packages//nexus.yaml` to declare your package metadata: + +```yaml +package: + name: your-package-name # Must match your Python package name +``` + +See the +[Nexus Package Configuration](nexus_package.md#31-nexus-package-configuration-nexusyaml) +for detailed configuration options. + +### 1.3 Configure Model Metadata + +For each model your package supports: + +1. Create a directory under `packages//models//` +1. Create a `model.yaml` file with the model configuration: + + ```yaml + model: + id: organization/model-name # Hugging Face model repository identifier + owner: github-username # Optional: defaults to package owner + vllm: # Optional: required only if the model is served with vLLM + enabled: true + plugins: # Optional: vLLM plugins required for serving your model + general: your_package.vllm_plugin # Optional: vLLM general plugin required for loading your model in vLLM + io_processors: # Optional: list of IO Processor plugins that can be used with your model + - my_io_processor + ``` + +1. Optionally add a `usage.md` file with usage documentation + +See the +[Model Configuration](nexus_package.md#32-model-configuration-modelsmodel-namemodelyaml) +for complete model configuration options. + +## Step 2: Validate Your Nexus Package + +Before submitting, validate your package structure and configuration: + +```bash +# From the repository root +uv run an validate packages/ +``` + +This command checks: + +- Required files exist (`nexus.yaml`, `model.yaml` for each model) +- YAML syntax is valid +- Configuration follows the required schema +- All declared models have corresponding directories + +Fix any validation errors before proceeding. + +## Step 3: Add Your Package to Algorithm Nexus + +Your Python package must be added to Algorithm Nexus as a dependency using `uv`. +The exact command depends on your package's relationship with `vllm`. + +Follow the instructions in the sections below to make sure your Python package +is added properly to the Algorithm Nexus dependencies. Read the +[Algorithm Nexus Dependency Resolution Process documentation](../design/dependency-resolution.md) +for full details. + +### 3.1 Classify Your Package + +Determine which category your package falls into: + +#### Ecosystem-Only Packages + +Your package is **ecosystem-only** if it: + +- Does **not** declare `vllm` as a default dependency +- Does **not** declare `vllm` as an optional dependency + +**Add to ecosystem variant only:** + +```bash +uv add --optional ecosystem +``` + +#### vLLM-Dependent Packages + +Your package is **vllm-dependent** if it: + +- Declares `vllm` as a **default (mandatory) dependency** + +**Add to candidate variant:** + +```bash +uv add --optional candidate +``` + +> [!NOTE] Do not add packages to the `product` variant. Algorithm Nexus +> maintainers will handle this once product requirements are met. + +#### vLLM-Agnostic Packages + +Your package is **vllm-agnostic** if it: + +- Does **not** declare `vllm` as a default dependency +- Declares `vllm` as an **optional dependency** (via extras) + +**Add to ecosystem variant (without vllm):** + +```bash +uv add --optional ecosystem +``` + +**Add to candidate variant (with vllm):** + +```bash +uv add [] --optional candidate +``` + +Replace `` with the extra name that enables vllm in your package. + +### 3.2 Git-Based Packages + +If your package is not yet published on PyPI, you can add it from a Git +repository: + +```bash +uv add git+https://github.com// --optional +``` + +> [!IMPORTANT] SSH-based cloning is not supported. All Git dependencies must be +> publicly accessible via HTTPS. + +### 3.3 Verify Dependency Resolution + +After adding your package, verify the lockfile is updated: + +```bash +uv lock --check +``` + +For detailed information about dependency resolution, see the +[Dependency Resolution Design Document](../design/dependency-resolution.md). + +## Step 4: Commit Your Changes + +Commit your changes with a descriptive message: + +```bash +git add packages// +git add pyproject.toml uv.lock requirements-*.txt +git commit -s -m "feat(package): Add Nexus package + +- Add Nexus package metadata and model configurations +- Add dependency to variant(s) +- Update lockfile and requirements exports" +``` + +> [!IMPORTANT] The `-s` flag adds a Signed-off-by line, which is required for +> all contributions. This indicates you accept the +> [Developer's Certificate of Origin](../../CONTRIBUTING.md#legal). Also, make +> sure you have installed the pre-commit hooks before committing your changes. +> This will reduce the likelihood for your contribution to fail the CI workflow. + +## Step 5: Open a Pull Request + +1. Push your changes to your fork: + + ```bash + git push origin + ``` + +2. Open a pull request on GitHub using the `New Nexus Package` template, and + fill all the required information. + +3. Wait for CI checks to complete: + - Lockfile consistency checks + - Requirements export validation + - Package availability verification + - Variant-specific dependency resolution checks + +## Step 6: Address Review Feedback + +Maintainers will review your PR and may request changes: + +- Respond to comments and questions +- Push updates to your branch (the PR will update automatically) +- Re-request review once changes are complete + +## Common Issues and Solutions + +### Validation Errors + +**Issue**: `an validate` reports errors + +**Solution**: Carefully read the error messages and fix the issues in your YAML +files. Common problems include: + +- Missing required fields +- Invalid YAML syntax +- Incorrect file paths +- Model directories without `model.yaml` + +### Dependency Conflicts + +**Issue**: `uv lock` fails with dependency conflicts + +**Solution**: Check if your package has conflicting dependencies with existing +packages. You may need to: + +- Update your package's dependency constraints +- Coordinate with maintainers about resolving conflicts + +If you are unable to resolve the dependency conflicts when adding your package, +you can still create the PR in draft state, and explicitly state that the +package cannot be added due to dependency conflicts. The maintainers will help +you to solve the conflicts, which might require also coordinating with other +Nexus package owners. + +### CI Failures + +**Issue**: CI checks fail after opening PR + +**Solution**: Carefully read the logs and try to identify the root cause. If +unable to solve the issue, send a message in the PR to request the maintainers +help. + +## Getting Help + +If you encounter issues: + +1. Check the [Nexus Package Structure Guide](nexus_package.md) +2. Review the + [Dependency Resolution Design Document](../design/dependency-resolution.md) +3. Search existing issues on GitHub +4. Open a new issue with details about your problem + +## Additional Resources + +- [Nexus Package Requirements](../requirements/nexus_package.md) +- [Packaging and Dependency Requirements](../requirements/packaging_and_dependency_reqs.md) +- [Contributing Guidelines](../../CONTRIBUTING.md) +- [Maintainers](../../MAINTAINERS.md) diff --git a/docs/contributing/nexus_package.md b/docs/design/nexus_package.md similarity index 91% rename from docs/contributing/nexus_package.md rename to docs/design/nexus_package.md index c1bda67..ef1c637 100644 --- a/docs/contributing/nexus_package.md +++ b/docs/design/nexus_package.md @@ -45,13 +45,13 @@ packages/ ``` The required root file is `nexus.yaml`, which declares the Nexus package -metadata. `skills` is optional and -should only be included when the package provides agent skills to assist users -in using the package. The `models/` folder is required whenever a Nexus package -wants to advertise one or more models, with one sub-folder for each model. Each -model folder must contain a `model.yaml` file describing the model metadata and -optional vLLM integration. Each model folder can optionally include a `usage.md` -file to provide users with model-specific usage guidance. +metadata. `skills` is optional and should only be included when the package +provides agent skills to assist users in using the package. The `models/` folder +is required whenever a Nexus package wants to advertise one or more models, with +one sub-folder for each model. Each model folder must contain a `model.yaml` +file describing the model metadata and optional vLLM integration. Each model +folder can optionally include a `usage.md` file to provide users with +model-specific usage guidance. --- From c0e2f870723e96e703d654088baaba691504d607 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Wed, 29 Apr 2026 09:04:27 +0100 Subject: [PATCH 8/9] feat(skills): First draft of the PR creation skill Signed-off-by: Christian Pinto --- .agents/skills/add-nexus-package/SKILL.md | 172 ++++++++++++++++++ .../assets/templates/new_nexus_package_pr.md | 1 + .../assets/templates/nexus-package-template | 1 + .bob | 1 + 4 files changed, 175 insertions(+) create mode 100644 .agents/skills/add-nexus-package/SKILL.md create mode 120000 .agents/skills/add-nexus-package/assets/templates/new_nexus_package_pr.md create mode 120000 .agents/skills/add-nexus-package/assets/templates/nexus-package-template create mode 120000 .bob diff --git a/.agents/skills/add-nexus-package/SKILL.md b/.agents/skills/add-nexus-package/SKILL.md new file mode 100644 index 0000000..97333e0 --- /dev/null +++ b/.agents/skills/add-nexus-package/SKILL.md @@ -0,0 +1,172 @@ +--- +name: add-nexus-package +description: Step-by-step guidance for contributing a new Nexus package to Algorithm Nexus, from initial setup through pull request submission. It ensures all configuration files are properly created, validates the package structure, and helps classify and add the package as a dependency. Use when users want to create a new Nexus package and contribute it via a pull request. +--- + +# Add Nexus Package + +## Steps + +The skill guides users through these steps: + +1. **Identify package information**: Collect all the information on package and + models to identify package classification and models. +2. **Create New Branch**: Create a new branch for the package addition +3. **Create Package Structure**: Copy template and create package directory, + created thenecessary subfolders and config files. +4. **Validate Package**: Run `an validate` to check structure +5. **Add Dependency**: Use `uv add` with correct variant classification +6. **Commit Changes**: Commit with DCO sign-off +7. **Open Pull Request**: Use the New Nexus Package PR template + +## 1. Identify package information + +The main information required for the package are split between package level +and model level information. + +Package level information: + +- `package_name` (string, required): Name of the Python package to add (e.g., + "terratorch") + +Model level information: Each model object contains: + +- `name` (string, required): Model directory name +- `huggingface_id` (string, required): Hugging Face model repository identifier +- `owner` (string, optional): GitHub username of model owner +- `requires_vllm` (boolean, required): Whether the model uses vLLM for serving +- `vllm_plugins` (string, required): If the model uses vLLM it might also + require a set of vlLM plugins. + - `general` (string, optional): name of the general plugin required for + loading the model with vLLM + - `io_processors` (list of strings, optional): list of io processor plugins + that the model supports for pre/post processing with vLLM + +If one of the models requires vLLM, ask the user whether vLLM is a required +dependency or an optional one. + +Using the above information you can infer the classification of the package: + +- _Ecosystem-Only Packages_: Don't require vLLM neither as a default or optional + dependency. +- _vLLM-Dependent Packages_: Requires vLLM as a mandatory dependency +- _vLLM-Agnostic Packages_: Declares vLLM as an optional dependency but not as a + required one. + +Also, ask the user if they want to create usage documentation for any of their +models. If that's the cas help them draft it. + +## 2. Create New Branch + +Create a new git branch named `add--package` + +## 3. Create package structure + +Copy the package template to the final package destination + +```bash +cp -r templates/nexus-package-template packages/ +``` + +Then populate the package `nexus.yaml` with the package level information. For +each model, if any, create a sub folder with the model name and populate the +`model.yaml` file with the model level information. + +If the users have created usage documentation for their models it should be +placed inside each model subfolder in a file named `usage.md`. + +The template `assets/templates/nexus-package-template` provides exampls for the +package structure and config filaes (package `nexus.yaml` and model +`model.yaml`). + +After populating the package structure, remove any files that are part of +the template and that are not needed for this package +(e.g., the sample model folder). + +## 4. Validate Package + +```bash +uv run an validate packages/ +``` + +Notes: + +- Ask for user confirmation before running the validation command. + +### 4.1 Example Package Validation Outputs + +If the validation is successful a message will be printed to the console. + +```bash +╭───────────────────────────────────────────────────────────────────────────────────────────────────────── Validation Successful ─────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ ✓ All validation checks passed │ +│ │ +│ Optional files/directories: │ +│ i Optional package directory missing: skills │ +│ i Optional model file missing for 'minimal-model': usage.md │ +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +In case the validation is not successful, the tool will provide a list of issues +like in the snippet below + +```bash +╭─────────────────────────────────────────────────────────────────────────────────────────────────────────── Validation Failed ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ ✗ /Users/christian/workspace/algorithm-nexus/tests/fixtures/packages/invalid-package/nexus.yaml │ +│ Field: models │ +│ Error: Extra inputs are not permitted │ +│ ✗ /Users/christian/workspace/algorithm-nexus/tests/fixtures/packages/invalid-package/models/undeclared-model/model.yaml │ +│ Field: model.testing │ +│ Error: Extra inputs are not permitted │ +│ ✗ /Users/christian/workspace/algorithm-nexus/tests/fixtures/packages/invalid-package/models/broken-model/model.yaml │ +│ Field: model.id │ +│ Error: This required field is missing │ +│ ✗ /Users/christian/workspace/algorithm-nexus/tests/fixtures/packages/invalid-package/models/broken-model/model.yaml │ +│ Field: model.testing │ +│ Error: Extra inputs are not permitted │ +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +## 5. Add Dependency + +Depending on the package classification coming out of step 1, add the python +package to the dependencies with uv. + +The general command for adding a package to the dependencies is: + +```bash +uv add --optional +``` + +The variant depends to the package classification: + +- `ecosystem`: for Ecosystems-Only and vLLM Agnostic packages +- `candidate`: for vLLM-Dependent and vLLM Agnostic packages + +Notes: + +- Never add a package to the `product` variant +- For packages to be added to multiple variants, run the command once for each + variant separately. +- Do not run `uv add` in any different way other than the above example. +- Ask for user confirmation before running any command. + +In case of failure, help the user troubleshooting the error by interpreting the +output messages and providing potential solutions. Ask the user for confirmation +before making any change towards solving the dependency issue. + +## 6. Commit Changes\*\* + +Commit with DCO sign-off, this is achieved by adding the -s flag to the git +commit command. + +Example commit command: + +```bash +git commit -s -m "feat(package): Add Nexus package" +``` + +## 7. Open Pull Request + +Once all the above have been done, create the pull request text by following the +template in `./assets/templates/new_nexus_package_pr.md`. diff --git a/.agents/skills/add-nexus-package/assets/templates/new_nexus_package_pr.md b/.agents/skills/add-nexus-package/assets/templates/new_nexus_package_pr.md new file mode 120000 index 0000000..b11d027 --- /dev/null +++ b/.agents/skills/add-nexus-package/assets/templates/new_nexus_package_pr.md @@ -0,0 +1 @@ +../../../../../.github/PULL_REQUEST_TEMPLATE/new_nexus_package.md \ No newline at end of file diff --git a/.agents/skills/add-nexus-package/assets/templates/nexus-package-template b/.agents/skills/add-nexus-package/assets/templates/nexus-package-template new file mode 120000 index 0000000..5de7877 --- /dev/null +++ b/.agents/skills/add-nexus-package/assets/templates/nexus-package-template @@ -0,0 +1 @@ +../../../../../templates/nexus-package-template/ \ No newline at end of file diff --git a/.bob b/.bob new file mode 120000 index 0000000..c0ca468 --- /dev/null +++ b/.bob @@ -0,0 +1 @@ +.agents \ No newline at end of file From 13cab857b60f35ef837a1c1ce6e6a347df7e3f3f Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Thu, 7 May 2026 12:59:06 +0100 Subject: [PATCH 9/9] feat(skills): Updated PR creation skill Signed-off-by: Christian Pinto --- .agents/skills/add-nexus-package/SKILL.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.agents/skills/add-nexus-package/SKILL.md b/.agents/skills/add-nexus-package/SKILL.md index 97333e0..356d2f7 100644 --- a/.agents/skills/add-nexus-package/SKILL.md +++ b/.agents/skills/add-nexus-package/SKILL.md @@ -56,6 +56,14 @@ Using the above information you can infer the classification of the package: Also, ask the user if they want to create usage documentation for any of their models. If that's the cas help them draft it. +Note: + +- Do not infer any information about the user package name and models and their + details. Let thenuser give all the details. +- If one of the models requires vLLM, ask the user whether vLLM is a required + dependency or an optional one. If optional, ask the user which extra dependency + from the package should be added. + ## 2. Create New Branch Create a new git branch named `add--package` @@ -86,7 +94,7 @@ the template and that are not needed for this package ## 4. Validate Package ```bash -uv run an validate packages/ +uv run nexus validate packages/ ``` Notes: @@ -135,7 +143,7 @@ package to the dependencies with uv. The general command for adding a package to the dependencies is: ```bash -uv add --optional +uv add --optional --no-sync ``` The variant depends to the package classification: @@ -159,6 +167,7 @@ before making any change towards solving the dependency issue. Commit with DCO sign-off, this is achieved by adding the -s flag to the git commit command. +Make sure the user has pre-commit installed and configured correctly. Example commit command: @@ -166,7 +175,13 @@ Example commit command: git commit -s -m "feat(package): Add Nexus package" ``` +The pre-commit hooks will generate requirements files and fail the commit. +The requirements files need to be added to the commit and the commit command +executed once more. +Ask the user if they want to push the branch to their fork or not. + ## 7. Open Pull Request Once all the above have been done, create the pull request text by following the -template in `./assets/templates/new_nexus_package_pr.md`. +template in `./assets/templates/new_nexus_package_pr.md`. Ask the user if they +want the PR to be created automatically or not.