From 3093e7b0089286c6f7e3accd6c466376ddda3594 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sun, 23 Aug 2026 13:40:36 -0700 Subject: [PATCH] Expose Apache Ossie across public workflows --- README.md | 27 +- docs/compatibility/index.md | 2 +- docs/compatibility/osi.md | 227 +---------------- docs/compatibility/ossie.md | 240 ++++++++++++++++++ scripts/generate_compatibility_docs.py | 4 + sidemantic/adapters/osi.py | 31 ++- sidemantic/adapters/ossie.py | 240 ++++++++++++++++++ sidemantic/cli.py | 96 ++++++- sidemantic/formats.py | 35 ++- sidemantic/loaders.py | 30 ++- sidemantic/man/sidemantic.1 | 27 ++ sidemantic/validation_runner.py | 4 +- tests/adapters/osi/conftest.py | 2 +- tests/adapters/osi/test_ossie_adapter.py | 222 ++++++++++++++++ tests/adapters/osi/test_parsing.py | 2 +- tests/adapters/osi/test_query_execution.py | 2 +- tests/adapters/osi/test_rust_parity.py | 2 +- tests/adapters/test_added_fixture_coverage.py | 2 +- .../test_fixture_functionality_contracts.py | 4 +- tests/core/test_directory_loaders.py | 26 +- tests/test_cli_contract.py | 141 ++++++++++ tests/test_formats.py | 64 +++++ 22 files changed, 1164 insertions(+), 266 deletions(-) create mode 100644 docs/compatibility/ossie.md create mode 100644 sidemantic/adapters/ossie.py create mode 100644 tests/adapters/osi/test_ossie_adapter.py diff --git a/README.md b/README.md index f81675fbb..6632127c5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Sidemantic is an open-source semantic runtime. Define governed metrics once—or import the semantic models you already have—and query them consistently from SQL, the CLI, Python, HTTP, PostgreSQL clients, notebooks, BI tools, and AI agents. -- **Bring existing models:** Power BI TMDL/DAX, Cube, dbt MetricFlow, LookML, Hex, Rill, Superset, Omni, BSL, GoodData LDM, Snowflake Cortex, Malloy, OSI, AtScale SML, and ThoughtSpot TML +- **Bring existing models:** Power BI TMDL/DAX, Cube, dbt MetricFlow, LookML, Hex, Rill, Superset, Omni, BSL, GoodData LDM, Snowflake Cortex, Malloy, Apache Ossie, AtScale SML, and ThoughtSpot TML - **Or author natively:** concise YAML, semantic SQL DDL, or Python - **Run on your warehouse:** DuckDB, MotherDuck, PostgreSQL, BigQuery, Snowflake, ClickHouse, Databricks, Spark SQL, and ADBC sources - **Consume metrics anywhere:** semantic SQL, CLI, Python, HTTP/Arrow, PostgreSQL wire protocol, MCP, notebooks, TypeScript/WASM, and embedded analytics @@ -245,13 +245,13 @@ git clone https://github.com/sidequery/sidemantic.git && cd sidemantic uv run examples/rill_demo/run_demo.py ``` -**OSI (complex adtech semantic model):** +**Apache Ossie (complex adtech semantic model):** ```bash git clone https://github.com/sidequery/sidemantic.git && cd sidemantic uv run examples/osi_demo/run_demo.py ``` -**OSI widget notebook (percent-cell Python notebook):** +**Apache Ossie widget notebook (percent-cell Python notebook):** ```bash git clone https://github.com/sidequery/sidemantic.git && cd sidemantic uv run examples/osi_demo/osi_widget_notebook.py @@ -263,7 +263,7 @@ See `examples/` for more. - SQL query interface with automatic rewriting - Automatic joins across models -- Multi-format adapters (Cube, MetricFlow, LookML, Hex, Rill, Superset, Omni, BSL, GoodData LDM, OSI, AtScale SML, ThoughtSpot TML, Graphene GSQL) +- Multi-format adapters (Cube, MetricFlow, LookML, Hex, Rill, Superset, Omni, BSL, GoodData LDM, Apache Ossie, AtScale SML, ThoughtSpot TML, Graphene GSQL) - SQLGlot-based SQL generation and transpilation - Pydantic validation and type safety - Pre-aggregations with explicit routing @@ -275,7 +275,7 @@ See `examples/` for more. ## Multi-Format Support -Auto-detects: Sidemantic (SQL/YAML), Power BI TMDL, Cube, MetricFlow (dbt), LookML, Hex, Rill, Superset, Omni, BSL, GoodData LDM, OSI, AtScale SML, ThoughtSpot TML, Graphene GSQL +Auto-detects: Sidemantic (SQL/YAML), Power BI TMDL, Cube, MetricFlow (dbt), LookML, Hex, Rill, Superset, Omni, BSL, GoodData LDM, Apache Ossie, AtScale SML, ThoughtSpot TML, Graphene GSQL ```bash sidemantic query "SELECT revenue FROM orders" --models ./my_models @@ -288,6 +288,23 @@ layer = SemanticLayer(connection="duckdb:///data.duckdb") load_from_directory(layer, "my_models/") # Auto-detects formats ``` +Apache Ossie is registered as format `ossie`; `osi`, `apache-ossie`, and +`open-semantic-interchange` remain accepted aliases. Use an explicit format when +selecting an Ossie scope or execution dialect: + +```bash +sidemantic convert commerce.ossie.yaml \ + --from ossie --to sidemantic --output models.yml \ + --ossie-scope commerce --ossie-target-dialect bigquery + +sidemantic convert models/orders.yml \ + --from sidemantic --to ossie --output commerce.ossie.yaml \ + --ossie-scope commerce --ossie-expression-dialect BIGQUERY +``` + +See the [Apache Ossie compatibility guide](docs/compatibility/ossie.md) for the +pinned schema profiles, validation policies, scope rules, and round-trip limits. + ## Databases | Database | Status | Installation | diff --git a/docs/compatibility/index.md b/docs/compatibility/index.md index d49f01587..eb09674c1 100644 --- a/docs/compatibility/index.md +++ b/docs/compatibility/index.md @@ -19,7 +19,7 @@ Detailed compatibility documents describe field-level limitations where they exi | `malloy` | — | `file_or_directory` | `.malloy` | Registered | Yes | 10 | [Detailed guide](malloy.md) | | `metricflow` | `dbt`, `dbt-semantic-layer` | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 6 | Registry summary only | | `omni` | — | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 4 | Registry summary only | -| `osi` | `open-semantic-interchange` | `file_or_directory` | `.yml`, `.yaml`, `.json` | Registered | Yes | 3 | [Detailed guide](osi.md) | +| `ossie` | `apache-ossie`, `osi`, `open-semantic-interchange` | `file_or_directory` | `.yml`, `.yaml`, `.json` | Registered | Yes | 4 | [Detailed guide](ossie.md) | | `rill` | — | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 8 | Registry summary only | | `sidemantic` | `native` | `file` | `.yml`, `.yaml`, `.sql` | Registered | Yes | 1 | Registry summary only | | `snowflake` | `cortex`, `snowflake-cortex` | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 8 | Registry summary only | diff --git a/docs/compatibility/osi.md b/docs/compatibility/osi.md index a8d8ca455..6d86f911e 100644 --- a/docs/compatibility/osi.md +++ b/docs/compatibility/osi.md @@ -1,222 +1,9 @@ -# OSI Compatibility +# OSI compatibility alias -Sidemantic's OSI adapter parses [Open Semantic Interchange](https://github.com/open-semantic-interchange/OSI) YAML files and maps OSI concepts to Sidemantic's semantic model (Model, Dimension, Metric, Relationship). It also supports exporting back to OSI YAML, including multi-dialect SQL transpilation via sqlglot. +The canonical name is now [Apache Ossie](ossie.md), and the canonical +Sidemantic format name is `ossie`. -Features are marked **supported**, **partial support**, or **unsupported**. Partial support entries include notes explaining the limitation. Properties that parse without error but have no Sidemantic equivalent are grouped together per section rather than listed individually. - ---- - -## Semantic Model (Top Level) - -| Feature | Status | -|---------|--------| -| `semantic_model` (list of models) | Supported | -| Multiple semantic models in one file | Supported (each iterated independently) | -| `version` | Supported. Current exports emit `0.2.0.dev0`; imports preserve the source version in `SemanticGraph.metadata["osi"]["version"]`. | -| `name` (semantic model name) | Supported as metadata. Stored in `SemanticGraph.metadata["osi"]["semantic_models"]` and reused on export. Individual datasets still become named Models. | -| `description` (semantic model description) | Supported as graph metadata and reused on export. | -| `ai_context` (semantic model level) | Supported as graph metadata and reused on export. | -| `custom_extensions` (semantic model level) | Supported as graph metadata and reused on export. | - ---- - -## Datasets - -| Feature | Status | -|---------|--------| -| `name` | Supported | -| `source` (table reference) | Supported (stored as `Model.table`) | -| `description` | Supported | -| `primary_key` (single column, e.g. `[id]`) | Supported (stored as string) | -| `primary_key` (composite, e.g. `[col_a, col_b]`) | Supported (stored as list) | -| `unique_keys` (list of column lists) | Supported | -| `fields` | Supported (mapped to Dimensions) | -| `ai_context` | Supported (stored in `Model.meta["ai_context"]`) | -| `custom_extensions` (dataset level) | Supported (stored in `Model.meta["custom_extensions"]`) | -| Dataset without `name` | Supported (gracefully skipped) | -| Multi-file directory parsing (recursive `.yml`/`.yaml` discovery) | Supported | -| Default primary key when omitted | Supported (defaults to `"id"`) | - ---- - -## Fields (Dimensions) - -| Feature | Status | -|---------|--------| -| `name` | Supported | -| `expression.dialects` (SQL expressions per dialect) | Supported | -| `description` | Supported | -| `label` | Supported | -| `dimension.is_time: true` | Supported (maps to `Dimension(type="time", granularity="day")`) | -| `dimension.is_time: false` | Supported (maps to `Dimension(type="categorical")`) | -| No `dimension` block | Supported (defaults to `categorical`) | -| `ai_context` | Supported (stored in `Dimension.meta["ai_context"]`) | -| `custom_extensions` | Supported (stored in `Dimension.meta["custom_extensions"]`) | -| Computed/derived expressions (e.g. `c_first_name \|\| ' ' \|\| c_last_name`) | Supported (expression preserved verbatim from chosen dialect) | -| Field without `name` | Supported (gracefully skipped) | - -### Dialect Preference - -When multiple dialects are present on a field expression, the adapter selects one using a fixed preference order: `ANSI_SQL` > `SNOWFLAKE` > `DATABRICKS`. If none of these are present, the first dialect in the list is used as fallback. - -Not mapped: `dimension.granularity`, `dimension.time_zone`, field `type` (all fields become either `time` or `categorical` based solely on `is_time`). - ---- - -## Metrics - -| Feature | Status | -|---------|--------| -| Simple aggregations (`SUM(...)`, `COUNT(...)`, `AVG(...)`, `MIN(...)`, `MAX(...)`) | Supported (aggregation type auto-detected by Metric's model_validator via sqlglot) | -| `COUNT(DISTINCT ...)` | Supported (detected as `count_distinct`) | -| Complex/ratio expressions (e.g. `SUM(x) / NULLIF(COUNT(DISTINCT y), 0)`) | Supported (parsed as derived metric) | -| `CASE WHEN` inside aggregations | Supported (expression preserved verbatim) | -| `FILTER (WHERE ...)` clause | Supported (expression preserved verbatim) | -| Cross-dataset references (e.g. `SUM(orders.amount)`) | Supported (dot-qualified references preserved in expression) | -| `description` | Supported | -| `ai_context` | Supported (stored in `Metric.meta["ai_context"]`) | -| `custom_extensions` | Supported (stored in `Metric.meta["custom_extensions"]`) | -| Metric without `name` | Supported (gracefully skipped) | -| Metric without `expression` | Supported (gracefully skipped) | - -Not mapped: `type` (the OSI `type` hint on metrics, e.g. `type: count`, is ignored; aggregation type is inferred from the SQL expression instead). - ---- - -## Relationships - -| Feature | Status | -|---------|--------| -| `from` / `to` (dataset references) | Supported | -| `from_columns` / `to_columns` (single column) | Supported | -| `from_columns` / `to_columns` (multi-column composite keys) | Supported | -| Relationship type | Partial support: all relationships are imported as `many_to_one`. The OSI spec does not define cardinality on the `from`/`to` relationship format, and the adapter always assumes many-to-one. | -| Relationship `name` | Supported (stored in `Relationship.metadata["osi_name"]` and reused on export) | -| Relationship `ai_context` | Supported (stored in `Relationship.metadata["ai_context"]`) | -| Relationship `custom_extensions` | Supported (stored in `Relationship.metadata["custom_extensions"]`) | -| Missing `from_columns` | Supported (defaults foreign key to `{to_model}_id`) | -| Missing `to_columns` | Supported (defaults primary key to `id`) | -| Relationship with missing `from` or `to` | Supported (gracefully skipped) | -| Relationship to non-existent model | Supported (gracefully skipped if the `from` model is not in the graph) | -| `left_dataset` / `right_dataset` / `cardinality` format | Unsupported | - -The `left_dataset`/`right_dataset`/`cardinality` relationship format (used by some community OSI files, e.g. mdb-engine models) is not parsed. Only the `from`/`to`/`from_columns`/`to_columns` format is recognized. Files using the alternative format will parse without error, but relationships will be silently skipped. - ---- - -## Expression Dialects - -### Import - -| Feature | Status | -|---------|--------| -| `ANSI_SQL` dialect | Supported (preferred) | -| `SNOWFLAKE` dialect | Supported (second preference) | -| `DATABRICKS` dialect | Supported (third preference) | -| `MAQL`, `TABLEAU`, `MDX` dialects | Supported as fallback expressions if no preferred SQL dialect is present. The expression is preserved as text; it is not translated. | -| Other/custom dialects | Supported on import as fallback if preferred dialects absent | -| Multiple dialects per expression | Supported (single dialect selected by preference order) | -| Missing `expression` block | Supported (field SQL becomes `None`) | - -### Export - -| Feature | Status | -|---------|--------| -| Export to `ANSI_SQL` | Supported (default, expression passed through as-is) | -| Export to `SNOWFLAKE` | Supported (transpiled from DuckDB/ANSI via sqlglot) | -| Export to `DATABRICKS` | Supported (transpiled via sqlglot) | -| Export to `BIGQUERY` | Unsupported (not in the current OSI dialect enum) | -| Export to `MAQL`, `TABLEAU`, `MDX` | Unsupported (not safely generated from Sidemantic SQL) | -| Multiple dialects in single export | Supported (pass `dialects=["ANSI_SQL", "SNOWFLAKE", ...]`) | -| Unknown dialect on export | Unsupported (raises `ValueError`) | -| Transpilation failure | Supported (falls back to original expression) | - ---- - -## AI Context - -`ai_context` is an OSI extension for providing hints to AI/LLM systems. Sidemantic preserves the full structure at all levels. - -| Level | Status | -|-------|--------| -| Dataset `ai_context` | Supported (stored in `Model.meta["ai_context"]`) | -| Field `ai_context` | Supported (stored in `Dimension.meta["ai_context"]`) | -| Metric `ai_context` | Supported (stored in `Metric.meta["ai_context"]`) | -| Semantic model level `ai_context` | Supported (stored in `SemanticGraph.metadata["osi"]["semantic_models"]`) | -| Relationship `ai_context` | Supported (stored in `Relationship.metadata["ai_context"]`) | - -Common sub-keys (`synonyms`, `instructions`, `examples`, `description_for_ai`) are all preserved as-is in the meta dictionary. No sub-key receives special handling. - ---- - -## Custom Extensions - -| Level | Status | -|-------|--------| -| Dataset `custom_extensions` | Supported (stored in `Model.meta["custom_extensions"]`) | -| Field `custom_extensions` | Supported (stored in `Dimension.meta["custom_extensions"]`) | -| Metric `custom_extensions` | Supported (stored in `Metric.meta["custom_extensions"]`) | -| Relationship `custom_extensions` | Supported (stored in `Relationship.metadata["custom_extensions"]`) | -| Semantic model level `custom_extensions` | Supported (stored in `SemanticGraph.metadata["osi"]["semantic_models"]`) | - -Current OSI schema represents `custom_extensions` as a list of `{vendor_name, data}` objects where `data` is a JSON string. Sidemantic still imports legacy dict/list extension shapes. On export, non-standard extension payloads are wrapped as a `SIDEMANTIC` extension with stringified JSON so emitted OSI stays schema-shaped. - ---- - -## OSI Export (Roundtrip) - -Sidemantic can export its semantic model back to OSI YAML format. - -| Feature | Status | -|---------|--------| -| Current OSI version | Supported (`version: "0.2.0.dev0"` emitted at file root) | -| Semantic-model-level metadata | Supported (`name`, `description`, `ai_context`, `custom_extensions`) | -| Datasets with `source` (table name) | Supported | -| Datasets with `sql` (derived/subquery source) | Supported (wrapped in parentheses) | -| Fields (dimensions) | Supported | -| Time dimension `is_time` flag | Supported | -| Field `label` | Supported | -| Field `description` | Supported | -| Primary key (single column) | Supported (exported as list) | -| Primary key (composite) | Supported (exported as list) | -| Unique keys | Supported | -| Standard aggregation metrics (sum, count, avg, min, max) | Supported (reconstructed as `AGG(inner)`) | -| `count_distinct` metrics | Supported (exported as `COUNT(DISTINCT inner)`) | -| Ratio metrics | Supported (exported as `numerator / NULLIF(denominator, 0)`) | -| Derived metrics | Supported (SQL expression exported as-is) | -| Model-level metrics | Supported (promoted to semantic_model-level metrics, field refs qualified with model name) | -| Graph-level metrics | Supported | -| Relationships (`many_to_one` only) | Supported | -| Relationship `name`, `ai_context`, `custom_extensions` | Supported | -| Non-`many_to_one` relationships | Unsupported (silently omitted from export) | -| Multi-column relationship keys | Supported | -| Related model PK lookup on export | Supported (when `rel.primary_key` is None, the related model's actual PK columns are used for `to_columns`) | -| `ai_context` roundtrip (dataset, field, metric) | Supported | -| `custom_extensions` roundtrip (dataset, field, metric) | Supported | -| Multi-dialect export via sqlglot | Supported | -| Model inheritance resolution before export | Supported | -| Roundtrip fidelity (OSI -> parse -> export -> re-parse) | Supported (semantically equivalent graphs) | - ---- - -## Version Field - -Supported. Imports preserve the source value in graph metadata. Exports always emit the current supported OSI draft version, `0.2.0.dev0`. - ---- - -## Ontology Files - -OSI's ontology spec defines conceptual `ontology` entries plus `ontology_mappings` that embed logical semantic models. Sidemantic does not implement ontology reasoning, but it does: - -| Feature | Status | -|---------|--------| -| Top-level `ontology` | Partial support: preserved in `SemanticGraph.metadata["osi"]["ontology"]`; not converted to Models. | -| `ontology_mappings[].semantic_model` | Supported: parsed into Sidemantic Models, Relationships, and Metrics. | -| `ontology_mappings[].concept_mappings` | Partial support: preserved in graph metadata; not used for query planning. | - ---- - -## Alternative Relationship Formats - -Unsupported. The `left_dataset`/`right_dataset`/`cardinality` relationship format found in some community OSI files (mdb-engine, etc.) is not recognized by the adapter. These relationships are silently skipped during parsing. Only the canonical `from`/`to`/`from_columns`/`to_columns` format is handled. +Existing commands may continue to use `osi`. The aliases `apache-ossie` and +`open-semantic-interchange` are also accepted. New documentation, automation, +and examples should use `ossie`; no source-file rename is required for the +migration. diff --git a/docs/compatibility/ossie.md b/docs/compatibility/ossie.md new file mode 100644 index 000000000..daf8232f3 --- /dev/null +++ b/docs/compatibility/ossie.md @@ -0,0 +1,240 @@ +# Apache Ossie compatibility + +Sidemantic's Python integration handles [Apache Ossie](https://github.com/apache/ossie) +logical semantic-model and ontology source documents. Executable import and +graph synthesis apply only to logical documents. The canonical format name is +`ossie`; `osi`, `apache-ossie`, and `open-semantic-interchange` remain accepted +aliases. + +The historical Python import spelling +`sidemantic.adapters.osi.OSIAdapter` now routes to this canonical contract. +The pre-profile implementation is isolated as `LegacyOSIAdapter` for explicit +migration work; format discovery, CLI aliases, and ordinary direct imports do +not bypass validation or scoped lowering. + +Install the optional validator dependencies with `uv add 'sidemantic[ossie]'`, +or use `uv sync --extra ossie` in a Sidemantic checkout. Validation is offline: +Sidemantic loads only the vendored, checksum-verified schemas listed below and +does not retrieve schema resources from the network. + +## Pinned profiles + +Schema version, consumer profile, and YAML-versus-JSON serialization are +independent choices. Sidemantic supports these explicit contracts: + +| Consumer profile | Declared version | Document family | Validation schema | Pinned upstream commit | +|---|---|---|---|---| +| `ossie-core` | `0.1.1` | Logical | `logical-0.1.1` | `faf581054dcf7964d5fe0ceae7d6f415c8ce32a5` | +| `ossie-core` | `0.2.0.dev0` | Logical | `logical-0.2.0.dev0` | `88e0011148283302c9a04cd0287e00e0b9d87354` | +| `ossie-core` | `0.2.0.dev0` | Ontology | `ontology-0.2.0.dev0` | `88e0011148283302c9a04cd0287e00e0b9d87354` | +| `dbt-1.12` | `0.1.0` | Logical compatibility alias | Pinned `logical-0.1.1`; the retained document still declares `0.1.0` | `faf581054dcf7964d5fe0ceae7d6f415c8ce32a5` | +| `dbt-1.12` | `0.1.1` | Logical | `logical-0.1.1` | `faf581054dcf7964d5fe0ceae7d6f415c8ce32a5` | + +`0.1.0` is not treated as an upstream Ossie schema version. It is accepted only +with the explicit `dbt-1.12` consumer profile. Validation, file-adapter and CLI +imports, lossless retained-source output, canonical serialization, and explicit +graph synthesis all preserve that consumer context while validating a temporary +copy against the pinned `logical-0.1.1` schema. The retained or emitted document +continues to declare `0.1.0`. + +The vendored ontology runtime schema has two recorded deterministic rewrites: a +unique local `$id` and local references to the pinned logical schema. The +untouched upstream ontology schema is retained alongside it. Exact paths, +source URLs, transformations, and SHA-256 values are in +[`sidemantic/interchange/ossie/schemas/manifest.json`](../../sidemantic/interchange/ossie/schemas/manifest.json). + +## Pinned upstream validator gate + +The conformance suite also runs the official Apache Ossie validator from a +pinned local fixture, using only the vendored upstream schemas and validator +code. It verifies that canonical Sidemantic exports pass for core `0.1.1` +(JSON) and core `0.2.0.dev0` (YAML), and that a deliberately invalid document is +rejected. The gate is implemented by +[`tests/interchange/ossie/test_upstream_validator_gate.py`](../../tests/interchange/ossie/test_upstream_validator_gate.py) +and does not require network access. The `dbt-1.12` compatibility alias is +validated separately against its declared compatibility contract; it is not +presented as an upstream Ossie schema version. + +## Source documents and runtime graphs are different contracts + +An Ossie source document is an immutable parsed YAML or JSON value. It retains +the declared document family and version, canonical data, source identity, and, +when requested, the original bytes. Unknown fields that pass the selected +schema remain in that source document. + +A runtime `SemanticGraph` is a target-specific executable projection of one +logical `semantic_model` scope. It contains only constructs Sidemantic can lower +safely for the selected runtime dialect. It is not an archival copy of the +source document and cannot reproduce alternate expression dialects, lexical +YAML details, ontology content, or every Ossie field. + +Use the preserved-document APIs for validation, migration, and source round +trips. Use a compiled scope or its graph for execution. Converting through a +graph is synthesis, not source-document round trip. + +## Validation and import policy + +Import has separate parsing, pinned JSON Schema validation, semantic validation, +and lowering stages. Diagnostics have stable codes and JSON pointers; schema +diagnostics also identify the profile, schema commit, and checksum. + +- `strict` is the default. Any parser, profile, schema, or semantic error blocks + executable lowering. A lowering error also removes all executable scopes + rather than returning a partly trusted strict result. +- `permissive` preserves the invalid source and projects only independently safe + constructs. Unsafe fields, metrics, relationships, or datasets are excluded + with diagnostics; they are not silently accepted. Resulting scopes remain + marked `valid=False`. +- An invalid permissive scope is not executable by default. + `SemanticLayer.from_catalog(...)` requires the explicit + `allow_invalid=True` escape hatch. The graph-returning compatibility adapter + also refuses an invalid project, so use the document/catalog APIs when the + goal is to inspect a partial migration result. + +The CLI exposes permissive parsing as `--ossie-permissive` and profile selection +as `--ossie-consumer-profile`. These options do not weaken the runtime binding +safeguards described above. + +## Scoped `SemanticCatalog` behavior + +Each logical `semantic_model` becomes an isolated `CompiledSemanticScope` in a +`SemanticCatalog`. Namespaces are never flattened or merged, so two scopes may +use the same dataset or metric names without colliding. + +- A catalog with one scope may infer that scope. +- A catalog with multiple scopes requires an explicit scope ID, supplied by the + CLI as `--ossie-scope`. +- Duplicate semantic-model names are diagnosed; deterministic permissive IDs + use `name@index`. +- Directory imports preserve document boundaries and qualify IDs as + `relative/path.yaml::scope`. Generated `target` and `dbt_packages` + directories are skipped. +- Catalog membership and compilation metadata are immutable snapshots. Runtime + layers receive isolated graph clones. +- A compiled scope records document/content/compilation identity, validation + state, lowering policy, schema provenance, and target dialect. Cache identity + changes when relevant source, profile, dialect, or policy inputs change. +- `SemanticLayer.from_catalog` rejects a runtime connection whose dialect does + not match the scope's compiled target dialect. + +## Target-aware expression selection + +Executable import requires a target dialect. For each field or metric, +Sidemantic selects the exact matching Ossie expression when Ossie defines a +label for that target: `BIGQUERY`, `SNOWFLAKE`, or `DATABRICKS`. Otherwise it +uses an explicit `ANSI_SQL` variant. If neither exists, the construct is +diagnosed and excluded; Sidemantic does not select an arbitrary first variant, +relabel SQL, or claim that unchanged text was transpiled. + +The selected text must parse as exactly one scalar expression in the target +runtime dialect. Following the structural boundary in the pinned +[Apache expression-language proposal](https://github.com/apache/ossie/blob/88e0011148283302c9a04cd0287e00e0b9d87354/core-spec/expression_language.md), +queries and nested subqueries, CTEs, set operations, query clauses, DDL, DML, +commands, malformed text, and multiple statements are rejected. The gate does +not impose an ANSI function allowlist on vendor-dialect variants. Current target +parsers are BigQuery, Databricks, DuckDB, Postgres, Snowflake, and Spark. +`BIGQUERY` is supported both for target-aware import and explicit graph +synthesis. + +`source_dialect` is separate and is used only when classifying a dataset +`source` as a table reference or SQL query. It is part of compilation identity. + +Semantic identifiers follow the pinned proposal's comparison rules. Regular +identifiers are resolved case-insensitively after uppercasing. Double-quoted +identifiers are resolved exactly after stripping the outer quotes and unescaping +doubled quotes, so `orders` and `Orders` match while `orders` and `"orders"` do +not. Duplicate detection, relationship references, and key references use that +same normalization, but declarations and retained source text are not renamed. +Identifiers longer than 128 decoded characters are diagnosed and excluded from +executable lowering. + +## Executable logical-model projection + +| Ossie construct | Runtime behavior | +|---|---| +| `semantic_model` | One isolated catalog scope per uniquely identified model; no cross-scope merge | +| Dataset `source` | Preserved as a table reference or SQL query only when it can be classified safely | +| `primary_key` and `unique_keys` | Imported only when explicitly declared and structurally usable; no default `id` or other fabricated key | +| Dataset fields | Become dimensions only with a selected, valid target expression; datatype and explicit time-role information are retained where representable | +| Metrics | Become graph-level scalar SQL metrics only with a selected, valid target expression | +| Relationships | Become many-to-one runtime edges only when identity, endpoints, key arrays, fields, arity, and target uniqueness are all verified | + +Relationship `name` is the edge identity. It is retained as `edge_id` so +multiple edges between the same datasets remain distinguishable. Import never +invents relationship names or missing key columns. The target columns must +match an explicitly declared primary or unique key; an unsafe relationship is +preserved in the source document but excluded from executable topology. + +## Ontology documents + +Ontology documents use the pinned ontology schema and receive semantic checks +for concept references and embedded logical-model structure. Their complete +validated canonical data can be preserved and serialized as an opaque source +document. + +Sidemantic does not perform ontology reasoning, does not turn ontology concepts +or mappings into runtime models, and does not execute ontology documents. +Ontology lowering therefore produces no executable catalog scopes. + +## Canonical and exact-byte source round trips + +Canonical serialization validates the retained document and emits deterministic, +Unicode-safe JSON or YAML. It preserves canonical data, not lexical form: YAML +comments, anchors, aliases, quoting, scalar styles, key order, and whitespace +may change. Cross-serialization between YAML and JSON is always canonical. + +Exact-byte reuse is an explicit source-document option. It succeeds only when +original bytes were retained, output serialization matches the input, and +reparsing those bytes produces the current canonical data. Otherwise Sidemantic +emits canonical output and reports an `ossie.serialization.exact_source_mismatch` +warning. This guarantee does not apply to an Ossie -> runtime graph -> Ossie +conversion. + +## Fail-closed graph synthesis + +Exporting a runtime graph creates a new logical document. It requires both: + +- `scope_name`: the containing Ossie semantic-model name. +- `expression_dialect`: the exact label for emitted SQL. Supported values are + `ANSI_SQL`, `BIGQUERY`, `DATABRICKS`, and `SNOWFLAKE`. + +`schema_version` may also be selected explicitly; the current default is +`0.2.0.dev0`. Sidemantic emits one expression variant with the supplied label +and validates the completed document against the pinned schema. It validates +that expression text in the named dialect but does not infer its origin, +transpile it, or relabel it as another dialect. + +Synthesis refuses the complete output when it encounters meaning it would have +to invent or misrepresent, including a missing or ambiguous dataset source, +untranslated non-SQL expressions, invalid scalar SQL, unsupported datatypes or +relationship cardinality, missing or duplicate edge identity, unusable key +arrays, non-unique relationship targets, and conflicting metric definitions. +Graph synthesis cannot create ontology documents or recover source-only fields. + +## CLI examples + +Import one explicit scope for a BigQuery runtime projection: + +```bash +sidemantic convert commerce.ossie.yaml \ + --from ossie \ + --to sidemantic \ + --output models.yml \ + --ossie-scope commerce \ + --ossie-target-dialect bigquery +``` + +Synthesize a new Ossie document while labeling only SQL already intended for +BigQuery: + +```bash +sidemantic convert models/orders.yml \ + --from sidemantic \ + --to ossie \ + --output commerce.ossie.yaml \ + --ossie-scope commerce \ + --ossie-expression-dialect BIGQUERY \ + --ossie-schema-version 0.2.0.dev0 +``` + +Use `--force` only when intentionally replacing an existing output file. diff --git a/scripts/generate_compatibility_docs.py b/scripts/generate_compatibility_docs.py index 72e62c689..9138dc4ae 100644 --- a/scripts/generate_compatibility_docs.py +++ b/scripts/generate_compatibility_docs.py @@ -95,6 +95,10 @@ def render_runtime_matrix(contract: dict) -> str: def _adapter_test_directory(name: str) -> str: if name == "sidemantic": return "sidemantic_adapter" + if name == "ossie": + # Keep counting the established adapter suite while ``ossie`` replaces + # the historical public format name ``osi``. + return "osi" return name.replace("-", "_") diff --git a/sidemantic/adapters/osi.py b/sidemantic/adapters/osi.py index a958c356e..527d42b3f 100644 --- a/sidemantic/adapters/osi.py +++ b/sidemantic/adapters/osi.py @@ -26,6 +26,7 @@ import yaml from sidemantic.adapters.base import BaseAdapter +from sidemantic.adapters.ossie import OssieAdapter as _CanonicalOssieAdapter from sidemantic.core.dimension import Dimension from sidemantic.core.metric import Metric from sidemantic.core.model import Model @@ -53,8 +54,13 @@ def _is_generated_artifact(file_path: Path, directory: Path) -> bool: return any(part in _GENERATED_ARTIFACT_DIRS for part in relative_parts[:-1]) -class OSIAdapter(BaseAdapter): - """Adapter for importing/exporting OSI (Open Semantic Interchange) YAML files. +class LegacyOSIAdapter(BaseAdapter): + """Legacy pre-Apache OSI adapter retained for migration compatibility. + + New integrations must use :class:`OSIAdapter`, which is the compatibility + spelling for the schema-validated :class:`~sidemantic.adapters.ossie.OssieAdapter`. + This implementation remains available only so callers can migrate source + documents that depended on the historical, pre-profile behavior. Transforms OSI definitions into Sidemantic format: - OSI semantic_model → SemanticGraph @@ -1027,3 +1033,24 @@ def _build_metric_expression(self, metric: Metric, model_name: str | None) -> st # No aggregation, just SQL return metric.sql + + +# ``osi`` was Sidemantic's public spelling before the project became Apache +# Ossie. Keep the import path, but route it through the canonical implementation +# so direct imports cannot bypass profiles, schema/semantic validation, scoped +# catalogs, target-aware expression selection, or fail-closed synthesis. + + +class OSIAdapter(_CanonicalOssieAdapter): + """Backward-compatible class spelling for the canonical Apache Ossie adapter.""" + + OSI_VERSION = LegacyOSIAdapter.OSI_VERSION + RELEASED_OSI_VERSION = LegacyOSIAdapter.RELEASED_OSI_VERSION + RELEASED_OSI_VERSIONS = LegacyOSIAdapter.RELEASED_OSI_VERSIONS + RELEASED_OSI_VENDORS = LegacyOSIAdapter.RELEASED_OSI_VENDORS + SUPPORTED_EXPORT_FORMATS = LegacyOSIAdapter.SUPPORTED_EXPORT_FORMATS + DIALECT_PREFERENCE = LegacyOSIAdapter.DIALECT_PREFERENCE + SUPPORTED_EXPORT_DIALECTS = ["ANSI_SQL", "BIGQUERY", "SNOWFLAKE", "DATABRICKS"] + + +__all__ = ["LegacyOSIAdapter", "OSIAdapter"] diff --git a/sidemantic/adapters/ossie.py b/sidemantic/adapters/ossie.py new file mode 100644 index 000000000..7cb7e3020 --- /dev/null +++ b/sidemantic/adapters/ossie.py @@ -0,0 +1,240 @@ +"""Apache Ossie importer backed by preserved documents and scoped lowering.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from sidemantic.adapters.base import BaseAdapter +from sidemantic.core.semantic_catalog import ( + AmbiguousSemanticScopeError, + CompiledSemanticScope, + DuplicateSemanticScopeError, + SemanticCatalog, + SemanticScopeNotFoundError, +) +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.interchange.ossie import ( + OssieConsumerProfile, + OssieDiagnostic, + OssieDocument, + OssieImportPolicy, + OssieLoweringResult, + OssieParseOptions, + OssieParseResult, + OssiePreservationPolicy, + OssieSerialization, + OssieSerializationResult, + lower_ossie_document, + parse_ossie_document, + require_synthesized_document, + serialize_ossie_document, + sort_diagnostics, + synthesize_ossie_document, +) + +_GENERATED_DIRECTORIES = frozenset({"dbt_packages", "target"}) + + +class OssieImportError(ValueError): + """Raised when an Ossie source cannot produce the requested executable scope.""" + + def __init__(self, message: str, *, diagnostics: tuple[OssieDiagnostic, ...] = ()) -> None: + super().__init__(message) + self.diagnostics = diagnostics + + +@dataclass(frozen=True, slots=True) +class OssieProjectResult: + """Preserved per-document results and their isolated compiled scopes.""" + + lowerings: tuple[OssieLoweringResult, ...] + catalog: SemanticCatalog + + @property + def diagnostics(self) -> tuple[OssieDiagnostic, ...]: + return sort_diagnostics(diagnostic for result in self.lowerings for diagnostic in result.diagnostics) + + @property + def valid(self) -> bool: + return all(result.valid for result in self.lowerings) + + +class OssieAdapter(BaseAdapter): + """Import Apache Ossie files without flattening document scopes.""" + + def __init__( + self, + *, + target_dialect: str = "duckdb", + source_dialect: str | None = None, + scope_id: str | None = None, + import_policy: OssieImportPolicy | str = OssieImportPolicy.STRICT, + preserve_source: bool = False, + consumer_profile: OssieConsumerProfile | str = OssieConsumerProfile.OSSIE_CORE, + export_scope_name: str | None = None, + expression_dialect: str | None = None, + schema_version: str = "0.2.0.dev0", + serialization: OssieSerialization | str | None = None, + ) -> None: + if not target_dialect.strip(): + raise ValueError("target_dialect must be a non-empty string") + self._target_dialect = target_dialect + self._scope_id = scope_id + self._export_scope_name = export_scope_name + self._expression_dialect = expression_dialect + self._schema_version = schema_version + self._serialization = OssieSerialization(serialization) if serialization is not None else None + self._parse_options = OssieParseOptions( + consumer_profile=consumer_profile, + import_policy=import_policy, + source_dialect=source_dialect, + target_dialect=target_dialect, + preservation_policy=( + OssiePreservationPolicy.SOURCE_BYTES if preserve_source else OssiePreservationPolicy.CANONICAL_DATA + ), + validate_schema=True, + ) + + def parse_document(self, source: str | Path) -> OssieLoweringResult: + """Parse, validate, preserve, and lower one exact Ossie file.""" + source_path = Path(source) + if not source_path.exists(): + raise FileNotFoundError(f"Path does not exist: {source_path}") + if not source_path.is_file(): + raise ValueError("parse_document requires one exact Ossie file") + parsed = parse_ossie_document( + source_path.read_bytes(), + source_identifier=str(source_path), + options=self._parse_options, + ) + return lower_ossie_document(parsed, target_dialect=self._target_dialect) + + def parse_catalog(self, source: str | Path) -> OssieProjectResult: + """Load one file or a deterministic directory of isolated documents.""" + source_path = Path(source) + if not source_path.exists(): + raise FileNotFoundError(f"Path does not exist: {source_path}") + + if source_path.is_file(): + lowerings = (self.parse_document(source_path),) + return OssieProjectResult(lowerings=lowerings, catalog=lowerings[0].catalog) + + candidates = sorted( + path + for path in source_path.rglob("*") + if path.is_file() + and path.suffix.lower() in {".json", ".yaml", ".yml"} + and not any(part in _GENERATED_DIRECTORIES for part in path.relative_to(source_path).parts[:-1]) + ) + lowerings = tuple(self.parse_document(path) for path in candidates) + scopes: list[CompiledSemanticScope] = [] + for lowering, path in zip(lowerings, candidates, strict=True): + relative = path.relative_to(source_path).as_posix() + for scope in lowering.catalog.scopes.values(): + qualified_id = f"{relative}::{scope.scope_id}" + scopes.append( + CompiledSemanticScope( + scope_id=qualified_id, + document_id=scope.document_id, + content_id=scope.content_id, + compilation_id=scope.compilation_id, + revision_id=scope.revision_id, + runtime=scope.runtime, + target_dialect=scope.target_dialect, + lowering_policy=scope.lowering_policy, + valid=scope.valid, + diagnostics=scope.diagnostics, + provenance={**dict(scope.provenance), "unqualified_scope_id": scope.scope_id}, + ) + ) + try: + catalog = SemanticCatalog(scopes) + except DuplicateSemanticScopeError as exc: # defensive: qualified IDs should already be unique + raise OssieImportError(str(exc)) from exc + return OssieProjectResult(lowerings=lowerings, catalog=catalog) + + def parse(self, source: str | Path) -> SemanticGraph: + """Return one explicitly selected graph for the legacy adapter surface.""" + project = self.parse_catalog(source) + if any(not lowering.valid for lowering in project.lowerings): + raise OssieImportError( + "Apache Ossie import failed validation; inspect diagnostics for the exact source paths.", + diagnostics=project.diagnostics, + ) + try: + return project.catalog.resolve_scope(self._scope_id).graph + except (AmbiguousSemanticScopeError, SemanticScopeNotFoundError) as exc: + raise OssieImportError(str(exc), diagnostics=project.diagnostics) from exc + + @staticmethod + def _output_serialization(output_path: Path, explicit: OssieSerialization | str | None) -> OssieSerialization: + if explicit is not None: + return OssieSerialization(explicit) + return OssieSerialization.JSON if output_path.suffix.lower() == ".json" else OssieSerialization.YAML + + def export_document( + self, + source: OssieDocument | OssieParseResult | OssieLoweringResult, + output_path: str | Path, + *, + serialization: OssieSerialization | str | None = None, + exact_source: bool = False, + ) -> OssieSerializationResult: + """Validate and write one preserved Ossie document.""" + if isinstance(source, OssieLoweringResult): + document = source.document + profile = source.parse_result.profile + elif isinstance(source, OssieParseResult): + document = source.document + profile = source.profile + else: + document = source + profile = None + destination = Path(output_path) + output_serialization = self._output_serialization(destination, serialization or self._serialization) + result = serialize_ossie_document( + document, + output_serialization, + exact_source=exact_source, + profile=profile, + consumer_profile=self._parse_options.consumer_profile, + ) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(result.data) + return result + + def export( + self, + graph: SemanticGraph, + output_path: str | Path, + *, + scope_name: str | None = None, + expression_dialect: str | None = None, + schema_version: str | None = None, + serialization: OssieSerialization | str | None = None, + ) -> None: + """Synthesize and write one schema-valid logical Ossie document. + + Graph synthesis is intentionally explicit: a lowered runtime graph does + not retain enough information to guess its semantic-model container or + the dialect of its SQL strings. + """ + destination = Path(output_path) + output_serialization = self._output_serialization(destination, serialization or self._serialization) + selected_scope = scope_name or self._export_scope_name + selected_dialect = expression_dialect or self._expression_dialect + if selected_scope is None: + raise ValueError("Ossie graph export requires an explicit scope_name") + if selected_dialect is None: + raise ValueError("Ossie graph export requires an explicit expression_dialect") + synthesis = synthesize_ossie_document( + graph, + scope_name=selected_scope, + expression_dialect=selected_dialect, + schema_version=schema_version or self._schema_version, + serialization=output_serialization, + consumer_profile=self._parse_options.consumer_profile, + ) + document = require_synthesized_document(synthesis) + self.export_document(document, destination, serialization=output_serialization) diff --git a/sidemantic/cli.py b/sidemantic/cli.py index f1ae4c1d1..9c6ccf606 100644 --- a/sidemantic/cli.py +++ b/sidemantic/cli.py @@ -221,6 +221,7 @@ def _load_query_layer( use_preaggregations: bool = False, engine: str | None = None, fallback: bool | None = None, + ossie_scope: str | None = None, ) -> SemanticLayer: """Load a semantic layer for CLI query/explain commands.""" engine, resolved_fallback = _resolve_engine_options(engine, fallback) @@ -250,9 +251,15 @@ def _load_query_layer( # sibling model or an unrelated broken draft must not pollute or fail the load. from sidemantic.loaders import load_from_file - load_from_file(layer, models) + if ossie_scope is None: + load_from_file(layer, models) + else: + load_from_file(layer, models, ossie_scope_id=ossie_scope) else: - load_from_directory(layer, str(models)) + if ossie_scope is None: + load_from_directory(layer, str(models)) + else: + load_from_directory(layer, str(models), ossie_scope_id=ossie_scope) if not layer.graph.models: raise ValueError("No models found") return layer @@ -1150,6 +1157,7 @@ def info( None, help="Directory containing semantic layer files (defaults to project models)" ), json_output: bool = typer.Option(False, "--json", help="Emit semantic-layer metadata as JSON"), + ossie_scope: str = typer.Option(None, "--ossie-scope", help="Explicit Apache Ossie semantic-model scope"), ): """ Show quick info about the semantic layer. @@ -1163,7 +1171,10 @@ def info( structured = json_output or cli_state().requested_format is not None or cli_state().plain directory = _models_path(directory) layer = SemanticLayer() - load_from_directory(layer, str(directory)) + if ossie_scope is None: + load_from_directory(layer, str(directory)) + else: + load_from_directory(layer, str(directory), ossie_scope_id=ossie_scope) models_payload = [ { @@ -1452,6 +1463,36 @@ def convert( target_format: str = typer.Option( "sidemantic", "--to", help="Destination format", autocompletion=complete_target_format ), + ossie_scope: str = typer.Option( + None, + "--ossie-scope", + help="Explicit semantic-model scope to select on Ossie import or name on Ossie export", + ), + ossie_target_dialect: str = typer.Option( + None, + "--ossie-target-dialect", + help="Execution dialect used when lowering an Ossie source (for example duckdb or bigquery)", + ), + ossie_expression_dialect: str = typer.Option( + None, + "--ossie-expression-dialect", + help="Exact dialect label for SQL synthesized into an Ossie output", + ), + ossie_schema_version: str = typer.Option( + None, + "--ossie-schema-version", + help="Explicit pinned Ossie schema version for synthesized output", + ), + ossie_consumer_profile: str = typer.Option( + None, + "--ossie-consumer-profile", + help="Ossie import or export consumer profile (ossie-core or dbt-1.12)", + ), + ossie_permissive: bool = typer.Option( + False, + "--ossie-permissive", + help="Preserve invalid Ossie source while lowering only independently safe constructs", + ), force: bool = typer.Option(False, "--force", help="Allow writing to an existing destination"), ): """Convert semantic definitions through the shared format registry.""" @@ -1470,11 +1511,11 @@ def convert( source = source if source_from_stdin else _models_path(source) output = output or (_project().root / f"converted.{target_format}.yml") + target_spec = get_semantic_format(target_format, operation="export") if output_to_stdout: - target = get_semantic_format(target_format, operation="export") - if target.output_kind != OutputKind.FILE: + if target_spec.output_kind != OutputKind.FILE: raise InvocationError( - f"Format '{target.name}' produces multiple or shape-dependent files and cannot use --output -" + f"Format '{target_spec.name}' produces multiple or shape-dependent files and cannot use --output -" ) if str(output) != "-" and output.exists() and not force: raise ValueError(f"Destination already exists: {output}; pass --force to replace it") @@ -1496,6 +1537,36 @@ def convert( converted_output = temp_root / f"stdout{suffix}" from sidemantic.fidelity import capture_import_report + source_adapter_options = None + if source_format != "auto" and get_semantic_format(source_format, operation="import").name == "ossie": + source_adapter_options = {} + if ossie_scope is not None: + source_adapter_options["scope_id"] = ossie_scope + if ossie_target_dialect is not None: + source_adapter_options["target_dialect"] = ossie_target_dialect + if ossie_consumer_profile is not None: + source_adapter_options["consumer_profile"] = ossie_consumer_profile + if ossie_permissive: + source_adapter_options["import_policy"] = "permissive" + + target_adapter_options = None + target_export_options = None + if target_spec.name == "ossie": + if ossie_scope is None: + raise InvocationError("--ossie-scope is required when exporting a runtime graph to Ossie") + if ossie_expression_dialect is None: + raise InvocationError( + "--ossie-expression-dialect is required so Sidemantic never relabels SQL as another dialect" + ) + target_export_options = { + "scope_name": ossie_scope, + "expression_dialect": ossie_expression_dialect, + } + if ossie_consumer_profile is not None: + target_adapter_options = {"consumer_profile": ossie_consumer_profile} + if ossie_schema_version is not None: + target_export_options["schema_version"] = ossie_schema_version + with progress(f"Converting semantic definitions to {target_format}"): with capture_import_report() as fidelity_report: graph = convert_semantic_source( @@ -1503,6 +1574,9 @@ def convert( converted_output, source_format=source_format, target_format=target_format, + source_adapter_options=source_adapter_options, + target_adapter_options=target_adapter_options, + target_export_options=target_export_options, ) if output_to_stdout: write_text_output("-", converted_output.read_text()) @@ -1616,6 +1690,7 @@ def query( use_preaggregations: bool = typer.Option( False, "--use-preaggregations", help="Enable automatic pre-aggregation routing" ), + ossie_scope: str = typer.Option(None, "--ossie-scope", help="Explicit Apache Ossie semantic-model scope"), ): """ Execute a SQL query and output results as CSV. @@ -1639,6 +1714,7 @@ def query( use_preaggregations=use_preaggregations, engine=engine, fallback=fallback, + ossie_scope=ossie_scope, ) # Dry run: show generated SQL without executing @@ -2452,6 +2528,7 @@ def validate( connection: str = typer.Option(None, "--connection", help="Database connection string for --live"), db: Path = typer.Option(None, "--db", help="DuckDB database or data file for --live"), json_output: bool = typer.Option(False, "--json", help="Emit the validation report as JSON"), + ossie_scope: str = typer.Option(None, "--ossie-scope", help="Explicit Apache Ossie semantic-model scope"), ): """ Validate semantic layer definitions. @@ -2524,7 +2601,10 @@ def validate( # below; the stdlib warning would print a second, uglier copy. warnings_module.simplefilter("ignore") with capture_import_report() as fidelity_report: - report = validate_directory(directory) + if ossie_scope is None: + report = validate_directory(directory) + else: + report = validate_directory(directory, ossie_scope_id=ossie_scope) for note in fidelity_report.notes: report.warnings.append(f"Import fidelity ({note.severity}): {note.detail}") except Exception as e: @@ -2553,7 +2633,7 @@ def validate( if live: try: _resolve_connection(connection=connection, database=db, models=directory, required=True) - layer = _load_query_layer(directory, connection=connection, db=db) + layer = _load_query_layer(directory, connection=connection, db=db, ossie_scope=ossie_scope) from sidemantic.testing import check_schema_drift with progress("Checking live database schema"): diff --git a/sidemantic/formats.py b/sidemantic/formats.py index a1c0e462c..5915e81b2 100644 --- a/sidemantic/formats.py +++ b/sidemantic/formats.py @@ -7,6 +7,7 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass from enum import Enum from importlib import import_module @@ -53,11 +54,11 @@ def supports_import(self) -> bool: def supports_export(self) -> bool: return self.output_kind is not None - def create_adapter(self) -> Any: + def create_adapter(self, **options: object) -> Any: """Construct this format's adapter without eagerly importing others.""" module = import_module(self.adapter_module) adapter_type = getattr(module, self.adapter_class) - return adapter_type() + return adapter_type(**options) class UnknownFormatError(ValueError): @@ -153,10 +154,10 @@ class UnsupportedFormatOperationError(ValueError): output_kind=OutputKind.DIRECTORY, ), SemanticFormat( - "osi", - "sidemantic.adapters.osi", - "OSIAdapter", - aliases=("open-semantic-interchange",), + "ossie", + "sidemantic.adapters.ossie", + "OssieAdapter", + aliases=("apache-ossie", "osi", "open-semantic-interchange"), extensions=(".yml", ".yaml", ".json"), output_kind=OutputKind.FILE, ), @@ -254,6 +255,7 @@ def load_semantic_source( source: str | Path, *, source_format: str = "auto", + adapter_options: Mapping[str, object] | None = None, ) -> SemanticGraph: """Load one semantic source using auto-discovery or an explicit adapter. @@ -266,6 +268,8 @@ def load_semantic_source( raise FileNotFoundError(f"Semantic source does not exist: {source_path}") if source_format.strip().lower() == "auto": + if adapter_options: + raise ValueError("adapter_options require an explicit source_format") from sidemantic.core.semantic_layer import SemanticLayer from sidemantic.loaders import load_from_directory, load_from_file @@ -280,7 +284,7 @@ def load_semantic_source( _validate_source_kind(source_path, spec) from sidemantic.loaders import parse_with_adapter - return parse_with_adapter(spec.create_adapter(), source_path) + return parse_with_adapter(spec.create_adapter(**dict(adapter_options or {})), source_path) def export_semantic_graph( @@ -288,10 +292,12 @@ def export_semantic_graph( output: str | Path, *, target_format: str = "sidemantic", + adapter_options: Mapping[str, object] | None = None, + export_options: Mapping[str, object] | None = None, ) -> None: """Export a graph through a registered format adapter.""" spec = get_semantic_format(target_format, operation="export") - spec.create_adapter().export(graph, Path(output)) + spec.create_adapter(**dict(adapter_options or {})).export(graph, Path(output), **dict(export_options or {})) def convert_semantic_source( @@ -300,10 +306,19 @@ def convert_semantic_source( *, source_format: str = "auto", target_format: str = "sidemantic", + source_adapter_options: Mapping[str, object] | None = None, + target_adapter_options: Mapping[str, object] | None = None, + target_export_options: Mapping[str, object] | None = None, ) -> SemanticGraph: """Load an exact source, export it, and return the intermediate graph.""" - graph = load_semantic_source(source, source_format=source_format) - export_semantic_graph(graph, output, target_format=target_format) + graph = load_semantic_source(source, source_format=source_format, adapter_options=source_adapter_options) + export_semantic_graph( + graph, + output, + target_format=target_format, + adapter_options=target_adapter_options, + export_options=target_export_options, + ) return graph diff --git a/sidemantic/loaders.py b/sidemantic/loaders.py index 16cbdb7dd..0ed410e81 100644 --- a/sidemantic/loaders.py +++ b/sidemantic/loaders.py @@ -172,6 +172,7 @@ def load_from_directory( *, strict: bool = True, only_file: "Path | None" = None, + ossie_scope_id: str | None = None, ) -> None: """Load all semantic layer definitions from a directory. @@ -200,7 +201,7 @@ def load_from_directory( from sidemantic.adapters.lookml import LookMLAdapter from sidemantic.adapters.metricflow import MetricFlowAdapter from sidemantic.adapters.omni import OmniAdapter - from sidemantic.adapters.osi import OSIAdapter + from sidemantic.adapters.ossie import OssieAdapter from sidemantic.adapters.rill import RillAdapter from sidemantic.adapters.sidemantic import SidemanticAdapter from sidemantic.adapters.snowflake import SnowflakeAdapter @@ -396,7 +397,11 @@ def load_from_directory( _handle_parse_error(file_path, e, strict=strict) continue if is_osi: - adapter = OSIAdapter() + adapter = OssieAdapter( + target_dialect=layer.dialect or "duckdb", + consumer_profile="dbt-1.12", + scope_id=ossie_scope_id, + ) elif suffix == ".aml": from sidemantic.adapters.holistics import HolisticsAdapter @@ -432,10 +437,13 @@ def load_from_directory( pass elif _yaml_has_top_level_key(yaml_data, "semantic_models"): adapter = MetricFlowAdapter() - elif _yaml_has_top_level_key(yaml_data, "semantic_model") and _yaml_has_top_level_key( - yaml_data, "datasets" - ): - adapter = OSIAdapter() + elif _yaml_has_top_level_key(yaml_data, "semantic_model") and _contains_yaml_key(yaml_data, "datasets"): + consumer_profile = "dbt-1.12" if yaml_data.get("version") == "0.1.0" else "ossie-core" + adapter = OssieAdapter( + target_dialect=layer.dialect or "duckdb", + consumer_profile=consumer_profile, + scope_id=ossie_scope_id, + ) elif _yaml_has_top_level_key(yaml_data, "cubes") or ( _yaml_has_top_level_key(yaml_data, "views") and _contains_yaml_key(yaml_data, "measures") ): @@ -585,7 +593,13 @@ def load_from_directory( layer.graph.build_adjacency() -def load_from_file(layer: "SemanticLayer", file: str | Path, *, strict: bool = True) -> None: +def load_from_file( + layer: "SemanticLayer", + file: str | Path, + *, + strict: bool = True, + ossie_scope_id: str | None = None, +) -> None: """Load semantic definitions from a single file, ignoring sibling files. Parses only ``file`` (so an unrelated broken file beside it cannot fail the @@ -603,7 +617,7 @@ def load_from_file(layer: "SemanticLayer", file: str | Path, *, strict: bool = T file = Path(file) if not file.is_file(): raise ValueError(f"File {file} does not exist") - load_from_directory(layer, file.parent, strict=strict, only_file=file) + load_from_directory(layer, file.parent, strict=strict, only_file=file, ossie_scope_id=ossie_scope_id) def _load_graphene_project( diff --git a/sidemantic/man/sidemantic.1 b/sidemantic/man/sidemantic.1 index 5d8f7208b..2e6ec11ca 100644 --- a/sidemantic/man/sidemantic.1 +++ b/sidemantic/man/sidemantic.1 @@ -132,6 +132,9 @@ Examples: sidemantic info sidemantic info ./models .TP \fB\-\-json\fR Emit semantic\-layer metadata as JSON +.TP +\fB\-\-ossie\-scope TEXT\fR +Explicit Apache Ossie semantic\-model scope .SH "SIDEMANTIC REWRITE" .SS SYNOPSIS .B @@ -181,6 +184,24 @@ File extension for stdin input when a format supports multiple syntaxes (for exa \fB\-\-to TEXT\fR Destination format .TP +\fB\-\-ossie\-scope TEXT\fR +Explicit semantic\-model scope to select on Ossie import or name on Ossie export +.TP +\fB\-\-ossie\-target\-dialect TEXT\fR +Execution dialect used when lowering an Ossie source (for example duckdb or bigquery) +.TP +\fB\-\-ossie\-expression\-dialect TEXT\fR +Exact dialect label for SQL synthesized into an Ossie output +.TP +\fB\-\-ossie\-schema\-version TEXT\fR +Explicit pinned Ossie schema version for synthesized output +.TP +\fB\-\-ossie\-consumer\-profile TEXT\fR +Ossie import or export consumer profile (ossie\-core or dbt\-1.12) +.TP +\fB\-\-ossie\-permissive\fR +Preserve invalid Ossie source while lowering only independently safe constructs +.TP \fB\-\-force\fR Allow writing to an existing destination .SH "SIDEMANTIC QUERY" @@ -217,6 +238,9 @@ Allow Rust engine fallback to Python .TP \fB\-\-use\-preaggregations\fR Enable automatic pre\-aggregation routing +.TP +\fB\-\-ossie\-scope TEXT\fR +Explicit Apache Ossie semantic\-model scope .SH "SIDEMANTIC EXPLAIN" .SS SYNOPSIS .B @@ -320,6 +344,9 @@ DuckDB database or data file for \-\-live .TP \fB\-\-json\fR Emit the validation report as JSON +.TP +\fB\-\-ossie\-scope TEXT\fR +Explicit Apache Ossie semantic\-model scope .SH "SIDEMANTIC WORKBENCH" .SS SYNOPSIS .B diff --git a/sidemantic/validation_runner.py b/sidemantic/validation_runner.py index b8e262deb..8846a793f 100644 --- a/sidemantic/validation_runner.py +++ b/sidemantic/validation_runner.py @@ -34,13 +34,13 @@ def _find_orphaned_models(models: dict[str, object]) -> list[str]: ] -def validate_directory(directory: str | Path) -> ValidationReport: +def validate_directory(directory: str | Path, *, ossie_scope_id: str | None = None) -> ValidationReport: """Load and validate semantic layer definitions from a directory.""" directory = Path(directory) report = ValidationReport(directory=directory) layer = SemanticLayer() - load_from_directory(layer, str(directory)) + load_from_directory(layer, str(directory), ossie_scope_id=ossie_scope_id) if not layer.graph.models: report.errors.append("No models found in directory") diff --git a/tests/adapters/osi/conftest.py b/tests/adapters/osi/conftest.py index 76d0e69b2..a42fea871 100644 --- a/tests/adapters/osi/conftest.py +++ b/tests/adapters/osi/conftest.py @@ -17,7 +17,7 @@ import pytest -from sidemantic.adapters.osi import OSIAdapter as _PythonOSIAdapter +from sidemantic.adapters.osi import LegacyOSIAdapter as _PythonOSIAdapter from sidemantic.core.semantic_graph import SemanticGraph diff --git a/tests/adapters/osi/test_ossie_adapter.py b/tests/adapters/osi/test_ossie_adapter.py new file mode 100644 index 000000000..07b582b5e --- /dev/null +++ b/tests/adapters/osi/test_ossie_adapter.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from sidemantic.adapters.osi import OSIAdapter +from sidemantic.adapters.ossie import OssieAdapter, OssieImportError +from sidemantic.core.dimension import Dimension +from sidemantic.core.model import Model +from sidemantic.core.relationship import Relationship +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.interchange.ossie import OssieConsumerProfile, OssieSynthesisError + +VALID_DOCUMENT = """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} +""" + + +def test_historical_osi_class_spelling_routes_to_canonical_adapter(tmp_path: Path) -> None: + assert issubclass(OSIAdapter, OssieAdapter) + assert OSIAdapter.SUPPORTED_EXPORT_DIALECTS == ["ANSI_SQL", "BIGQUERY", "SNOWFLAKE", "DATABRICKS"] + invalid = tmp_path / "invalid.yaml" + invalid.write_text("version: 0.2.0.dev0\nsemantic_model: {}\n") + with pytest.raises(OssieImportError): + OSIAdapter().parse(invalid) + + +ONTOLOGY_DOCUMENT = """# ontology source +version: 0.2.0.dev0 +name: business_terms +description: Shared business vocabulary +ai_context: + instructions: Use the business vocabulary when answering questions. +ontology: + - concept: Customer + type: EntityType + - concept: Order + type: EntityType +ontology_mappings: + - semantic_model: + name: commerce + datasets: + - name: customers + source: analytics.customers + concept_mappings: + - concept: Customer +""" + + +def test_parse_document_returns_preserved_source_and_scoped_catalog(tmp_path: Path) -> None: + source = tmp_path / "commerce.yaml" + source.write_text(VALID_DOCUMENT) + + result = OssieAdapter(preserve_source=True).parse_document(source) + + assert result.document.source.original_bytes == source.read_bytes() + assert result.catalog.scope_ids == ("commerce",) + assert result.catalog["commerce"].graph.get_model("orders").primary_key == "id" + + +def test_legacy_parse_surface_requires_scope_selection_when_ambiguous(tmp_path: Path) -> None: + source = tmp_path / "multiple.yaml" + source.write_text( + VALID_DOCUMENT + + """ - name: operations + datasets: + - name: orders + source: operations.orders + fields: [] +""" + ) + + with pytest.raises(OssieImportError, match="Select one explicitly"): + OssieAdapter().parse(source) + + graph = OssieAdapter(scope_id="operations").parse(source) + assert graph.get_model("orders").table == "operations.orders" + + +def test_directory_scopes_are_qualified_and_generated_directories_are_skipped(tmp_path: Path) -> None: + first = tmp_path / "one.yaml" + second = tmp_path / "nested" / "two.json" + generated = tmp_path / "target" / "generated.yaml" + first.write_text(VALID_DOCUMENT) + second.parent.mkdir() + second.write_text( + '{"version":"0.2.0.dev0","semantic_model":[{"name":"commerce","datasets":' + '[{"name":"orders","source":"nested.orders","fields":[]}]}]}' + ) + generated.parent.mkdir() + generated.write_text(VALID_DOCUMENT) + + project = OssieAdapter().parse_catalog(tmp_path) + + assert project.catalog.scope_ids == ("nested/two.json::commerce", "one.yaml::commerce") + + +def test_validation_failure_is_not_silently_lowered_through_legacy_surface(tmp_path: Path) -> None: + source = tmp_path / "invalid.yaml" + source.write_text("version: 0.2.0.dev0\nsemantic_model: {}\n") + + with pytest.raises(OssieImportError) as error: + OssieAdapter().parse(source) + + assert any(diagnostic.code == "ossie.schema.type" for diagnostic in error.value.diagnostics) + + +def test_ontology_only_document_reports_no_executable_scope(tmp_path: Path) -> None: + source = tmp_path / "ontology.yaml" + source.write_text(ONTOLOGY_DOCUMENT) + + with pytest.raises(OssieImportError, match="contains no scopes"): + OssieAdapter().parse(source) + + +def test_ontology_document_canonical_serialization_preserves_ontology_data(tmp_path: Path) -> None: + source = tmp_path / "ontology.yaml" + output = tmp_path / "ontology.json" + source.write_text(ONTOLOGY_DOCUMENT) + expected = yaml.safe_load(ONTOLOGY_DOCUMENT) + + lowered = OssieAdapter(preserve_source=True).parse_document(source) + result = OssieAdapter().export_document(lowered, output, serialization="json") + + assert not lowered.executable + assert lowered.catalog.scope_ids == () + assert result.serialization.value == "json" + assert not result.exact_source_reused + canonical = json.loads(result.data) + assert canonical["ontology"] == expected["ontology"] + assert canonical["ontology_mappings"] == expected["ontology_mappings"] + assert canonical == expected + + +def test_ontology_document_exact_source_round_trip_reuses_original_bytes(tmp_path: Path) -> None: + source = tmp_path / "ontology.yaml" + output = tmp_path / "ontology-copy.yaml" + source_bytes = ONTOLOGY_DOCUMENT.encode() + source.write_bytes(source_bytes) + + lowered = OssieAdapter(preserve_source=True).parse_document(source) + result = OssieAdapter().export_document(lowered, output, exact_source=True) + + assert not lowered.executable + assert lowered.catalog.scope_ids == () + assert lowered.document.ontology is not None + assert lowered.document.ontology_mappings is not None + assert result.exact_source_reused + assert result.data == source_bytes + assert output.read_bytes() == source_bytes + + +def test_export_document_reuses_exact_validated_source_bytes(tmp_path: Path) -> None: + source = tmp_path / "source.yaml" + output = tmp_path / "output.yaml" + source.write_text("# retained\n" + VALID_DOCUMENT) + lowered = OssieAdapter(preserve_source=True).parse_document(source) + + result = OssieAdapter().export_document(lowered, output, exact_source=True) + + assert result.exact_source_reused + assert output.read_bytes() == source.read_bytes() + + +def test_dbt_alias_profile_survives_parse_lower_and_document_export(tmp_path: Path) -> None: + source = tmp_path / "dbt.json" + exact_output = tmp_path / "exact.json" + canonical_output = tmp_path / "canonical.json" + source.write_bytes(b'{"version":"0.1.0","semantic_model":[]}\n') + adapter = OssieAdapter(consumer_profile=OssieConsumerProfile.DBT_1_12, preserve_source=True) + + lowered = adapter.parse_document(source) + exact = adapter.export_document(lowered, exact_output, exact_source=True) + canonical = adapter.export_document(lowered, canonical_output) + + assert lowered.valid + assert exact.exact_source_reused + assert exact_output.read_bytes() == source.read_bytes() + assert yaml.safe_load(canonical.data)["version"] == "0.1.0" + + +def test_graph_export_requires_explicit_scope_and_dialect(tmp_path: Path) -> None: + graph = SemanticGraph() + graph.add_model( + Model( + name="orders", + table="analytics.orders", + dimensions=[Dimension(name="id", type="numeric", sql="id")], + ) + ) + + with pytest.raises(ValueError, match="scope_name"): + OssieAdapter().export(graph, tmp_path / "model.yaml") + + output = tmp_path / "model.yaml" + OssieAdapter(export_scope_name="commerce", expression_dialect="BIGQUERY").export(graph, output) + data = yaml.safe_load(output.read_text()) + dialect = data["semantic_model"][0]["datasets"][0]["fields"][0]["expression"]["dialects"][0] + assert dialect == {"dialect": "BIGQUERY", "expression": "id"} + + +def test_graph_export_refuses_unidentified_relationships(tmp_path: Path) -> None: + graph = SemanticGraph() + graph.add_model(Model(name="orders", table="orders")) + graph.add_model(Model(name="customers", table="customers", primary_key="id")) + graph.models["orders"].relationships.append( + Relationship(name="customers", type="many_to_one", foreign_key="customer_id", primary_key="id") + ) + + with pytest.raises(OssieSynthesisError): + OssieAdapter(export_scope_name="commerce", expression_dialect="ANSI_SQL").export(graph, tmp_path / "model.yaml") diff --git a/tests/adapters/osi/test_parsing.py b/tests/adapters/osi/test_parsing.py index 9f5d8b2f0..c3dfdadc0 100644 --- a/tests/adapters/osi/test_parsing.py +++ b/tests/adapters/osi/test_parsing.py @@ -7,7 +7,7 @@ import pytest import yaml -from sidemantic.adapters.osi import OSIAdapter +from sidemantic.adapters.osi import LegacyOSIAdapter as OSIAdapter from sidemantic.core.dimension import Dimension from sidemantic.core.metric import Metric from sidemantic.core.model import Model diff --git a/tests/adapters/osi/test_query_execution.py b/tests/adapters/osi/test_query_execution.py index 548e0643f..b00baab88 100644 --- a/tests/adapters/osi/test_query_execution.py +++ b/tests/adapters/osi/test_query_execution.py @@ -6,7 +6,7 @@ import pytest from sidemantic import SemanticLayer -from sidemantic.adapters.osi import OSIAdapter +from sidemantic.adapters.osi import LegacyOSIAdapter as OSIAdapter def _load_layer_from_graph(graph, conn: duckdb.DuckDBPyConnection) -> SemanticLayer: diff --git a/tests/adapters/osi/test_rust_parity.py b/tests/adapters/osi/test_rust_parity.py index c5b7cab2f..2cd061746 100644 --- a/tests/adapters/osi/test_rust_parity.py +++ b/tests/adapters/osi/test_rust_parity.py @@ -14,7 +14,7 @@ pytest.importorskip("sidemantic_rs") -from sidemantic.adapters.osi import OSIAdapter as PyOSIAdapter +from sidemantic.adapters.osi import LegacyOSIAdapter as PyOSIAdapter from sidemantic.rust_bridge import export_osi_with_rust, load_osi_graph_with_rust FIXTURE_DIR = Path("tests/fixtures/osi") diff --git a/tests/adapters/test_added_fixture_coverage.py b/tests/adapters/test_added_fixture_coverage.py index c9a0b16af..0e87d88f3 100644 --- a/tests/adapters/test_added_fixture_coverage.py +++ b/tests/adapters/test_added_fixture_coverage.py @@ -20,7 +20,7 @@ from sidemantic.adapters.malloy import MalloyAdapter from sidemantic.adapters.metricflow import MetricFlowAdapter from sidemantic.adapters.omni import OmniAdapter -from sidemantic.adapters.osi import OSIAdapter +from sidemantic.adapters.osi import LegacyOSIAdapter as OSIAdapter from sidemantic.adapters.rill import RillAdapter from sidemantic.adapters.snowflake import SnowflakeAdapter from sidemantic.adapters.superset import SupersetAdapter diff --git a/tests/adapters/test_fixture_functionality_contracts.py b/tests/adapters/test_fixture_functionality_contracts.py index 0ad940082..fabd6db9c 100644 --- a/tests/adapters/test_fixture_functionality_contracts.py +++ b/tests/adapters/test_fixture_functionality_contracts.py @@ -19,7 +19,7 @@ from sidemantic.adapters.malloy import MalloyAdapter from sidemantic.adapters.metricflow import MetricFlowAdapter from sidemantic.adapters.omni import OmniAdapter -from sidemantic.adapters.osi import OSIAdapter +from sidemantic.adapters.osi import LegacyOSIAdapter as OSIAdapter from sidemantic.adapters.rill import RillAdapter from sidemantic.adapters.snowflake import SnowflakeAdapter from sidemantic.adapters.superset import SupersetAdapter @@ -176,7 +176,7 @@ "MalloyAdapter": "tests/fixtures/malloy/ga4.malloy", "MetricFlowAdapter": "tests/fixtures/metricflow/accounts_source.yml", "OmniAdapter": "tests/fixtures/omni/estore/views/dim_categories.view.yaml", - "OSIAdapter": "tests/fixtures/osi/ecommerce.yaml", + "LegacyOSIAdapter": "tests/fixtures/osi/ecommerce.yaml", "RillAdapter": "tests/fixtures/rill/cost_monitoring.yaml", "SnowflakeAdapter": "tests/fixtures/snowflake/customer_loyalty_metrics.yaml", "SupersetAdapter": "tests/fixtures/superset/covid_dashboard.yaml", diff --git a/tests/core/test_directory_loaders.py b/tests/core/test_directory_loaders.py index ce6ea5726..d1dd23450 100644 --- a/tests/core/test_directory_loaders.py +++ b/tests/core/test_directory_loaders.py @@ -1,5 +1,7 @@ """Tests for auto-discovery loaders.""" +import pytest + from sidemantic import SemanticLayer from sidemantic.loaders import load_from_directory @@ -494,10 +496,28 @@ def test_load_from_directory_detects_released_osi_json(tmp_path): assert "orders" in layer.graph.models orders = layer.graph.models["orders"] assert orders.table.endswith("fct_orders") - assert getattr(orders, "_source_format", None) == "OSI" + assert getattr(orders, "_source_format", None) == "Ossie" assert "order_count" in layer.graph.metrics +def test_auto_loader_rejects_multi_scope_ossie_instead_of_flattening_namespaces(tmp_path): + source = tmp_path / "multiple.yaml" + source.write_text( + """version: 0.2.0.dev0 +semantic_model: + - name: finance + datasets: + - {name: orders, source: finance.orders} + - name: marketing + datasets: + - {name: orders, source: marketing.orders} +""" + ) + + with pytest.raises(ValueError, match="ambiguous.*Select one explicitly"): + load_from_directory(SemanticLayer(), tmp_path) + + def test_load_from_directory_accepts_osi_dir_as_loader_root(tmp_path): """Pointing the loader straight at the ``OSI/`` directory routes its JSON to OSI. @@ -547,7 +567,7 @@ def test_load_from_directory_accepts_osi_dir_as_loader_root(tmp_path): assert "orders" in layer.graph.models orders = layer.graph.models["orders"] assert orders.table.endswith("fct_orders") - assert getattr(orders, "_source_format", None) == "OSI" + assert getattr(orders, "_source_format", None) == "Ossie" assert "order_count" in layer.graph.metrics @@ -601,7 +621,7 @@ def test_load_from_directory_accepts_osi_dir_as_cwd_loader_root(tmp_path, monkey assert "orders" in layer.graph.models orders = layer.graph.models["orders"] assert orders.table.endswith("fct_orders") - assert getattr(orders, "_source_format", None) == "OSI" + assert getattr(orders, "_source_format", None) == "Ossie" assert "order_count" in layer.graph.metrics diff --git a/tests/test_cli_contract.py b/tests/test_cli_contract.py index d932e835c..8b850c7e8 100644 --- a/tests/test_cli_contract.py +++ b/tests/test_cli_contract.py @@ -278,6 +278,147 @@ def test_convert_preserves_or_infers_sql_extension_for_stdin(extension_arguments assert not result.stderr +def test_convert_to_ossie_requires_explicit_scope_and_expression_dialect(tmp_path: Path): + source = tmp_path / "source.yml" + output = tmp_path / "output.yml" + source.write_text( + """version: 1 +models: + - name: orders + table: analytics.orders +""" + ) + + missing = runner.invoke( + app, + ["convert", str(source), "--from", "sidemantic", "--to", "ossie", "--output", str(output)], + ) + assert missing.exit_code == 2 + assert "--ossie-scope is required" in missing.stderr + + converted = runner.invoke( + app, + [ + "convert", + str(source), + "--from", + "sidemantic", + "--to", + "ossie", + "--output", + str(output), + "--ossie-scope", + "commerce", + "--ossie-expression-dialect", + "BIGQUERY", + ], + ) + assert converted.exit_code == 0, converted.output + assert "name: commerce" in output.read_text() + + +def test_convert_to_dbt_alias_carries_explicit_consumer_profile(tmp_path: Path): + source = tmp_path / "source.yml" + output = tmp_path / "output.json" + source.write_text( + """version: 1 +models: + - name: orders + table: analytics.orders +""" + ) + + result = runner.invoke( + app, + [ + "convert", + str(source), + "--from", + "sidemantic", + "--to", + "ossie", + "--output", + str(output), + "--ossie-scope", + "commerce", + "--ossie-expression-dialect", + "ANSI_SQL", + "--ossie-schema-version", + "0.1.0", + "--ossie-consumer-profile", + "dbt-1.12", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(output.read_text())["version"] == "0.1.0" + + +def test_convert_from_ossie_selects_scope_and_target_dialect(tmp_path: Path): + source = tmp_path / "source.ossie.yaml" + output = tmp_path / "output.yml" + source.write_text( + """version: 0.2.0.dev0 +semantic_model: + - name: finance + datasets: + - name: orders + source: finance.orders + - name: marketing + datasets: + - name: orders + source: marketing.orders +""" + ) + + result = runner.invoke( + app, + [ + "convert", + str(source), + "--from", + "ossie", + "--to", + "sidemantic", + "--output", + str(output), + "--ossie-scope", + "marketing", + "--ossie-target-dialect", + "duckdb", + ], + ) + + assert result.exit_code == 0, result.output + assert "marketing.orders" in output.read_text() + + +def test_info_and_validate_can_select_a_multi_scope_ossie_document(tmp_path: Path): + source = tmp_path / "multiple.ossie.yaml" + source.write_text( + """version: 0.2.0.dev0 +semantic_model: + - name: finance + datasets: + - {name: orders, source: finance.orders} + - name: marketing + datasets: + - {name: orders, source: marketing.orders} +""" + ) + + ambiguous = runner.invoke(app, ["info", str(tmp_path), "--json"]) + assert ambiguous.exit_code == 1 + assert "ambiguous" in ambiguous.stderr + + selected = runner.invoke(app, ["info", str(tmp_path), "--json", "--ossie-scope", "marketing"]) + assert selected.exit_code == 0, selected.output + assert json.loads(selected.stdout)["models"][0]["table"] == "marketing.orders" + + validated = runner.invoke(app, ["validate", str(tmp_path), "--json", "--ossie-scope", "marketing"]) + assert validated.exit_code == 0, validated.output + + def test_generated_output_dash_writes_stdout(tmp_path: Path): models = tmp_path / "models" _write_model(models) diff --git a/tests/test_formats.py b/tests/test_formats.py index 0d58094cd..faf8d808d 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -7,6 +7,7 @@ UnknownFormatError, UnsupportedFormatOperationError, convert_semantic_source, + export_semantic_graph, get_semantic_format, load_semantic_source, semantic_formats, @@ -33,6 +34,10 @@ def test_registry_has_stable_names_aliases_and_capabilities(): assert get_semantic_format("native").name == "sidemantic" assert get_semantic_format("cube_js").name == "cube" assert get_semantic_format("powerbi").name == "tmdl" + assert get_semantic_format("ossie").name == "ossie" + assert get_semantic_format("apache_ossie").name == "ossie" + assert get_semantic_format("osi").name == "ossie" + assert get_semantic_format("open-semantic-interchange").name == "ossie" assert get_semantic_format("rill").output_kind == OutputKind.DIRECTORY assert get_semantic_format("tableau").supports_export is False @@ -79,3 +84,62 @@ def test_convert_auto_file_to_native_yaml(tmp_path: Path): assert set(graph.models) == {"orders"} assert "version: 1" in output.read_text() assert "name: orders" in output.read_text() + + +def test_explicit_ossie_format_uses_scoped_validated_importer(tmp_path: Path): + source = tmp_path / "orders.ossie.yaml" + source.write_text( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders +""" + ) + + graph = load_semantic_source( + source, + source_format="ossie", + adapter_options={"scope_id": "commerce", "target_dialect": "duckdb"}, + ) + + assert graph.get_model("orders").table == "analytics.orders" + + +def test_ossie_graph_export_requires_and_accepts_explicit_synthesis_options(tmp_path: Path): + source = tmp_path / "source.yml" + output = tmp_path / "output.json" + source.write_text(_native_model("orders")) + graph = load_semantic_source(source, source_format="native") + + with pytest.raises(ValueError, match="scope_name"): + export_semantic_graph(graph, output, target_format="ossie") + + export_semantic_graph( + graph, + output, + target_format="ossie", + export_options={"scope_name": "commerce", "expression_dialect": "ANSI_SQL"}, + ) + + assert '"version": "0.2.0.dev0"' in output.read_text() + assert '"name": "commerce"' in output.read_text() + + +def test_convert_plumbs_explicit_ossie_export_options(tmp_path: Path): + source = tmp_path / "source.yml" + output = tmp_path / "output.yaml" + source.write_text(_native_model("orders")) + + convert_semantic_source( + source, + output, + source_format="native", + target_format="ossie", + target_export_options={"scope_name": "commerce", "expression_dialect": "SNOWFLAKE"}, + ) + + text = output.read_text() + assert "version: 0.2.0.dev0" in text + assert "dialect: SNOWFLAKE" in text