From ccadaf862ad1407d4d17bb73e5cd56b169ca404f Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 16 Jul 2026 10:18:56 +0200 Subject: [PATCH 01/13] Initial development of resource-info-fetcher --- Cargo.lock | 46 +++++ Cargo.toml | 1 + Tiltfile | 54 ------ _TEST/clientId | 1 + _TEST/clientSecret | 1 + _TEST/rif.json | 8 + rust/info-fetcher-commons/Cargo.toml | 22 +++ rust/info-fetcher-commons/src/config.rs | 33 ++++ .../src/http_error.rs | 0 rust/info-fetcher-commons/src/lib.rs | 3 + .../src/utils/http.rs | 0 .../src/utils/mod.rs | 0 .../src/utils/tls.rs | 0 rust/operator-binary/src/crd/cache.rs | 28 +++ rust/operator-binary/src/crd/mod.rs | 2 + .../src/crd/resource_info_fetcher/mod.rs | 50 +++++ .../src/crd/user_info_fetcher/mod.rs | 17 +- .../crd/user_info_fetcher/v1alpha1_impl.rs | 18 +- .../crd/user_info_fetcher/v1alpha2_impl.rs | 18 +- rust/resource-info-fetcher/Cargo.toml | 31 +++ rust/resource-info-fetcher/build.rs | 3 + rust/resource-info-fetcher/src/api.rs | 66 +++++++ .../src/backend/data_hub/mod.rs | 179 +++++++++++++++++ .../data_hub/resource_to_urn_mapping.rs | 17 ++ .../src/backend/data_hub/upstream_api.rs | 79 ++++++++ rust/resource-info-fetcher/src/backend/mod.rs | 1 + rust/resource-info-fetcher/src/main.rs | 180 ++++++++++++++++++ rust/user-info-fetcher/Cargo.toml | 1 + .../src/backend/active_directory.rs | 3 +- rust/user-info-fetcher/src/backend/entra.rs | 27 ++- .../user-info-fetcher/src/backend/keycloak.rs | 14 +- .../user-info-fetcher/src/backend/openldap.rs | 3 +- .../user-info-fetcher/src/backend/xfsc_aas.rs | 5 +- rust/user-info-fetcher/src/main.rs | 30 +-- 34 files changed, 790 insertions(+), 151 deletions(-) delete mode 100644 Tiltfile create mode 100644 _TEST/clientId create mode 100644 _TEST/clientSecret create mode 100644 _TEST/rif.json create mode 100644 rust/info-fetcher-commons/Cargo.toml create mode 100644 rust/info-fetcher-commons/src/config.rs rename rust/{user-info-fetcher => info-fetcher-commons}/src/http_error.rs (100%) create mode 100644 rust/info-fetcher-commons/src/lib.rs rename rust/{user-info-fetcher => info-fetcher-commons}/src/utils/http.rs (100%) rename rust/{user-info-fetcher => info-fetcher-commons}/src/utils/mod.rs (100%) rename rust/{user-info-fetcher => info-fetcher-commons}/src/utils/tls.rs (100%) create mode 100644 rust/operator-binary/src/crd/cache.rs create mode 100644 rust/operator-binary/src/crd/resource_info_fetcher/mod.rs create mode 100644 rust/resource-info-fetcher/Cargo.toml create mode 100644 rust/resource-info-fetcher/build.rs create mode 100644 rust/resource-info-fetcher/src/api.rs create mode 100644 rust/resource-info-fetcher/src/backend/data_hub/mod.rs create mode 100644 rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs create mode 100644 rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs create mode 100644 rust/resource-info-fetcher/src/backend/mod.rs create mode 100644 rust/resource-info-fetcher/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index b61e7a3c..4efafe03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1607,6 +1607,22 @@ dependencies = [ "rustversion", ] +[[package]] +name = "info-fetcher-commons" +version = "0.0.0-dev" +dependencies = [ + "axum", + "hyper", + "native-tls", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "snafu 0.9.1", + "stackable-operator", + "tokio", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -3439,6 +3455,29 @@ dependencies = [ name = "stackable-opa-regorule-library" version = "0.0.0-dev" +[[package]] +name = "stackable-opa-resource-info-fetcher" +version = "0.0.0-dev" +dependencies = [ + "axum", + "built", + "clap", + "futures", + "hyper", + "info-fetcher-commons", + "moka", + "reqwest", + "serde", + "serde_json", + "snafu 0.9.1", + "stackable-opa-operator", + "stackable-operator", + "tokio", + "tracing", + "url", + "urlencoding", +] + [[package]] name = "stackable-opa-user-info-fetcher" version = "0.0.0-dev" @@ -3450,6 +3489,7 @@ dependencies = [ "clap", "futures", "hyper", + "info-fetcher-commons", "krb5", "ldap3", "moka", @@ -4234,6 +4274,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index c7557a50..32cef8af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ tar = "0.4" tokio = { version = "1.52", features = ["full"] } tracing = "0.1" url = "2.5" +urlencoding = "2.1" uuid = "1.10" [patch."https://github.com/stackabletech/operator-rs.git"] diff --git a/Tiltfile b/Tiltfile deleted file mode 100644 index 1859de01..00000000 --- a/Tiltfile +++ /dev/null @@ -1,54 +0,0 @@ -# Load the metadata first, so that we immediately get access to the operator name -meta = read_json('nix/meta.json') -operator_name = meta['operator']['name'] - -# If tilt_options.json exists read it and load the default_registry, default_operator_repository, -# and default_product_repository value from it -settings = read_json('tilt_options.json', default={}) -registry = settings.get('default_registry', 'oci.stackable.tech') - -# Construct the operator image repository. It uses the "sandbox" instead of the "sdp" namespace to -# separate "testing/local" images from official (published) images. -operator_repository = settings.get('default_operator_repository', registry + '/' + 'sandbox') -operator_image_name = operator_repository + '/' + operator_name - -# For the product image, we want to use the images in the "sdp" namespace, because "sandbox" doesn't -# contain those images (by default). -product_repository = settings.get('default_product_repository', registry + '/' + 'sdp') -# Configure default registry either read from config file above, or with default value of -# "oci.stackable.tech" -default_registry(registry) - -custom_build( - operator_image_name, - 'make regenerate-nix && nix-build . -A docker --argstr dockerName "' + operator_image_name + '" && ./result/load-image | docker load', - deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], - ignore=['*.~undo-tree~'], - # ignore=['result*', 'Cargo.nix', 'target', *.yaml], - outputs_image_ref_to='result/ref', -) - -# We need to set the correct image annotation on the operator Deployment to use e.g. -# oci.stackable.tech/sandbox/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of -# oci.stackable.tech/sandbox/opa-operator:0.0.0-dev (which does not exist) -k8s_kind('Deployment', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') -k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') - -# Optionally specify a custom Helm values file to be passed to the Helm deployment below. -# This file can for example be used to set custom telemetry options (like log level) which is not -# supported by helm(set). -helm_values = settings.get('helm_values', None) - -helm_override_operator_image_repository = 'image.repository=' + operator_repository -helm_override_product_image_repository = 'image.productRepository=' + product_repository - -k8s_yaml(helm( - 'deploy/helm/' + operator_name, - name=operator_name, - namespace="stackable-operators", - set=[ - helm_override_operator_image_repository, - helm_override_product_image_repository, - ], - values=helm_values, -)) diff --git a/_TEST/clientId b/_TEST/clientId new file mode 100644 index 00000000..f8f9bdf0 --- /dev/null +++ b/_TEST/clientId @@ -0,0 +1 @@ +__datahub_system \ No newline at end of file diff --git a/_TEST/clientSecret b/_TEST/clientSecret new file mode 100644 index 00000000..414890cd --- /dev/null +++ b/_TEST/clientSecret @@ -0,0 +1 @@ +datahub-demo-system-secret \ No newline at end of file diff --git a/_TEST/rif.json b/_TEST/rif.json new file mode 100644 index 00000000..189dc7c2 --- /dev/null +++ b/_TEST/rif.json @@ -0,0 +1,8 @@ +{ + "backend": { + "dataHub": { + "hostname": "localhost", + "port": 8080 + } + } +} diff --git a/rust/info-fetcher-commons/Cargo.toml b/rust/info-fetcher-commons/Cargo.toml new file mode 100644 index 00000000..8949798e --- /dev/null +++ b/rust/info-fetcher-commons/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "info-fetcher-commons" +description = "Common shared code between info fetchers" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +publish = false + +[dependencies] +stackable-operator.workspace = true + +axum.workspace = true +hyper.workspace = true +native-tls.workspace = true +reqwest.workspace = true +rustls-pki-types.workspace = true +serde.workspace = true +serde_json.workspace = true +snafu.workspace = true +tokio.workspace = true diff --git a/rust/info-fetcher-commons/src/config.rs b/rust/info-fetcher-commons/src/config.rs new file mode 100644 index 00000000..33633860 --- /dev/null +++ b/rust/info-fetcher-commons/src/config.rs @@ -0,0 +1,33 @@ +use std::{ + fs::File, + io::BufReader, + path::{Path, PathBuf}, +}; + +use serde::de::DeserializeOwned; +use snafu::{ResultExt, Snafu}; + +#[derive(Snafu, Debug)] +pub enum ConfigError { + #[snafu(display("failed to open config file from {path:?}"))] + OpenFile { + source: std::io::Error, + path: PathBuf, + }, + + #[snafu(display("unable to read config file from {path:?}"))] + ParseConfigFile { + source: serde_json::Error, + path: PathBuf, + }, +} + +pub fn read_config_file(path: &Path) -> Result +where + C: DeserializeOwned, +{ + let file = File::open(path).with_context(|_| OpenFileSnafu { path })?; + let reader = BufReader::new(file); + + serde_json::from_reader(reader).with_context(|_| ParseConfigFileSnafu { path }) +} diff --git a/rust/user-info-fetcher/src/http_error.rs b/rust/info-fetcher-commons/src/http_error.rs similarity index 100% rename from rust/user-info-fetcher/src/http_error.rs rename to rust/info-fetcher-commons/src/http_error.rs diff --git a/rust/info-fetcher-commons/src/lib.rs b/rust/info-fetcher-commons/src/lib.rs new file mode 100644 index 00000000..143215d6 --- /dev/null +++ b/rust/info-fetcher-commons/src/lib.rs @@ -0,0 +1,3 @@ +pub mod config; +pub mod http_error; +pub mod utils; diff --git a/rust/user-info-fetcher/src/utils/http.rs b/rust/info-fetcher-commons/src/utils/http.rs similarity index 100% rename from rust/user-info-fetcher/src/utils/http.rs rename to rust/info-fetcher-commons/src/utils/http.rs diff --git a/rust/user-info-fetcher/src/utils/mod.rs b/rust/info-fetcher-commons/src/utils/mod.rs similarity index 100% rename from rust/user-info-fetcher/src/utils/mod.rs rename to rust/info-fetcher-commons/src/utils/mod.rs diff --git a/rust/user-info-fetcher/src/utils/tls.rs b/rust/info-fetcher-commons/src/utils/tls.rs similarity index 100% rename from rust/user-info-fetcher/src/utils/tls.rs rename to rust/info-fetcher-commons/src/utils/tls.rs diff --git a/rust/operator-binary/src/crd/cache.rs b/rust/operator-binary/src/crd/cache.rs new file mode 100644 index 00000000..fae0f2c2 --- /dev/null +++ b/rust/operator-binary/src/crd/cache.rs @@ -0,0 +1,28 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + schemars::{self, JsonSchema}, + shared::time::Duration, +}; + +/// Default time-to-live for cached responses. +pub(crate) const DEFAULT_CACHE_ENTRY_TIME_TO_LIVE: Duration = Duration::from_minutes_unchecked(1); + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Cache { + /// How long responses should be cached for. + #[serde(default = "default_entry_time_to_live")] + pub entry_time_to_live: Duration, +} + +impl Default for Cache { + fn default() -> Self { + Self { + entry_time_to_live: DEFAULT_CACHE_ENTRY_TIME_TO_LIVE, + } + } +} + +const fn default_entry_time_to_live() -> Duration { + DEFAULT_CACHE_ENTRY_TIME_TO_LIVE +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1c090312..fb6af2da 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -27,6 +27,8 @@ use stackable_operator::{ }; use strum::{Display, EnumIter, EnumString}; +pub mod cache; +pub mod resource_info_fetcher; pub mod user_info_fetcher; pub const APP_NAME: &str = "opa"; diff --git a/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs b/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs new file mode 100644 index 00000000..3755e8ed --- /dev/null +++ b/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs @@ -0,0 +1,50 @@ +use crate::crd::cache::Cache; +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + commons::{networking::HostName, tls_verification::TlsClientDetails}, + schemars::{self, JsonSchema}, + versioned::versioned, +}; + +#[versioned(version(name = "v1alpha1"))] +pub mod versioned { + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct Config { + /// The backend directory service to use. + pub backend: Backend, + + /// Caching configuration. + #[serde(default)] + pub cache: Cache, + } + + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub enum Backend { + /// Backend that fetches resource information from DataHub. + DataHub(DataHubBackend), + } + + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct DataHubBackend { + /// Hostname of DataHub + pub hostname: HostName, + + /// Port of DataHub. If TLS is used defaults to `443`, otherwise to `80`. + pub port: Option, + + /// Use a TLS connection. If not specified then no TLS will be used. + #[serde(flatten)] + pub tls: TlsClientDetails, + + /// The env in DataHub, defaults to `PROD` + #[serde(default = "default_data_hub_env")] + pub env: String, + } +} + +fn default_data_hub_env() -> String { + "PROD".to_owned() +} diff --git a/rust/operator-binary/src/crd/user_info_fetcher/mod.rs b/rust/operator-binary/src/crd/user_info_fetcher/mod.rs index 085844a9..706dfc7f 100644 --- a/rust/operator-binary/src/crd/user_info_fetcher/mod.rs +++ b/rust/operator-binary/src/crd/user_info_fetcher/mod.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, str::FromStr}; +use crate::crd::cache::Cache; use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::{ @@ -8,7 +9,6 @@ use stackable_operator::{ tls_verification::{CaCert, Tls, TlsClientDetails, TlsServerVerification, TlsVerification}, }, schemars::{self, JsonSchema}, - shared::time::Duration, v2::types::kubernetes::{SecretClassName, SecretName}, versioned::versioned, }; @@ -208,21 +208,6 @@ pub mod versioned { #[serde(default)] pub custom_attribute_mappings: BTreeMap, } - - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] - #[serde(rename_all = "camelCase")] - pub struct Cache { - /// How long metadata about each user should be cached for. - #[serde(default = "default_entry_time_to_live")] - pub entry_time_to_live: Duration, - } -} - -/// Default time-to-live for cached user metadata. -pub(crate) const DEFAULT_CACHE_ENTRY_TIME_TO_LIVE: Duration = Duration::from_minutes_unchecked(1); - -const fn default_entry_time_to_live() -> Duration { - DEFAULT_CACHE_ENTRY_TIME_TO_LIVE } fn default_root_path() -> String { diff --git a/rust/operator-binary/src/crd/user_info_fetcher/v1alpha1_impl.rs b/rust/operator-binary/src/crd/user_info_fetcher/v1alpha1_impl.rs index 7ebb1e0a..b1b0d28d 100644 --- a/rust/operator-binary/src/crd/user_info_fetcher/v1alpha1_impl.rs +++ b/rust/operator-binary/src/crd/user_info_fetcher/v1alpha1_impl.rs @@ -1,6 +1,4 @@ -use stackable_operator::shared::time::Duration; - -use crate::crd::user_info_fetcher::{DEFAULT_CACHE_ENTRY_TIME_TO_LIVE, v1alpha1}; +use crate::crd::user_info_fetcher::v1alpha1; // TODO (@Techassi): Most of these impls are the exact same across v1alpha1 and v1alpha2. Explore // and design a more elegant solution for it. @@ -9,17 +7,3 @@ impl Default for v1alpha1::Backend { Self::None {} } } - -impl Default for v1alpha1::Cache { - fn default() -> Self { - Self { - entry_time_to_live: Self::default_entry_time_to_live(), - } - } -} - -impl v1alpha1::Cache { - pub const fn default_entry_time_to_live() -> Duration { - DEFAULT_CACHE_ENTRY_TIME_TO_LIVE - } -} diff --git a/rust/operator-binary/src/crd/user_info_fetcher/v1alpha2_impl.rs b/rust/operator-binary/src/crd/user_info_fetcher/v1alpha2_impl.rs index e7c2958c..1f2ad195 100644 --- a/rust/operator-binary/src/crd/user_info_fetcher/v1alpha2_impl.rs +++ b/rust/operator-binary/src/crd/user_info_fetcher/v1alpha2_impl.rs @@ -1,6 +1,6 @@ -use stackable_operator::{crd::authentication::ldap, shared::time::Duration}; +use stackable_operator::crd::authentication::ldap; -use crate::crd::user_info_fetcher::{DEFAULT_CACHE_ENTRY_TIME_TO_LIVE, v1alpha2}; +use crate::crd::user_info_fetcher::v1alpha2; // TODO (@Techassi): Most of these impls are the exact same across v1alpha1 and v1alpha2. Explore // and design a more elegant solution for it. @@ -10,20 +10,6 @@ impl Default for v1alpha2::Backend { } } -impl Default for v1alpha2::Cache { - fn default() -> Self { - Self { - entry_time_to_live: Self::default_entry_time_to_live(), - } - } -} - -impl v1alpha2::Cache { - pub const fn default_entry_time_to_live() -> Duration { - DEFAULT_CACHE_ENTRY_TIME_TO_LIVE - } -} - impl v1alpha2::OpenLdapBackend { /// Returns an LDAP [`AuthenticationProvider`](ldap::v1alpha1::AuthenticationProvider) for /// connecting to the OpenLDAP server. diff --git a/rust/resource-info-fetcher/Cargo.toml b/rust/resource-info-fetcher/Cargo.toml new file mode 100644 index 00000000..a19bde90 --- /dev/null +++ b/rust/resource-info-fetcher/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "stackable-opa-resource-info-fetcher" +description = "Fetches resource metadata" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +publish = false + +[dependencies] +stackable-opa-operator = { path = "../operator-binary" } +info-fetcher-commons = { path = "../info-fetcher-commons" } +stackable-operator.workspace = true + +axum.workspace = true +clap.workspace = true +futures.workspace = true +hyper.workspace = true +moka.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +snafu.workspace = true +tokio.workspace = true +tracing.workspace = true +url.workspace = true +urlencoding.workspace = true + +[build-dependencies] +built.workspace = true diff --git a/rust/resource-info-fetcher/build.rs b/rust/resource-info-fetcher/build.rs new file mode 100644 index 00000000..fa809bfd --- /dev/null +++ b/rust/resource-info-fetcher/build.rs @@ -0,0 +1,3 @@ +fn main() { + built::write_built_file().unwrap(); +} diff --git a/rust/resource-info-fetcher/src/api.rs b/rust/resource-info-fetcher/src/api.rs new file mode 100644 index 00000000..7b35194b --- /dev/null +++ b/rust/resource-info-fetcher/src/api.rs @@ -0,0 +1,66 @@ +use hyper::StatusCode; +use info_fetcher_commons::http_error; +use serde::{Deserialize, Serialize}; +use snafu::Snafu; + +use crate::backend; + +pub trait ResourceInfoBackend { + type Response: Serialize; + + async fn get_resource_info( + &self, + request: &ResourceInfoRequest, + ) -> Result; +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResourceInfoRequest { + // Global arguments shared between all resources + pub stacklet: String, + + #[serde(flatten)] + pub resource: ResourceInfoRequestResource, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ResourceInfoRequestResource { + TrinoTable { + catalog: String, + schema: String, + table: String, + }, + // TrinoSchema { + // catalog: String, + // schema: String, + // }, +} + +#[derive(Snafu, Debug)] +pub enum GetResourceInfoError { + #[snafu(display("failed to serialize response as JSON"))] + SerializeResponseAsJson { source: serde_json::Error }, + + #[snafu( + context(false), + display("failed to get resource information from DataHub") + )] + DataHub { source: backend::data_hub::Error }, +} + +impl http_error::Error for GetResourceInfoError { + fn status_code(&self) -> StatusCode { + // todo: the warn here loses context about the scope in which the error occurred, eg: stackable_opa_resource_info_fetcher::backend::DATA_HUB + // Also, we should make the log level (warn vs error) more dynamic in the backend's impl `http_error::Error for Error` + tracing::warn!( + error = self as &dyn std::error::Error, + "Error while processing request" + ); + match self { + Self::SerializeResponseAsJson { .. } => StatusCode::INTERNAL_SERVER_ERROR, + Self::DataHub { source } => source.status_code(), + } + } +} diff --git a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs new file mode 100644 index 00000000..70e69b82 --- /dev/null +++ b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs @@ -0,0 +1,179 @@ +use std::path::Path; + +use hyper::StatusCode; +use info_fetcher_commons::{ + http_error, + utils::{self, http::send_json_request}, +}; +use reqwest::{ClientBuilder, Url}; +use serde::Serialize; +use snafu::{ResultExt, Snafu}; +use stackable_opa_operator::crd::resource_info_fetcher::v1alpha1::{self, DataHubBackend}; + +use crate::{ + api::{ + GetResourceInfoError, ResourceInfoBackend, ResourceInfoRequest, + }, + backend::data_hub::{resource_to_urn_mapping::urn_for_request, upstream_api::DataHubEntityResponse}, +}; + +mod resource_to_urn_mapping; +mod upstream_api; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to read client ID from {path:?}"))] + ReadClientId { + source: std::io::Error, + path: String, + }, + + #[snafu(display("failed to read client secret from {path:?}"))] + ReadClientSecret { + source: std::io::Error, + path: String, + }, + + #[snafu(display("failed to configure TLS"))] + ConfigureTls { source: utils::tls::Error }, + + #[snafu(display("failed to construct HTTP client"))] + ConstructHttpClient { source: reqwest::Error }, + + #[snafu(display("failed to to build DataHub endpoint for {endpoint}"))] + BuildDataHubEndpoint { + source: url::ParseError, + endpoint: String, + }, + + #[snafu(display("failed to query for entity with URN {urn:?}"))] + QueryForUrn { + source: utils::http::Error, + urn: String, + }, +} + +impl http_error::Error for Error { + fn status_code(&self) -> StatusCode { + match self { + Self::ReadClientId { .. } => StatusCode::SERVICE_UNAVAILABLE, + Self::ReadClientSecret { .. } => StatusCode::SERVICE_UNAVAILABLE, + Self::ConfigureTls { .. } => StatusCode::SERVICE_UNAVAILABLE, + Self::ConstructHttpClient { .. } => StatusCode::SERVICE_UNAVAILABLE, + Self::BuildDataHubEndpoint { .. } => StatusCode::BAD_REQUEST, + Self::QueryForUrn { .. } => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +/// This struct combines the CRD configuration with credentials loaded from the filesystem. +/// Credentials and the HTTP client are initialized once at startup and stored internally. +pub struct ResolvedDataHubBackend { + config: v1alpha1::DataHubBackend, + client_id: String, + client_secret: String, + http_client: reqwest::Client, + // TODO: Think about a cache for tag names? +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DataHubResourceInfoResponse { + owners: Vec, + tags: Vec, +} + +impl ResolvedDataHubBackend { + /// Resolves a DataHub backend by loading credentials from the filesystem. + pub async fn resolve( + config: v1alpha1::DataHubBackend, + credentials_dir: &Path, + ) -> Result { + let client_id_path = credentials_dir.join("clientId"); + let client_secret_path = credentials_dir.join("clientSecret"); + + let client_id = tokio::fs::read_to_string(&client_id_path) + .await + .with_context(|_| ReadClientIdSnafu { + path: client_id_path.display().to_string(), + })?; + let client_secret = tokio::fs::read_to_string(&client_secret_path) + .await + .with_context(|_| ReadClientSecretSnafu { + path: client_secret_path.display().to_string(), + })?; + + let mut client_builder = ClientBuilder::new(); + client_builder = utils::tls::configure_reqwest(&config.tls, client_builder) + .await + .context(ConfigureTlsSnafu)?; + let http_client = client_builder.build().context(ConstructHttpClientSnafu)?; + + Ok(Self { + config, + client_id, + client_secret, + http_client, + }) + } +} + +impl ResourceInfoBackend for ResolvedDataHubBackend { + type Response = DataHubResourceInfoResponse; + + async fn get_resource_info( + &self, + request: &ResourceInfoRequest, + ) -> Result { + let urn = urn_for_request(request, &self.config.env); + let entity_response = self.query_entity(&urn).await?; + + dbg!(&entity_response); + let tags = entity_response.tag_urns(); + let owners = entity_response.owner_urns(); + + Ok(DataHubResourceInfoResponse { tags, owners }) + } +} + +impl ResolvedDataHubBackend { + async fn query_entity(&self, urn: &str) -> Result { + let Self { + config: + DataHubBackend { + hostname, + port, + tls, + .. + }, + client_id, + client_secret, + http_client, + } = &self; + + let schema = if tls.uses_tls() { "https" } else { "http" }; + let port = port.unwrap_or(if tls.uses_tls() { 443 } else { 80 }); + + let entity = format!( + "{schema}://{hostname}:{port}/entitiesV2/{url_encoded_urn}", + url_encoded_urn = urlencoding::encode(urn) + ); + let entity_url = + Url::parse(&entity).with_context(|_| BuildDataHubEndpointSnafu { endpoint: entity })?; + + send_json_request( + http_client + .get(entity_url) + // DataHub's system authenticator strips the leading "Basic " and compares the rest + // VERBATIM against ":" — it is NOT standard RFC 7617 Basic auth. So do + // NOT use reqwest's .basic_auth(), which base64-encodes the credentials; set the + // header manually so the value goes out unencoded. + .header( + reqwest::header::AUTHORIZATION, + format!("Basic {client_id}:{client_secret}"), + ), + ) + .await + .with_context(|_| QueryForUrnSnafu { urn }) + } +} diff --git a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs new file mode 100644 index 00000000..3d3da7a3 --- /dev/null +++ b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs @@ -0,0 +1,17 @@ +use crate::api::{ResourceInfoRequest, ResourceInfoRequestResource}; + + +pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> String { + let stacklet = &request.stacklet; + match &request.resource { + ResourceInfoRequestResource::TrinoTable { + catalog, + schema, + table, + } => { + format!( + "urn:li:dataset:(urn:li:dataPlatform:{stacklet},{catalog}.{schema}.{table},{env})" + ) + } + } +} diff --git a/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs b/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs new file mode 100644 index 00000000..509f9f04 --- /dev/null +++ b/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs @@ -0,0 +1,79 @@ +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DataHubEntityResponse { + aspects: Aspects, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Aspects { + global_tags: Option, + ownership: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsGlobalTags { + value: AspectsGlobalTagsValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsGlobalTagsValue { + tags: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsGlobalTagsValueTag { + #[serde(rename = "tag")] + tag_urn: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsOwnership { + value: AspectsOwnershipValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsOwnershipValue { + owners: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsOwnershipValueOwner { + #[serde(rename = "owner")] + owner_urn: String, +} + +impl DataHubEntityResponse { + pub fn tag_urns(&self) -> Vec { + self.aspects + .global_tags + .iter() + .flat_map(|global_tag| &global_tag.value.tags) + .map(|tag| tag.tag_urn.clone()) + .collect() + } + + // pub fn tags(&self) -> Vec { + // self.tag_urns() + // .iter() + // .map(|urn| urn.trim_start_matches("urn:li:tag:").to_owned()) + // .collect() + // } + + pub fn owner_urns(&self) -> Vec { + self.aspects + .ownership + .iter() + .flat_map(|owner| &owner.value.owners) + .map(|owner| owner.owner_urn.clone()) + .collect() + } +} diff --git a/rust/resource-info-fetcher/src/backend/mod.rs b/rust/resource-info-fetcher/src/backend/mod.rs new file mode 100644 index 00000000..9fa83c04 --- /dev/null +++ b/rust/resource-info-fetcher/src/backend/mod.rs @@ -0,0 +1 @@ +pub mod data_hub; diff --git a/rust/resource-info-fetcher/src/main.rs b/rust/resource-info-fetcher/src/main.rs new file mode 100644 index 00000000..fe124425 --- /dev/null +++ b/rust/resource-info-fetcher/src/main.rs @@ -0,0 +1,180 @@ +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use axum::{Json, Router, extract::State, routing::post}; +use clap::Parser; +use futures::{FutureExt, future}; +use info_fetcher_commons::{ + config::{ConfigError, read_config_file}, + http_error, +}; +use moka::future::Cache; +use snafu::{ResultExt, Snafu}; +use stackable_opa_operator::crd::resource_info_fetcher::v1alpha1::{self}; +use stackable_operator::{cli::CommonOptions, telemetry::Tracing}; +use tokio::net::TcpListener; + +use crate::api::{GetResourceInfoError, ResourceInfoBackend, ResourceInfoRequest}; + +mod api; +mod backend; + +pub mod built_info { + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} + +pub const APP_NAME: &str = "opa-user-info-fetcher"; + +#[derive(clap::Parser)] +pub struct Args { + #[clap(flatten)] + common: CommonOptions, + + #[clap(long, env)] + config: PathBuf, + + #[clap(long, env)] + credentials_dir: PathBuf, +} + +#[derive(Clone)] +struct AppState { + backend: Arc, + // Note: Although we might no talk JSON to the underlying backend, we always return JSON as a + // result to the caller, so we can cache that. + resource_info_cache: Cache, +} + +/// Backend with resolved credentials. +/// +/// This enum wraps backend-specific implementations that have already loaded their credentials +/// and initialized their HTTP clients. +enum ResolvedBackend { + DataHub(backend::data_hub::ResolvedDataHubBackend), +} + +#[derive(Snafu, Debug)] +enum StartupError { + #[snafu(display("failed to initialize stackable-telemetry"))] + TracingInit { + source: stackable_operator::telemetry::tracing::Error, + }, + + #[snafu(display("failed to register SIGTERM handler"))] + RegisterSigterm { source: std::io::Error }, + + #[snafu(display("unable to parse config file from {path:?}"))] + ParseConfigFile { source: ConfigError, path: PathBuf }, + + #[snafu(display("failed to bind listener"))] + BindListener { source: std::io::Error }, + + #[snafu(display("failed to run server"))] + RunServer { source: std::io::Error }, + + #[snafu(display("failed to resolve Keycloak backend"))] + ResolveDataHubBackend { source: backend::data_hub::Error }, +} + +/// Resolves a backend configuration by loading credentials and creating the appropriate backend implementation. +/// +/// This function reads credentials from the filesystem once at startup and returns a backend that +/// contains both the configuration and the resolved credentials. +async fn resolve_backend( + backend: v1alpha1::Backend, + credentials_dir: &Path, +) -> Result { + match backend { + v1alpha1::Backend::DataHub(config) => { + let resolved = + backend::data_hub::ResolvedDataHubBackend::resolve(config, credentials_dir) + .await + .context(ResolveDataHubBackendSnafu)?; + Ok(ResolvedBackend::DataHub(resolved)) + } + } +} + +#[tokio::main] +#[snafu::report] +async fn main() -> Result<(), StartupError> { + let args = Args::parse(); + + let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, args.common.telemetry) + .init() + .context(TracingInitSnafu)?; + + tracing::info!( + built_info.pkg_version = built_info::PKG_VERSION, + built_info.git_version = built_info::GIT_VERSION, + built_info.target = built_info::TARGET, + built_info.built_time_utc = built_info::BUILT_TIME_UTC, + built_info.rustc_version = built_info::RUSTC_VERSION, + "Starting resource-info-fetcher", + ); + + let shutdown_requested = tokio::signal::ctrl_c().map(|_| ()); + #[cfg(unix)] + let shutdown_requested = { + let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .context(RegisterSigtermSnafu)?; + async move { + use futures::pin_mut; + + let sigterm = sigterm.recv().map(|_| ()); + pin_mut!(shutdown_requested, sigterm); + future::select(shutdown_requested, sigterm).await; + } + }; + + let config: v1alpha1::Config = read_config_file(&args.config) + .with_context(|_| ParseConfigFileSnafu { path: args.config })?; + let backend = Arc::new(resolve_backend(config.backend, &args.credentials_dir).await?); + let resource_info_cache = { + Cache::builder() + .name("resource-info") + .time_to_live(*config.cache.entry_time_to_live) + .build() + }; + let app = Router::new() + .route("/resource", post(get_resource_info)) + .with_state(AppState { + backend, + resource_info_cache, + }); + let listener = TcpListener::bind("127.0.0.1:9477") + .await + .context(BindListenerSnafu)?; + + axum::serve(listener, app.into_make_service()) + .with_graceful_shutdown(shutdown_requested) + .await + .context(RunServerSnafu)?; + Ok(()) +} + +async fn get_resource_info( + State(state): State, + Json(req): Json, +) -> Result, http_error::JsonResponse>> { + let AppState { + backend, + resource_info_cache, + } = state; + let resource_info = resource_info_cache + .try_get_with_by_ref(&req, async { + match backend.as_ref() { + ResolvedBackend::DataHub(data_hub) => { + let response = data_hub.get_resource_info(&req).await?; + serde_json::to_value(&response).map_err(|err| { + GetResourceInfoError::SerializeResponseAsJson { source: err } + }) + } + } + }) + .await?; + + Ok(Json(resource_info)) +} diff --git a/rust/user-info-fetcher/Cargo.toml b/rust/user-info-fetcher/Cargo.toml index a8260c66..9a6b33cd 100644 --- a/rust/user-info-fetcher/Cargo.toml +++ b/rust/user-info-fetcher/Cargo.toml @@ -10,6 +10,7 @@ publish = false [dependencies] stackable-opa-operator = { path = "../operator-binary" } +info-fetcher-commons = { path = "../info-fetcher-commons" } stackable-operator.workspace = true krb5.workspace = true diff --git a/rust/user-info-fetcher/src/backend/active_directory.rs b/rust/user-info-fetcher/src/backend/active_directory.rs index 93fc487a..d339ba16 100644 --- a/rust/user-info-fetcher/src/backend/active_directory.rs +++ b/rust/user-info-fetcher/src/backend/active_directory.rs @@ -8,13 +8,14 @@ use std::{ use byteorder::{BigEndian, LittleEndian, ReadBytesExt}; use hyper::StatusCode; +use info_fetcher_commons::utils; use krb5::KrbContext; use ldap3::{Ldap, LdapConnAsync, LdapConnSettings, LdapError, Scope, SearchEntry, ldap_escape}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::commons::tls_verification::TlsClientDetails; use uuid::Uuid; -use crate::{ErrorRenderUserInfoRequest, UserInfo, UserInfoRequest, http_error, utils}; +use crate::{ErrorRenderUserInfoRequest, UserInfo, UserInfoRequest, http_error}; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/user-info-fetcher/src/backend/entra.rs b/rust/user-info-fetcher/src/backend/entra.rs index 721e92ee..d1c13b16 100644 --- a/rust/user-info-fetcher/src/backend/entra.rs +++ b/rust/user-info-fetcher/src/backend/entra.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, path::Path}; use hyper::StatusCode; +use info_fetcher_commons::utils::{self, http::send_json_request}; use reqwest::ClientBuilder; use serde::Deserialize; use snafu::{ResultExt, Snafu}; @@ -8,25 +9,22 @@ use stackable_opa_operator::crd::user_info_fetcher::v1alpha2; use stackable_operator::commons::{networking::HostName, tls_verification::TlsClientDetails}; use url::Url; -use crate::{ - UserInfo, UserInfoRequest, http_error, - utils::{self, http::send_json_request}, -}; +use crate::{UserInfo, UserInfoRequest, http_error}; #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to get access_token"))] - AccessToken { source: crate::utils::http::Error }, + AccessToken { source: utils::http::Error }, #[snafu(display("failed to search for user with username {username:?}"))] SearchForUser { - source: crate::utils::http::Error, + source: utils::http::Error, username: String, }, #[snafu(display("failed to search for user with id {user_id:?}"))] UserNotFoundById { - source: crate::utils::http::Error, + source: utils::http::Error, user_id: String, }, @@ -34,13 +32,13 @@ pub enum Error { "failed to request groups for user with username {username:?} (user_id: {user_id:?})" ))] RequestUserGroups { - source: crate::utils::http::Error, + source: utils::http::Error, username: String, user_id: String, }, #[snafu(display("failed to to build entra endpoint for {endpoint}"))] - BuildEntraEndpointFailed { + BuildEntraEndpoint { source: url::ParseError, endpoint: String, }, @@ -71,7 +69,7 @@ impl http_error::Error for Error { Self::SearchForUser { .. } => StatusCode::BAD_GATEWAY, Self::UserNotFoundById { .. } => StatusCode::NOT_FOUND, Self::RequestUserGroups { .. } => StatusCode::BAD_GATEWAY, - Self::BuildEntraEndpointFailed { .. } => StatusCode::BAD_REQUEST, + Self::BuildEntraEndpoint { .. } => StatusCode::BAD_REQUEST, Self::ConstructHttpClient { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::ConfigureTls { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::ReadClientId { .. } => StatusCode::SERVICE_UNAVAILABLE, @@ -254,14 +252,13 @@ impl EntraBackend { let token_endpoint = format!("{schema}://{token_endpoint}:{port}/{tenant_id}/oauth2/v2.0/token"); - let token_endpoint_url = - Url::parse(&token_endpoint).context(BuildEntraEndpointFailedSnafu { - endpoint: token_endpoint, - })?; + let token_endpoint_url = Url::parse(&token_endpoint).context(BuildEntraEndpointSnafu { + endpoint: token_endpoint, + })?; let user_info_endpoint = format!("{schema}://{user_info_endpoint}:{port}"); let user_info_endpoint_url = - Url::parse(&user_info_endpoint).context(BuildEntraEndpointFailedSnafu { + Url::parse(&user_info_endpoint).context(BuildEntraEndpointSnafu { endpoint: user_info_endpoint, })?; diff --git a/rust/user-info-fetcher/src/backend/keycloak.rs b/rust/user-info-fetcher/src/backend/keycloak.rs index 49c19822..cb975e2c 100644 --- a/rust/user-info-fetcher/src/backend/keycloak.rs +++ b/rust/user-info-fetcher/src/backend/keycloak.rs @@ -1,28 +1,26 @@ use std::{collections::HashMap, path::Path}; use hyper::StatusCode; +use info_fetcher_commons::utils::{self, http::send_json_request}; use reqwest::ClientBuilder; use serde::Deserialize; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_opa_operator::crd::user_info_fetcher::v1alpha2; use stackable_operator::crd::authentication::oidc; -use crate::{ - UserInfo, UserInfoRequest, http_error, - utils::{self, http::send_json_request}, -}; +use crate::{UserInfo, UserInfoRequest, http_error}; #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to get access_token"))] - AccessToken { source: crate::utils::http::Error }, + AccessToken { source: utils::http::Error }, #[snafu(display("failed to search for user"))] - SearchForUser { source: crate::utils::http::Error }, + SearchForUser { source: utils::http::Error }, #[snafu(display("unable to find user with id {user_id:?}"))] UserNotFoundById { - source: crate::utils::http::Error, + source: utils::http::Error, user_id: String, }, @@ -36,7 +34,7 @@ pub enum Error { "failed to request groups for user with username {username:?} (user_id: {user_id:?})" ))] RequestUserGroups { - source: crate::utils::http::Error, + source: utils::http::Error, username: String, user_id: String, }, diff --git a/rust/user-info-fetcher/src/backend/openldap.rs b/rust/user-info-fetcher/src/backend/openldap.rs index 07ed34f8..51386912 100644 --- a/rust/user-info-fetcher/src/backend/openldap.rs +++ b/rust/user-info-fetcher/src/backend/openldap.rs @@ -1,12 +1,13 @@ use std::collections::{BTreeMap, HashMap}; use hyper::StatusCode; +use info_fetcher_commons::utils; use ldap3::{LdapConnAsync, LdapConnSettings, LdapError, Scope, SearchEntry, ldap_escape}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_opa_operator::crd::user_info_fetcher::v1alpha2; use stackable_operator::crd::authentication::ldap; -use crate::{ErrorRenderUserInfoRequest, UserInfo, UserInfoRequest, http_error, utils}; +use crate::{ErrorRenderUserInfoRequest, UserInfo, UserInfoRequest, http_error}; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/user-info-fetcher/src/backend/xfsc_aas.rs b/rust/user-info-fetcher/src/backend/xfsc_aas.rs index 0b4df775..d9745331 100644 --- a/rust/user-info-fetcher/src/backend/xfsc_aas.rs +++ b/rust/user-info-fetcher/src/backend/xfsc_aas.rs @@ -13,13 +13,14 @@ use std::collections::HashMap; use hyper::StatusCode; +use info_fetcher_commons::utils::{self, http::send_json_request}; use reqwest::ClientBuilder; use serde::Deserialize; use snafu::{ResultExt, Snafu}; use stackable_opa_operator::crd::user_info_fetcher::v1alpha2; use url::Url; -use crate::{UserInfo, UserInfoRequest, http_error, utils::http::send_json_request}; +use crate::{UserInfo, UserInfoRequest, http_error}; static API_PATH: &str = "/cip/claims"; static SUB_CLAIM: &str = "sub"; @@ -35,7 +36,7 @@ pub enum Error { }, #[snafu(display("request failed"))] - Request { source: crate::utils::http::Error }, + Request { source: utils::http::Error }, #[snafu(display("the XFSC AAS does not support querying by username, only by user ID"))] UserInfoByUsernameNotSupported {}, diff --git a/rust/user-info-fetcher/src/main.rs b/rust/user-info-fetcher/src/main.rs index 92179b11..3552773e 100644 --- a/rust/user-info-fetcher/src/main.rs +++ b/rust/user-info-fetcher/src/main.rs @@ -8,6 +8,10 @@ use std::{ use axum::{Json, Router, extract::State, routing::post}; use clap::Parser; use futures::{FutureExt, future, pin_mut}; +use info_fetcher_commons::{ + config::{ConfigError, read_config_file}, + http_error, +}; use moka::future::Cache; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; @@ -16,8 +20,6 @@ use stackable_operator::{cli::CommonOptions, telemetry::Tracing}; use tokio::net::TcpListener; mod backend; -mod http_error; -mod utils; pub mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); @@ -64,14 +66,8 @@ enum ResolvedBackend { #[derive(Snafu, Debug)] enum StartupError { - #[snafu(display("unable to read config file from {path:?}"))] - ReadConfigFile { - source: std::io::Error, - path: PathBuf, - }, - - #[snafu(display("failed to parse config file"))] - ParseConfig { source: serde_json::Error }, + #[snafu(display("unable to parse config file from {path:?}"))] + ParseConfigFile { source: ConfigError, path: PathBuf }, #[snafu(display("failed to register SIGTERM handler"))] RegisterSigterm { source: std::io::Error }, @@ -100,12 +96,6 @@ enum StartupError { ResolveXfscAasBackend { source: backend::xfsc_aas::Error }, } -async fn read_config_file(path: &Path) -> Result { - tokio::fs::read_to_string(path) - .await - .context(ReadConfigFileSnafu { path }) -} - /// Resolves a backend configuration by loading credentials and creating the appropriate backend implementation. /// /// This function reads credentials from the filesystem once at startup and returns a backend that @@ -184,16 +174,14 @@ async fn main() -> Result<(), StartupError> { } }; - let config: v1alpha2::Config = - serde_json::from_str(&read_config_file(&args.config).await?).context(ParseConfigSnafu)?; - + let config: v1alpha2::Config = read_config_file(&args.config) + .with_context(|_| ParseConfigFileSnafu { path: args.config })?; let backend = Arc::new(resolve_backend(config.backend, &args.credentials_dir).await?); let user_info_cache = { - let v1alpha2::Cache { entry_time_to_live } = config.cache; Cache::builder() .name("user-info") - .time_to_live(*entry_time_to_live) + .time_to_live(*config.cache.entry_time_to_live) .build() }; let app = Router::new() From a1da9db6f083f29941e60b9c951aba68ea4ab0a7 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 16 Jul 2026 14:21:37 +0200 Subject: [PATCH 02/13] feat: Implement Trino catalog and schema --- Cargo.lock | 7 ++ Cargo.toml | 1 + extra/crds.yaml | 4 +- .../src/crd/resource_info_fetcher/mod.rs | 2 + rust/resource-info-fetcher/Cargo.toml | 1 + rust/resource-info-fetcher/src/api.rs | 13 ++-- .../data_hub/resource_to_urn_mapping.rs | 72 ++++++++++++++++++- 7 files changed, 93 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4efafe03..695806c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2097,6 +2097,12 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.8.3" @@ -3465,6 +3471,7 @@ dependencies = [ "futures", "hyper", "info-fetcher-commons", + "md5", "moka", "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index 32cef8af..c7051fcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ futures = { version = "0.3" } hyper = "1.4" indoc = "2.0" ldap3 = { version = "0.12", features = ["gssapi", "tls"] } +md5 = "0.7" moka = { version = "0.12", features = ["future"] } native-tls = "0.2.12" pin-project = "1.1" diff --git a/extra/crds.yaml b/extra/crds.yaml index f5ce8061..529722d5 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -518,7 +518,7 @@ spec: properties: entryTimeToLive: default: 1m - description: How long metadata about each user should be cached for. + description: How long responses should be cached for. type: string type: object type: object @@ -2449,7 +2449,7 @@ spec: properties: entryTimeToLive: default: 1m - description: How long metadata about each user should be cached for. + description: How long responses should be cached for. type: string type: object type: object diff --git a/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs b/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs index 3755e8ed..57e76154 100644 --- a/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs +++ b/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs @@ -45,6 +45,8 @@ pub mod versioned { } } +// We actually use it as serde(default) value, but Rust fails to see that +#[allow(dead_code)] fn default_data_hub_env() -> String { "PROD".to_owned() } diff --git a/rust/resource-info-fetcher/Cargo.toml b/rust/resource-info-fetcher/Cargo.toml index a19bde90..5a00634c 100644 --- a/rust/resource-info-fetcher/Cargo.toml +++ b/rust/resource-info-fetcher/Cargo.toml @@ -17,6 +17,7 @@ axum.workspace = true clap.workspace = true futures.workspace = true hyper.workspace = true +md5.workspace = true moka.workspace = true reqwest.workspace = true serde.workspace = true diff --git a/rust/resource-info-fetcher/src/api.rs b/rust/resource-info-fetcher/src/api.rs index 7b35194b..616a8c5c 100644 --- a/rust/resource-info-fetcher/src/api.rs +++ b/rust/resource-info-fetcher/src/api.rs @@ -26,16 +26,21 @@ pub struct ResourceInfoRequest { #[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] #[serde(rename_all = "camelCase")] +// Soon there will be something else than Trino +#[expect(clippy::enum_variant_names)] pub enum ResourceInfoRequestResource { TrinoTable { catalog: String, schema: String, table: String, }, - // TrinoSchema { - // catalog: String, - // schema: String, - // }, + TrinoSchema { + catalog: String, + schema: String, + }, + TrinoCatalog { + catalog: String, + }, } #[derive(Snafu, Debug)] diff --git a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs index 3d3da7a3..927ec7e1 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs @@ -1,5 +1,6 @@ -use crate::api::{ResourceInfoRequest, ResourceInfoRequestResource}; +use std::collections::BTreeMap; +use crate::api::{ResourceInfoRequest, ResourceInfoRequestResource}; pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> String { let stacklet = &request.stacklet; @@ -13,5 +14,74 @@ pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> String { "urn:li:dataset:(urn:li:dataPlatform:{stacklet},{catalog}.{schema}.{table},{env})" ) } + // Trino catalogs and schemas are modelled as DataHub `Container`s (subtypes `Database` and + // `Schema` respectively). Unlike datasets, their URN is not human-readable but a GUID derived + // from the container key - see `container_urn`. + ResourceInfoRequestResource::TrinoCatalog { catalog } => { + container_urn(&BTreeMap::from([ + ("platform", stacklet.as_str()), + ("instance", env), + ("database", catalog.as_str()), + ])) + } + ResourceInfoRequestResource::TrinoSchema { catalog, schema } => { + container_urn(&BTreeMap::from([ + ("platform", stacklet.as_str()), + ("instance", env), + ("database", catalog.as_str()), + ("schema", schema.as_str()), + ])) + } + } +} + +/// Reproduces DataHub's `datahub_guid`: the container key is serialized to compact, key-sorted JSON +/// and MD5-hashed. A [`BTreeMap`] yields sorted keys and `serde_json` emits no whitespace, which +/// matches Python's `json.dumps(key, sort_keys=True, separators=(",", ":"))`. +/// +/// Note that the SQL ingestion source sets `backcompat_env_as_instance`, so the configured `env` +/// (e.g. `PROD`) ends up in the `instance` field and no `env` field is present in the key. The +/// `platform` is the bare platform name (e.g. `trino`), *not* the `urn:li:dataPlatform:` form. +fn container_urn(container_key: &BTreeMap<&str, &str>) -> String { + let key_json = + serde_json::to_string(container_key).expect("serializing a BTreeMap<&str, &str> cannot fail"); + format!("urn:li:container:{:x}", md5::compute(key_json.as_bytes())) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Verified against a live DataHub: the schema `customer_analytics` in catalog `lakehouse`. + #[test] + fn trino_schema_urn_matches_data_hub() { + let request = ResourceInfoRequest { + stacklet: "trino".to_owned(), + resource: ResourceInfoRequestResource::TrinoSchema { + catalog: "lakehouse".to_owned(), + schema: "customer_analytics".to_owned(), + }, + }; + + assert_eq!( + urn_for_request(&request, "PROD"), + "urn:li:container:c8531e5a52cacf56768d0bf77ca8787c" + ); + } + + /// Verified against a live DataHub: the catalog `lakehouse`. + #[test] + fn trino_catalog_urn_matches_data_hub() { + let request = ResourceInfoRequest { + stacklet: "trino".to_owned(), + resource: ResourceInfoRequestResource::TrinoCatalog { + catalog: "lakehouse".to_owned(), + }, + }; + + assert_eq!( + urn_for_request(&request, "PROD"), + "urn:li:container:39967cd09b38e2d4736d1eb604cd5247" + ); } } From d56a198510110f3e6dc883b3b2ca385df5d76085 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 20 Jul 2026 09:18:02 +0200 Subject: [PATCH 03/13] feat: Cache tag and user lookups --- Cargo.lock | 238 +++++++++--------- Cargo.toml | 4 +- _TEST/rif.json | 3 + rust/info-fetcher-commons/Cargo.toml | 1 + rust/info-fetcher-commons/src/utils/http.rs | 12 +- rust/resource-info-fetcher/Cargo.toml | 1 + rust/resource-info-fetcher/src/api.rs | 18 +- .../src/backend/data_hub/mod.rs | 225 +++++++++++++++-- .../data_hub/resource_to_urn_mapping.rs | 36 +-- .../src/backend/data_hub/upstream_api.rs | 189 ++++++++++++-- rust/resource-info-fetcher/src/main.rs | 6 +- 11 files changed, 557 insertions(+), 176 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 695806c9..49af0902 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,7 +169,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -180,7 +180,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -305,7 +305,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -316,7 +316,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -342,9 +342,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block-buffer" @@ -379,15 +379,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.66" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -434,9 +434,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" dependencies = [ "clap_builder", "clap_derive", @@ -444,9 +444,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -463,7 +463,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -592,7 +592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d4ddf7139e64dc916b11d434421031bcc5ba02e521a49a011652a0f68775188" dependencies = [ "anyhow", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "libgssapi", "windows", @@ -664,7 +664,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -675,7 +675,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -697,7 +697,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -717,7 +717,7 @@ checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -741,7 +741,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -768,7 +768,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -791,7 +791,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -849,7 +849,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -922,7 +922,7 @@ checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1119,7 +1119,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1204,7 +1204,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "libgit2-sys", "log", @@ -1323,9 +1323,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1333,9 +1333,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1621,6 +1621,7 @@ dependencies = [ "snafu 0.9.1", "stackable-operator", "tokio", + "tracing", ] [[package]] @@ -1681,9 +1682,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" dependencies = [ "defmt", "jiff-static", @@ -1697,20 +1698,20 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "jiff-tzdb" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" [[package]] name = "jiff-tzdb-platform" @@ -1757,9 +1758,9 @@ dependencies = [ [[package]] name = "jsonpath-rust" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633a7320c4bb672863a3782e89b9094ad70285e097ff6832cddd0ec615beadfa" +checksum = "4222e00941bfe18bf81b79fa23ad933e233bfa18f3ee27254c5dd9b1543b5d5f" dependencies = [ "pest", "pest_derive", @@ -1912,7 +1913,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2011,7 +2012,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "834339e86b2561169d45d3b01741967fee3e5716c7d0b6e33cd4e3b34c9558cd" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "lazy_static", "libgssapi-sys", @@ -2099,9 +2100,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "md5" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" [[package]] name = "memchr" @@ -2133,9 +2134,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -2215,7 +2216,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.6", + "rand 0.8.7", "smallvec", "zeroize", ] @@ -2273,7 +2274,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -2289,7 +2290,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2399,7 +2400,7 @@ dependencies = [ "opentelemetry", "percent-encoding", "portable-atomic", - "rand 0.9.4", + "rand 0.9.5", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -2510,7 +2511,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2539,7 +2540,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2621,7 +2622,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2678,7 +2679,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2713,9 +2714,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "rand_chacha 0.3.1", "rand_core 0.6.4", @@ -2723,9 +2724,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2775,7 +2776,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -2795,14 +2796,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2812,9 +2813,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2935,7 +2936,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -2944,9 +2945,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", @@ -3035,7 +3036,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3073,7 +3074,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3152,7 +3153,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3163,7 +3164,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3217,9 +3218,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures", @@ -3280,9 +3281,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "slab" @@ -3344,7 +3345,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3356,14 +3357,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3371,9 +3372,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spki" @@ -3401,7 +3402,7 @@ dependencies = [ "k8s-openapi", "kube", "p256", - "rand 0.9.4", + "rand 0.9.5", "rand_core 0.6.4", "rsa", "sha2", @@ -3479,6 +3480,7 @@ dependencies = [ "snafu 0.9.1", "stackable-opa-operator", "stackable-operator", + "strum", "tokio", "tracing", "url", @@ -3537,7 +3539,7 @@ dependencies = [ "k8s-openapi", "kube", "product-config", - "rand 0.9.4", + "rand 0.9.5", "regex", "schemars", "semver", @@ -3569,7 +3571,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3642,7 +3644,7 @@ dependencies = [ "kube", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3661,7 +3663,7 @@ dependencies = [ "kube", "opentelemetry", "opentelemetry-semantic-conventions", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "snafu 0.9.1", @@ -3701,7 +3703,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3729,9 +3731,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -3755,7 +3757,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3764,7 +3766,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -3835,7 +3837,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3846,14 +3848,14 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -3916,14 +3918,14 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "tokio" -version = "1.52.3" +version = "1.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af" dependencies = [ "bytes", "libc", @@ -3944,7 +3946,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4067,7 +4069,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -4087,7 +4089,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "http", "http-body", @@ -4143,7 +4145,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4301,9 +4303,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -4394,7 +4396,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -4494,7 +4496,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4505,7 +4507,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4729,28 +4731,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.53" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.53" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4770,7 +4772,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -4791,7 +4793,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4824,11 +4826,11 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index c7051fcb..f6224f1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,8 +26,8 @@ futures = { version = "0.3" } hyper = "1.4" indoc = "2.0" ldap3 = { version = "0.12", features = ["gssapi", "tls"] } -md5 = "0.7" -moka = { version = "0.12", features = ["future"] } +md5 = "0.8" +moka = { version = "0.12", default-features = false, features = ["future"] } native-tls = "0.2.12" pin-project = "1.1" # `default-features = false` drops reqwest 0.13's `default-tls = ["rustls"]`; we select diff --git a/_TEST/rif.json b/_TEST/rif.json index 189dc7c2..5c67ce36 100644 --- a/_TEST/rif.json +++ b/_TEST/rif.json @@ -1,4 +1,7 @@ { + "cache": { + "entryTimeToLive": "0s" + }, "backend": { "dataHub": { "hostname": "localhost", diff --git a/rust/info-fetcher-commons/Cargo.toml b/rust/info-fetcher-commons/Cargo.toml index 8949798e..aa246ee2 100644 --- a/rust/info-fetcher-commons/Cargo.toml +++ b/rust/info-fetcher-commons/Cargo.toml @@ -19,4 +19,5 @@ rustls-pki-types.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true +tracing.workspace = true tokio.workspace = true diff --git a/rust/info-fetcher-commons/src/utils/http.rs b/rust/info-fetcher-commons/src/utils/http.rs index a59ddc43..b67338ee 100644 --- a/rust/info-fetcher-commons/src/utils/http.rs +++ b/rust/info-fetcher-commons/src/utils/http.rs @@ -2,6 +2,7 @@ use hyper::StatusCode; use reqwest::{RequestBuilder, Response}; use serde::de::DeserializeOwned; use snafu::{ResultExt, Snafu}; +use tracing::{instrument, trace}; #[derive(Snafu, Debug)] pub enum Error { @@ -9,7 +10,7 @@ pub enum Error { HttpRequest { source: reqwest::Error }, #[snafu(display("failed to parse json response"))] - ParseJson { source: reqwest::Error }, + ParseJson { source: serde_json::Error }, #[snafu(display("http response {status:?} for {url:?} with response body {text:?}"))] HttpErrorResponse { @@ -26,14 +27,19 @@ pub enum Error { }, } +#[instrument(skip_all)] pub async fn send_json_request(req: RequestBuilder) -> Result { // make the request let response = req.send().await.context(HttpRequestSnafu)?; // check for client or server errors + let url = response.url().clone(); let non_error_response = error_for_status(response).await?; // parse the result - let result = non_error_response.json().await.context(ParseJsonSnafu)?; - Ok(result) + let json = non_error_response.text().await.context(HttpRequestSnafu)?; + + trace!(%url, json, "Got HTTP JSON response"); + + serde_json::from_str(&json).context(ParseJsonSnafu) } /// Wraps a Response into a Result. If there is an HTTP Client or Server error, diff --git a/rust/resource-info-fetcher/Cargo.toml b/rust/resource-info-fetcher/Cargo.toml index 5a00634c..a8c317d0 100644 --- a/rust/resource-info-fetcher/Cargo.toml +++ b/rust/resource-info-fetcher/Cargo.toml @@ -23,6 +23,7 @@ reqwest.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true +strum.workspace = true tokio.workspace = true tracing.workspace = true url.workspace = true diff --git a/rust/resource-info-fetcher/src/api.rs b/rust/resource-info-fetcher/src/api.rs index 616a8c5c..390fec9d 100644 --- a/rust/resource-info-fetcher/src/api.rs +++ b/rust/resource-info-fetcher/src/api.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use hyper::StatusCode; use info_fetcher_commons::http_error; use serde::{Deserialize, Serialize}; @@ -26,8 +28,6 @@ pub struct ResourceInfoRequest { #[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] #[serde(rename_all = "camelCase")] -// Soon there will be something else than Trino -#[expect(clippy::enum_variant_names)] pub enum ResourceInfoRequestResource { TrinoTable { catalog: String, @@ -41,6 +41,11 @@ pub enum ResourceInfoRequestResource { TrinoCatalog { catalog: String, }, + + SupersetChart(String), + SupersetDashboard(String), + + RawDataHubUrn(String), } #[derive(Snafu, Debug)] @@ -53,6 +58,14 @@ pub enum GetResourceInfoError { display("failed to get resource information from DataHub") )] DataHub { source: backend::data_hub::Error }, + + #[snafu( + visibility(pub(crate)), + display("failed to get user information from DataHub") + )] + GetDataHubUser { + source: Arc, + }, } impl http_error::Error for GetResourceInfoError { @@ -66,6 +79,7 @@ impl http_error::Error for GetResourceInfoError { match self { Self::SerializeResponseAsJson { .. } => StatusCode::INTERNAL_SERVER_ERROR, Self::DataHub { source } => source.status_code(), + Self::GetDataHubUser { source } => source.status_code(), } } } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs index 70e69b82..8020e571 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs @@ -1,20 +1,28 @@ -use std::path::Path; +use std::{collections::BTreeMap, path::Path, sync::Arc}; +use futures::future::try_join_all; use hyper::StatusCode; use info_fetcher_commons::{ http_error, utils::{self, http::send_json_request}, }; +use moka::future::Cache; use reqwest::{ClientBuilder, Url}; use serde::Serialize; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_opa_operator::crd::resource_info_fetcher::v1alpha1::{self, DataHubBackend}; +use strum::IntoEnumIterator; +use tracing::{debug, instrument, trace}; use crate::{ - api::{ - GetResourceInfoError, ResourceInfoBackend, ResourceInfoRequest, + api::{GetDataHubUserSnafu, GetResourceInfoError, ResourceInfoBackend, ResourceInfoRequest}, + backend::data_hub::{ + resource_to_urn_mapping::urn_for_request, + upstream_api::{ + AspectsCorpGroupInfoValue, AspectsCorpUserInfoValue, AspectsOwnershipValueOwnerType, + DataHubEntityResponse, OwnerTypeUrn, RawOwnerType, Tag, Urn, + }, }, - backend::data_hub::{resource_to_urn_mapping::urn_for_request, upstream_api::DataHubEntityResponse}, }; mod resource_to_urn_mapping; @@ -49,8 +57,23 @@ pub enum Error { #[snafu(display("failed to query for entity with URN {urn:?}"))] QueryForUrn { source: utils::http::Error, - urn: String, + urn: Urn, }, + + #[snafu(display( + "the entity information send by DataHub for the tag with the URN {urn:?} must contain tag properties" + ))] + EntityResponseMustContainTagProperties { urn: Urn }, + + #[snafu(display( + "the entity information send by DataHub for the user with the URN {urn:?} must contain user information" + ))] + EntityResponseMustContainUserInfo { urn: Urn }, + + #[snafu(display( + "the entity information send by DataHub for the group with the URN {urn:?} must contain group information" + ))] + EntityResponseMustContainGroupInfo { urn: Urn }, } impl http_error::Error for Error { @@ -62,6 +85,11 @@ impl http_error::Error for Error { Self::ConstructHttpClient { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::BuildDataHubEndpoint { .. } => StatusCode::BAD_REQUEST, Self::QueryForUrn { .. } => StatusCode::INTERNAL_SERVER_ERROR, + Self::EntityResponseMustContainTagProperties { .. } => { + StatusCode::INTERNAL_SERVER_ERROR + } + Self::EntityResponseMustContainUserInfo { .. } => StatusCode::INTERNAL_SERVER_ERROR, + Self::EntityResponseMustContainGroupInfo { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -73,20 +101,57 @@ pub struct ResolvedDataHubBackend { client_id: String, client_secret: String, http_client: reqwest::Client, - // TODO: Think about a cache for tag names? + + tag_cache: Cache, + user_cache: Cache, + group_cache: Cache, } #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] #[serde(rename_all = "camelCase")] pub struct DataHubResourceInfoResponse { - owners: Vec, - tags: Vec, + owners: BTreeMap, + tags: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, PartialOrd, Ord, strum::EnumIter)] +#[serde(rename_all = "camelCase")] +pub enum OwnerType { + BusinessOwner, + TechnicalOwner, + DataSteward, + None, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DataHubResourceInfoResponseOwners { + users: Vec, + groups: Vec, +} + +impl From<&AspectsOwnershipValueOwnerType> for OwnerType { + fn from(value: &AspectsOwnershipValueOwnerType) -> Self { + match value { + AspectsOwnershipValueOwnerType::Urn { type_urn } => match type_urn { + OwnerTypeUrn::BusinessOwner => Self::BusinessOwner, + OwnerTypeUrn::TechnicalOwner => Self::TechnicalOwner, + OwnerTypeUrn::DataSteward => Self::DataSteward, + OwnerTypeUrn::None => Self::None, + }, + AspectsOwnershipValueOwnerType::Raw { + type_: RawOwnerType::TechnicalOwner, + } => Self::TechnicalOwner, + } + } } impl ResolvedDataHubBackend { /// Resolves a DataHub backend by loading credentials from the filesystem. + #[instrument(skip_all)] pub async fn resolve( config: v1alpha1::DataHubBackend, + cache: &stackable_opa_operator::crd::cache::Cache, credentials_dir: &Path, ) -> Result { let client_id_path = credentials_dir.join("clientId"); @@ -109,11 +174,27 @@ impl ResolvedDataHubBackend { .context(ConfigureTlsSnafu)?; let http_client = client_builder.build().context(ConstructHttpClientSnafu)?; + let tag_cache = Cache::builder() + .time_to_live(*cache.entry_time_to_live) + .name("tag-info") + .build(); + let user_cache = Cache::builder() + .time_to_live(*cache.entry_time_to_live) + .name("user-info") + .build(); + let group_cache = Cache::builder() + .time_to_live(*cache.entry_time_to_live) + .name("group-info") + .build(); + Ok(Self { config, client_id, client_secret, http_client, + tag_cache, + user_cache, + group_cache, }) } } @@ -121,23 +202,69 @@ impl ResolvedDataHubBackend { impl ResourceInfoBackend for ResolvedDataHubBackend { type Response = DataHubResourceInfoResponse; + // TODO: Do more parallelism + #[instrument(skip(self))] async fn get_resource_info( &self, request: &ResourceInfoRequest, ) -> Result { let urn = urn_for_request(request, &self.config.env); - let entity_response = self.query_entity(&urn).await?; + let entity_response: DataHubEntityResponse = self.query_entity(&urn).await?; + + let tag_urns = entity_response.tag_urns(); + let tags = try_join_all( + tag_urns + .iter() + .map(|urn| async { self.query_tag(urn).await }), + ) + .await + .context(GetDataHubUserSnafu)?; + + let mut owners = BTreeMap::new(); + for owner_type in OwnerType::iter() { + let (user_urns, users_without_urn, group_urns) = + entity_response.owners_for_type(&owner_type); - dbg!(&entity_response); - let tags = entity_response.tag_urns(); - let owners = entity_response.owner_urns(); + let mut users = try_join_all( + user_urns + .iter() + .map(|urn| async { self.query_user(urn).await }), + ) + .await + .context(GetDataHubUserSnafu)?; + users.extend( + users_without_urn + .iter() + .map(|username| AspectsCorpUserInfoValue { + full_name: None, + display_name: username.trim_start_matches("urn:li:corpuser").to_owned(), + email: None, + active: true, + data_hub_user: false, + }), + ); + + let groups = try_join_all( + group_urns + .iter() + .map(|urn| async { self.query_group(urn).await }), + ) + .await + .context(GetDataHubUserSnafu)?; + + owners.insert( + owner_type, + DataHubResourceInfoResponseOwners { users, groups }, + ); + } Ok(DataHubResourceInfoResponse { tags, owners }) } } impl ResolvedDataHubBackend { - async fn query_entity(&self, urn: &str) -> Result { + #[instrument(skip(self))] + async fn query_entity(&self, urn: &Urn) -> Result { let Self { config: DataHubBackend { @@ -149,6 +276,7 @@ impl ResolvedDataHubBackend { client_id, client_secret, http_client, + .. } = &self; let schema = if tls.uses_tls() { "https" } else { "http" }; @@ -156,14 +284,16 @@ impl ResolvedDataHubBackend { let entity = format!( "{schema}://{hostname}:{port}/entitiesV2/{url_encoded_urn}", - url_encoded_urn = urlencoding::encode(urn) + url_encoded_urn = urlencoding::encode(&urn.0) ); let entity_url = Url::parse(&entity).with_context(|_| BuildDataHubEndpointSnafu { endpoint: entity })?; - send_json_request( + trace!(%entity_url, "Sending request to DataHub's entity API"); + + let entity = send_json_request( http_client - .get(entity_url) + .get(entity_url.clone()) // DataHub's system authenticator strips the leading "Basic " and compares the rest // VERBATIM against ":" — it is NOT standard RFC 7617 Basic auth. So do // NOT use reqwest's .basic_auth(), which base64-encodes the credentials; set the @@ -174,6 +304,65 @@ impl ResolvedDataHubBackend { ), ) .await - .with_context(|_| QueryForUrnSnafu { urn }) + .with_context(|_| QueryForUrnSnafu { urn: urn.clone() })?; + + debug!(%urn, %entity_url, "Fetched entity from DataHub"); + trace!(?entity, "DataHub entity payload"); + + Ok(entity) + } + + #[instrument(skip(self))] + async fn query_tag(&self, urn: &Urn) -> Result> { + self.tag_cache + .try_get_with_by_ref(urn, async { + let tag = self + .query_entity(urn) + .await? + .tag_properties() + .with_context(|| EntityResponseMustContainTagPropertiesSnafu { + urn: urn.clone(), + })? + .name + .clone(); + debug!(%urn, %tag, "Fetched tag from DataHub"); + + Ok(tag) + }) + .await + } + + #[instrument(skip(self))] + async fn query_user(&self, urn: &Urn) -> Result> { + self.user_cache + .try_get_with_by_ref(urn, async { + let user = self + .query_entity(urn) + .await? + .user_info() + .with_context(|| EntityResponseMustContainUserInfoSnafu { urn: urn.clone() })? + .clone(); + debug!(%urn, ?user, "Fetched user from DataHub"); + + Ok(user) + }) + .await + } + + #[instrument(skip(self))] + async fn query_group(&self, urn: &Urn) -> Result> { + self.group_cache + .try_get_with_by_ref(urn, async { + let group = self + .query_entity(urn) + .await? + .group_info() + .with_context(|| EntityResponseMustContainGroupInfoSnafu { urn: urn.clone() })? + .clone(); + debug!(%urn, ?group, "Fetched group from DataHub"); + + Ok(group) + }) + .await } } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs index 927ec7e1..cce304a7 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs @@ -1,10 +1,13 @@ use std::collections::BTreeMap; -use crate::api::{ResourceInfoRequest, ResourceInfoRequestResource}; +use crate::{ + api::{ResourceInfoRequest, ResourceInfoRequestResource}, + backend::data_hub::upstream_api::Urn, +}; -pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> String { +pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> Urn { let stacklet = &request.stacklet; - match &request.resource { + let urn = match &request.resource { ResourceInfoRequestResource::TrinoTable { catalog, schema, @@ -17,13 +20,11 @@ pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> String { // Trino catalogs and schemas are modelled as DataHub `Container`s (subtypes `Database` and // `Schema` respectively). Unlike datasets, their URN is not human-readable but a GUID derived // from the container key - see `container_urn`. - ResourceInfoRequestResource::TrinoCatalog { catalog } => { - container_urn(&BTreeMap::from([ - ("platform", stacklet.as_str()), - ("instance", env), - ("database", catalog.as_str()), - ])) - } + ResourceInfoRequestResource::TrinoCatalog { catalog } => container_urn(&BTreeMap::from([ + ("platform", stacklet.as_str()), + ("instance", env), + ("database", catalog.as_str()), + ])), ResourceInfoRequestResource::TrinoSchema { catalog, schema } => { container_urn(&BTreeMap::from([ ("platform", stacklet.as_str()), @@ -32,7 +33,12 @@ pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> String { ("schema", schema.as_str()), ])) } - } + ResourceInfoRequestResource::SupersetChart(_) => todo!(), + ResourceInfoRequestResource::SupersetDashboard(_) => todo!(), + ResourceInfoRequestResource::RawDataHubUrn(urn) => urn.to_owned(), + }; + + Urn(urn) } /// Reproduces DataHub's `datahub_guid`: the container key is serialized to compact, key-sorted JSON @@ -43,8 +49,8 @@ pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> String { /// (e.g. `PROD`) ends up in the `instance` field and no `env` field is present in the key. The /// `platform` is the bare platform name (e.g. `trino`), *not* the `urn:li:dataPlatform:` form. fn container_urn(container_key: &BTreeMap<&str, &str>) -> String { - let key_json = - serde_json::to_string(container_key).expect("serializing a BTreeMap<&str, &str> cannot fail"); + let key_json = serde_json::to_string(container_key) + .expect("serializing a BTreeMap<&str, &str> cannot fail"); format!("urn:li:container:{:x}", md5::compute(key_json.as_bytes())) } @@ -64,7 +70,7 @@ mod tests { }; assert_eq!( - urn_for_request(&request, "PROD"), + urn_for_request(&request, "PROD").0, "urn:li:container:c8531e5a52cacf56768d0bf77ca8787c" ); } @@ -80,7 +86,7 @@ mod tests { }; assert_eq!( - urn_for_request(&request, "PROD"), + urn_for_request(&request, "PROD").0, "urn:li:container:39967cd09b38e2d4736d1eb604cd5247" ); } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs b/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs index 509f9f04..0d547251 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs @@ -1,4 +1,26 @@ -use serde::Deserialize; +use std::fmt::{self, Display}; + +use serde::{Deserialize, Serialize}; + +use crate::backend::data_hub::OwnerType; + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Urn(pub String); + +impl Display for Urn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Tag(pub String); + +impl Display for Tag { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} #[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] #[serde(rename_all = "camelCase")] @@ -11,6 +33,9 @@ pub struct DataHubEntityResponse { pub struct Aspects { global_tags: Option, ownership: Option, + tag_properties: Option, + corp_user_info: Option, + corp_group_info: Option, } #[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] @@ -29,7 +54,7 @@ pub struct AspectsGlobalTagsValue { #[serde(rename_all = "camelCase")] pub struct AspectsGlobalTagsValueTag { #[serde(rename = "tag")] - tag_urn: String, + tag_urn: Urn, } #[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] @@ -47,12 +72,96 @@ pub struct AspectsOwnershipValue { #[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AspectsOwnershipValueOwner { - #[serde(rename = "owner")] - owner_urn: String, + owner: String, + + #[serde(flatten)] + type_: AspectsOwnershipValueOwnerType, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(untagged)] +pub enum AspectsOwnershipValueOwnerType { + Urn { + #[serde(rename = "typeUrn")] + type_urn: OwnerTypeUrn, + }, + Raw { + #[serde(rename = "type")] + type_: RawOwnerType, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum OwnerTypeUrn { + #[serde(rename = "urn:li:ownershipType:__system__business_owner")] + BusinessOwner, + #[serde(rename = "urn:li:ownershipType:__system__technical_owner")] + TechnicalOwner, + #[serde(rename = "urn:li:ownershipType:__system__data_steward")] + DataSteward, + #[serde(rename = "urn:li:ownershipType:__system__none")] + None, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum RawOwnerType { + #[serde(rename = "TECHNICAL_OWNER")] + TechnicalOwner, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsTagProperties { + value: AspectsTagPropertiesValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsTagPropertiesValue { + pub name: Tag, + color_hex: String, + description: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsCorpUserInfo { + value: AspectsCorpUserInfoValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsCorpUserInfoValue { + pub full_name: Option, + pub display_name: String, + pub email: Option, + pub active: bool, + + /// We default this field to `true`, so that we set it during deserialization from DataHub. When + /// "faking" a user that doesn't exist in DataHub, we set it explicitly to `false`. + #[serde(default = "true_default")] + pub data_hub_user: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsCorpGroupInfo { + value: AspectsCorpGroupInfoValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AspectsCorpGroupInfoValue { + display_name: String, + description: String, +} + +fn true_default() -> bool { + true } impl DataHubEntityResponse { - pub fn tag_urns(&self) -> Vec { + pub fn tag_urns(&self) -> Vec { self.aspects .global_tags .iter() @@ -61,19 +170,67 @@ impl DataHubEntityResponse { .collect() } - // pub fn tags(&self) -> Vec { - // self.tag_urns() - // .iter() - // .map(|urn| urn.trim_start_matches("urn:li:tag:").to_owned()) - // .collect() - // } - - pub fn owner_urns(&self) -> Vec { - self.aspects + /// Returns + /// + /// 1. The list of owner user URNs that are known to DataHub + /// 3. The list of owner users that are not know to DataHub + /// 2. The list of owner group URNs + pub fn owners_for_type(&self, owner_type: &OwnerType) -> (Vec, Vec, Vec) { + let owners = self + .aspects .ownership .iter() .flat_map(|owner| &owner.value.owners) - .map(|owner| owner.owner_urn.clone()) - .collect() + .filter(|owner| &OwnerType::from(&owner.type_) == owner_type); + + let user_urns = owners + .clone() + .filter_map(|owner| match &owner.type_ { + AspectsOwnershipValueOwnerType::Urn { type_urn } + if owner.owner.starts_with("urn:li:corpuser:") => + { + Some(Urn(owner.owner.to_owned())) + } + _ => None, + }) + .collect(); + let users_without_urn = owners + .clone() + .filter_map(|owner| match &owner.type_ { + AspectsOwnershipValueOwnerType::Raw { .. } => Some(owner.owner.to_owned()), + AspectsOwnershipValueOwnerType::Urn { .. } => None, + }) + .collect(); + let group_urns = owners + .clone() + .filter_map(|owner| match &owner.type_ { + AspectsOwnershipValueOwnerType::Urn { type_urn } + if owner.owner.starts_with("urn:li:corpGroup:") => + { + Some(Urn(owner.owner.to_owned())) + } + _ => None, + }) + .collect(); + + (user_urns, users_without_urn, group_urns) + } + + pub fn tag_properties(&self) -> Option<&AspectsTagPropertiesValue> { + self.aspects + .tag_properties + .as_ref() + .map(|properties| &properties.value) + } + + pub fn user_info(&self) -> Option<&AspectsCorpUserInfoValue> { + self.aspects.corp_user_info.as_ref().map(|info| &info.value) + } + + pub fn group_info(&self) -> Option<&AspectsCorpGroupInfoValue> { + self.aspects + .corp_group_info + .as_ref() + .map(|info| &info.value) } } diff --git a/rust/resource-info-fetcher/src/main.rs b/rust/resource-info-fetcher/src/main.rs index fe124425..544bc1dd 100644 --- a/rust/resource-info-fetcher/src/main.rs +++ b/rust/resource-info-fetcher/src/main.rs @@ -84,12 +84,13 @@ enum StartupError { /// contains both the configuration and the resolved credentials. async fn resolve_backend( backend: v1alpha1::Backend, + cache: &stackable_opa_operator::crd::cache::Cache, credentials_dir: &Path, ) -> Result { match backend { v1alpha1::Backend::DataHub(config) => { let resolved = - backend::data_hub::ResolvedDataHubBackend::resolve(config, credentials_dir) + backend::data_hub::ResolvedDataHubBackend::resolve(config, cache, credentials_dir) .await .context(ResolveDataHubBackendSnafu)?; Ok(ResolvedBackend::DataHub(resolved)) @@ -131,7 +132,8 @@ async fn main() -> Result<(), StartupError> { let config: v1alpha1::Config = read_config_file(&args.config) .with_context(|_| ParseConfigFileSnafu { path: args.config })?; - let backend = Arc::new(resolve_backend(config.backend, &args.credentials_dir).await?); + let backend = + Arc::new(resolve_backend(config.backend, &config.cache, &args.credentials_dir).await?); let resource_info_cache = { Cache::builder() .name("resource-info") From f28fdb790a70b346dd37a103644fa7ba6c00d966 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 20 Jul 2026 09:55:33 +0200 Subject: [PATCH 04/13] feat: Lookup Superset things (by id) --- rust/resource-info-fetcher/src/api.rs | 9 ++++++--- .../src/backend/data_hub/resource_to_urn_mapping.rs | 8 ++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/rust/resource-info-fetcher/src/api.rs b/rust/resource-info-fetcher/src/api.rs index 390fec9d..23f382e3 100644 --- a/rust/resource-info-fetcher/src/api.rs +++ b/rust/resource-info-fetcher/src/api.rs @@ -42,9 +42,12 @@ pub enum ResourceInfoRequestResource { catalog: String, }, - SupersetChart(String), - SupersetDashboard(String), - + SupersetChart { + id: u64, + }, + SupersetDashboard { + id: u64, + }, RawDataHubUrn(String), } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs index cce304a7..2549f1cb 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs @@ -33,8 +33,12 @@ pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> Urn { ("schema", schema.as_str()), ])) } - ResourceInfoRequestResource::SupersetChart(_) => todo!(), - ResourceInfoRequestResource::SupersetDashboard(_) => todo!(), + ResourceInfoRequestResource::SupersetChart { id } => { + format!("urn:li:chart:({stacklet},{id})") + } + ResourceInfoRequestResource::SupersetDashboard { id } => { + format!("urn:li:dashboard:({stacklet},{id})") + } ResourceInfoRequestResource::RawDataHubUrn(urn) => urn.to_owned(), }; From 90b010d2b644506f8d1ef273b6bc1707b23d209d Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 20 Jul 2026 11:31:56 +0200 Subject: [PATCH 05/13] chore: More parallism --- Cargo.lock | 182 ++++++++++-------- .../src/backend/data_hub/mod.rs | 71 ++++--- 2 files changed, 145 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49af0902..b7cdf7da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" @@ -174,13 +174,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.2", ] [[package]] @@ -197,9 +197,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ "aws-lc-sys", "zeroize", @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", @@ -385,9 +385,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.67" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -706,7 +706,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -975,9 +975,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "ff" @@ -1065,9 +1065,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -1080,9 +1080,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -1090,15 +1090,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -1107,15 +1107,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", @@ -1124,21 +1124,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -1682,11 +1682,12 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.32" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" +checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16" dependencies = [ "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "log", @@ -1696,12 +1697,22 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + [[package]] name = "jiff-static" -version = "0.2.32" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" +checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.119", @@ -1753,7 +1764,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -1766,7 +1777,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -1875,7 +1886,7 @@ dependencies = [ "serde", "serde-saphyr", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tower", @@ -1899,7 +1910,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -1937,7 +1948,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tracing", @@ -1980,7 +1991,7 @@ dependencies = [ "nom", "percent-encoding", "ring", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-native-tls", "tokio-stream", @@ -2321,7 +2332,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", ] @@ -2363,7 +2374,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tonic", "tonic-types", @@ -2401,7 +2412,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", ] @@ -2578,9 +2589,9 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -2636,9 +2647,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -2693,9 +2704,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -2781,22 +2792,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.2", ] [[package]] @@ -3099,9 +3110,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -3138,22 +3149,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.2", ] [[package]] @@ -3169,9 +3180,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -3740,6 +3751,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -3822,11 +3844,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.19", ] [[package]] @@ -3842,13 +3864,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.2", ] [[package]] @@ -3862,9 +3884,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.53" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", "num-conv", @@ -3882,9 +3904,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -3923,9 +3945,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.4" +version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af" +checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" dependencies = [ "bytes", "libc", @@ -3940,9 +3962,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", @@ -4132,7 +4154,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.18", + "thiserror 2.0.19", "time", "tracing-subscriber", ] diff --git a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs index 8020e571..9625b5de 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs @@ -202,7 +202,14 @@ impl ResolvedDataHubBackend { impl ResourceInfoBackend for ResolvedDataHubBackend { type Response = DataHubResourceInfoResponse; - // TODO: Do more parallelism + // The individual URN lookups already run concurrently via `try_join_all`. On top of that we + // overlap the coarse-grained groups as well: tags are fetched at the same time as the owners, + // all owner types are fetched concurrently, and within each owner type the user and group + // lookups run at the same time. + // + // TODO: This is unbounded concurrency. For entities with many owners/tags this can burst a lot + // of simultaneous requests at DataHub. If that becomes a problem, bound the concurrency (e.g. + // via `futures::stream::StreamExt::buffer_unordered` or a shared `tokio::sync::Semaphore`). #[instrument(skip(self))] async fn get_resource_info( &self, @@ -210,28 +217,39 @@ impl ResourceInfoBackend for ResolvedDataHubBackend { ) -> Result { let urn = urn_for_request(request, &self.config.env); let entity_response: DataHubEntityResponse = self.query_entity(&urn).await?; + let entity_response = &entity_response; - let tag_urns = entity_response.tag_urns(); - let tags = try_join_all( - tag_urns - .iter() - .map(|urn| async { self.query_tag(urn).await }), - ) - .await - .context(GetDataHubUserSnafu)?; + // Fetch the tags of the entity. + let tags_fut = async move { + let tag_urns = entity_response.tag_urns(); + try_join_all( + tag_urns + .iter() + .map(|urn| async { self.query_tag(urn).await }), + ) + .await + .context(GetDataHubUserSnafu) + }; - let mut owners = BTreeMap::new(); - for owner_type in OwnerType::iter() { + // Fetch the owners of every type. + let owners_fut = try_join_all(OwnerType::iter().map(|owner_type| async move { let (user_urns, users_without_urn, group_urns) = entity_response.owners_for_type(&owner_type); - let mut users = try_join_all( - user_urns - .iter() - .map(|urn| async { self.query_user(urn).await }), + let (mut users, groups) = futures::try_join!( + try_join_all( + user_urns + .iter() + .map(|urn| async { self.query_user(urn).await }), + ), + try_join_all( + group_urns + .iter() + .map(|urn| async { self.query_group(urn).await }), + ), ) - .await .context(GetDataHubUserSnafu)?; + users.extend( users_without_urn .iter() @@ -244,21 +262,18 @@ impl ResourceInfoBackend for ResolvedDataHubBackend { }), ); - let groups = try_join_all( - group_urns - .iter() - .map(|urn| async { self.query_group(urn).await }), - ) - .await - .context(GetDataHubUserSnafu)?; - - owners.insert( + Ok::<_, GetResourceInfoError>(( owner_type, DataHubResourceInfoResponseOwners { users, groups }, - ); - } + )) + })); - Ok(DataHubResourceInfoResponse { tags, owners }) + let (tags, owners) = futures::try_join!(tags_fut, owners_fut)?; + + Ok(DataHubResourceInfoResponse { + tags, + owners: owners.into_iter().collect(), + }) } } From 53e83c6c306d5a33a516cb4ef5e5af47395d6636 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 20 Jul 2026 12:19:58 +0200 Subject: [PATCH 06/13] feat: Add Kafka topic support --- rust/resource-info-fetcher/src/api.rs | 5 +++++ rust/resource-info-fetcher/src/backend/data_hub/mod.rs | 2 ++ .../src/backend/data_hub/resource_to_urn_mapping.rs | 3 +++ 3 files changed, 10 insertions(+) diff --git a/rust/resource-info-fetcher/src/api.rs b/rust/resource-info-fetcher/src/api.rs index 23f382e3..5d82eed3 100644 --- a/rust/resource-info-fetcher/src/api.rs +++ b/rust/resource-info-fetcher/src/api.rs @@ -48,6 +48,11 @@ pub enum ResourceInfoRequestResource { SupersetDashboard { id: u64, }, + + KafkaTopic { + topic: String, + }, + RawDataHubUrn(String), } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs index 9625b5de..ce7410e0 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs @@ -110,6 +110,7 @@ pub struct ResolvedDataHubBackend { #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] #[serde(rename_all = "camelCase")] pub struct DataHubResourceInfoResponse { + urn: Urn, owners: BTreeMap, tags: Vec, } @@ -271,6 +272,7 @@ impl ResourceInfoBackend for ResolvedDataHubBackend { let (tags, owners) = futures::try_join!(tags_fut, owners_fut)?; Ok(DataHubResourceInfoResponse { + urn, tags, owners: owners.into_iter().collect(), }) diff --git a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs index 2549f1cb..4afdce37 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs @@ -39,6 +39,9 @@ pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> Urn { ResourceInfoRequestResource::SupersetDashboard { id } => { format!("urn:li:dashboard:({stacklet},{id})") } + ResourceInfoRequestResource::KafkaTopic { topic } => { + format!("urn:li:dataset:(urn:li:dataPlatform:{stacklet},{topic},{env})") + } ResourceInfoRequestResource::RawDataHubUrn(urn) => urn.to_owned(), }; From 0e2857a3422346bb11c29700e50befeab7b9cb63 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 20 Jul 2026 14:36:39 +0200 Subject: [PATCH 07/13] Crying in GraphQL --- Cargo.lock | 8 - rust/resource-info-fetcher/Cargo.toml | 2 - rust/resource-info-fetcher/src/api.rs | 11 - .../src/backend/data_hub/graphql.rs | 278 +++++++++++++ .../src/backend/data_hub/mod.rs | 387 ++++++------------ .../data_hub/resource_to_urn_mapping.rs | 2 +- .../src/backend/data_hub/upstream_api.rs | 236 ----------- rust/resource-info-fetcher/src/main.rs | 6 +- 8 files changed, 414 insertions(+), 516 deletions(-) create mode 100644 rust/resource-info-fetcher/src/backend/data_hub/graphql.rs delete mode 100644 rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs diff --git a/Cargo.lock b/Cargo.lock index b7cdf7da..31c8162e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3491,11 +3491,9 @@ dependencies = [ "snafu 0.9.1", "stackable-opa-operator", "stackable-operator", - "strum", "tokio", "tracing", "url", - "urlencoding", ] [[package]] @@ -4305,12 +4303,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/rust/resource-info-fetcher/Cargo.toml b/rust/resource-info-fetcher/Cargo.toml index a8c317d0..684bd835 100644 --- a/rust/resource-info-fetcher/Cargo.toml +++ b/rust/resource-info-fetcher/Cargo.toml @@ -23,11 +23,9 @@ reqwest.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true -strum.workspace = true tokio.workspace = true tracing.workspace = true url.workspace = true -urlencoding.workspace = true [build-dependencies] built.workspace = true diff --git a/rust/resource-info-fetcher/src/api.rs b/rust/resource-info-fetcher/src/api.rs index 5d82eed3..da8702b9 100644 --- a/rust/resource-info-fetcher/src/api.rs +++ b/rust/resource-info-fetcher/src/api.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use hyper::StatusCode; use info_fetcher_commons::http_error; use serde::{Deserialize, Serialize}; @@ -66,14 +64,6 @@ pub enum GetResourceInfoError { display("failed to get resource information from DataHub") )] DataHub { source: backend::data_hub::Error }, - - #[snafu( - visibility(pub(crate)), - display("failed to get user information from DataHub") - )] - GetDataHubUser { - source: Arc, - }, } impl http_error::Error for GetResourceInfoError { @@ -87,7 +77,6 @@ impl http_error::Error for GetResourceInfoError { match self { Self::SerializeResponseAsJson { .. } => StatusCode::INTERNAL_SERVER_ERROR, Self::DataHub { source } => source.status_code(), - Self::GetDataHubUser { source } => source.status_code(), } } } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/graphql.rs b/rust/resource-info-fetcher/src/backend/data_hub/graphql.rs new file mode 100644 index 00000000..25925e98 --- /dev/null +++ b/rust/resource-info-fetcher/src/backend/data_hub/graphql.rs @@ -0,0 +1,278 @@ +//! DataHub GraphQL query and the types used to (de)serialize it. +//! +//! A single `POST /api/graphql` fetches the entity together with its tags and owners, and DataHub +//! resolves the referenced tag/user/group and ownership-type entities server-side. The [`Entity`] +//! response is then flattened into the crate's public [`DataHubResourceInfoResponse`]. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::backend::data_hub::{DataHubResourceInfoResponse, Group, Owners, Tag, Urn, User}; + +/// A single query covering every entity kind we build URNs for. We use the generic `entity(urn:)` +/// resolver plus per-type inline fragments, because a request can target a dataset (Trino table or +/// Kafka topic), a container (Trino catalog or schema), a chart or a dashboard. +const RESOURCE_INFO_QUERY: &str = r" +query ResourceInfo($urn: String!) { + entity(urn: $urn) { + ...ResourceInfo + } +} +fragment ResourceInfo on Entity { + ... on Dataset { tags { ...Tags } ownership { ...Owners } } + ... on Container { tags { ...Tags } ownership { ...Owners } } + ... on Chart { tags { ...Tags } ownership { ...Owners } } + ... on Dashboard { tags { ...Tags } ownership { ...Owners } } +} +fragment Tags on GlobalTags { + tags { tag { urn properties { name } } } +} +fragment Owners on Ownership { + owners { + owner { + __typename + ... on CorpUser { urn properties { fullName displayName email active } } + ... on CorpGroup { urn properties { displayName description } } + } + ownershipType { urn info { name } } + type + } +} +"; + +/// Builds the request body for the [`RESOURCE_INFO_QUERY`], parameterized by the entity's URN. +pub fn request(urn: &Urn) -> GraphQlRequest<'_> { + GraphQlRequest { + query: RESOURCE_INFO_QUERY, + variables: Variables { urn: &urn.0 }, + } +} + +#[derive(Debug, Serialize)] +pub struct GraphQlRequest<'a> { + query: &'static str, + variables: Variables<'a>, +} + +#[derive(Debug, Serialize)] +struct Variables<'a> { + urn: &'a str, +} + +/// A GraphQL server answers `200 OK` even when the query fails; the failures are reported in +/// `errors`. Callers must therefore inspect `errors` explicitly rather than relying on the HTTP +/// status code. +#[derive(Debug, Deserialize)] +pub struct GraphQlResponse { + pub data: Option, + + #[serde(default)] + pub errors: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct GraphQlError { + pub message: String, +} + +#[derive(Debug, Deserialize)] +pub struct ResponseData { + pub entity: Option, +} + +/// The resolved entity. `tags` and `ownership` come from the per-type inline fragments; GraphQL +/// merges them onto the entity object, so a single flat struct reads them regardless of the +/// concrete entity type. [`Default`] yields the "no metadata" entity used when DataHub does not +/// return an entity for a URN. +#[derive(Debug, Default, Deserialize)] +pub struct Entity { + tags: Option, + ownership: Option, +} + +#[derive(Debug, Deserialize)] +struct GlobalTags { + tags: Vec, +} + +#[derive(Debug, Deserialize)] +struct TagAssociation { + tag: TagNode, +} + +#[derive(Debug, Deserialize)] +struct TagNode { + urn: Urn, + properties: Option, +} + +#[derive(Debug, Deserialize)] +struct TagProperties { + name: String, +} + +#[derive(Debug, Deserialize)] +struct Ownership { + owners: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Owner { + owner: OwnerEntity, + + /// The modern ownership type entity, e.g. `urn:li:ownershipType:__system__technical_owner`. + ownership_type: Option, + + /// The legacy ownership type enum, e.g. `TECHNICAL_OWNER`. Used as a fallback key for owners + /// that predate ownership type entities. + #[serde(rename = "type")] + legacy_type: Option, +} + +#[derive(Debug, Deserialize)] +struct OwnershipType { + urn: Urn, + info: Option, +} + +#[derive(Debug, Deserialize)] +struct OwnershipTypeInfo { + name: String, +} + +/// The resolved owner. DataHub only ever resolves owners to users or groups. +#[derive(Debug, Deserialize)] +#[serde(tag = "__typename")] +enum OwnerEntity { + CorpUser { + urn: Urn, + properties: Option, + }, + CorpGroup { + urn: Urn, + properties: Option, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CorpUserProperties { + full_name: Option, + display_name: Option, + email: Option, + active: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CorpGroupProperties { + display_name: Option, + description: Option, +} + +impl Entity { + /// Flattens the GraphQL response into the crate's public [`DataHubResourceInfoResponse`]. + pub fn into_response(self, urn: Urn) -> DataHubResourceInfoResponse { + let tags = self + .tags + .into_iter() + .flat_map(|global_tags| global_tags.tags) + .map(|association| { + let TagNode { urn, properties } = association.tag; + // A tag without a properties aspect has no name; fall back to its URN. + let name = properties + .map(|properties| properties.name) + .unwrap_or_else(|| urn.0.clone()); + Tag { urn, name } + }) + .collect(); + + let mut owners: BTreeMap = BTreeMap::new(); + for owner in self + .ownership + .into_iter() + .flat_map(|ownership| ownership.owners) + { + let (type_urn, type_name) = owner_type(owner.ownership_type, owner.legacy_type); + let bucket = owners.entry(type_urn).or_default(); + if bucket.ownership_type_name.is_none() { + bucket.ownership_type_name = type_name; + } + match owner.owner { + OwnerEntity::CorpUser { urn, properties } => { + bucket.users.push(user(urn, properties)) + } + OwnerEntity::CorpGroup { urn, properties } => { + bucket.groups.push(group(urn, properties)) + } + } + } + + DataHubResourceInfoResponse { urn, tags, owners } + } +} + +/// Resolves the map key and human-readable name for an owner's ownership type, preferring the +/// modern ownership type entity and falling back to the legacy `type` enum. Unlike the previous +/// REST implementation, this handles arbitrary (including user-defined) ownership types instead of +/// assuming a fixed enum. +fn owner_type( + ownership_type: Option, + legacy_type: Option, +) -> (Urn, Option) { + match ownership_type { + Some(ownership_type) => ( + ownership_type.urn, + ownership_type.info.map(|info| info.name), + ), + // Fallback for owners where DataHub only populated the legacy `type` field. + None => ( + Urn(legacy_type.unwrap_or_else(|| "unknown".to_owned())), + None, + ), + } +} + +fn user(urn: Urn, properties: Option) -> User { + match properties { + Some(properties) => User { + full_name: properties.full_name, + display_name: properties + .display_name + .unwrap_or_else(|| strip_user_urn(&urn)), + email: properties.email, + active: properties.active.unwrap_or(true), + urn, + }, + // An owner reference without a corpUserInfo aspect: derive a display name from the URN. + None => User { + full_name: None, + display_name: strip_user_urn(&urn), + email: None, + active: true, + urn, + }, + } +} + +fn group(urn: Urn, properties: Option) -> Group { + let (display_name, description) = match properties { + Some(properties) => ( + properties.display_name.unwrap_or_else(|| urn.0.clone()), + properties.description, + ), + None => (urn.0.clone(), None), + }; + + Group { + urn, + display_name, + description, + } +} + +fn strip_user_urn(urn: &Urn) -> String { + urn.0.trim_start_matches("urn:li:corpuser:").to_owned() +} diff --git a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs index ce7410e0..d40fcb65 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs @@ -1,32 +1,27 @@ -use std::{collections::BTreeMap, path::Path, sync::Arc}; +use std::{ + collections::BTreeMap, + fmt::{self, Display}, + path::Path, +}; -use futures::future::try_join_all; use hyper::StatusCode; use info_fetcher_commons::{ http_error, utils::{self, http::send_json_request}, }; -use moka::future::Cache; use reqwest::{ClientBuilder, Url}; -use serde::Serialize; -use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_opa_operator::crd::resource_info_fetcher::v1alpha1::{self, DataHubBackend}; -use strum::IntoEnumIterator; +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt, Snafu}; +use stackable_opa_operator::crd::resource_info_fetcher::v1alpha1; use tracing::{debug, instrument, trace}; use crate::{ - api::{GetDataHubUserSnafu, GetResourceInfoError, ResourceInfoBackend, ResourceInfoRequest}, - backend::data_hub::{ - resource_to_urn_mapping::urn_for_request, - upstream_api::{ - AspectsCorpGroupInfoValue, AspectsCorpUserInfoValue, AspectsOwnershipValueOwnerType, - DataHubEntityResponse, OwnerTypeUrn, RawOwnerType, Tag, Urn, - }, - }, + api::{GetResourceInfoError, ResourceInfoBackend, ResourceInfoRequest}, + backend::data_hub::resource_to_urn_mapping::urn_for_request, }; +mod graphql; mod resource_to_urn_mapping; -mod upstream_api; #[derive(Snafu, Debug)] pub enum Error { @@ -48,32 +43,20 @@ pub enum Error { #[snafu(display("failed to construct HTTP client"))] ConstructHttpClient { source: reqwest::Error }, - #[snafu(display("failed to to build DataHub endpoint for {endpoint}"))] + #[snafu(display("failed to build DataHub GraphQL endpoint {endpoint:?}"))] BuildDataHubEndpoint { source: url::ParseError, endpoint: String, }, - #[snafu(display("failed to query for entity with URN {urn:?}"))] - QueryForUrn { + #[snafu(display("failed to execute GraphQL query for URN {urn:?}"))] + ExecuteGraphQlQuery { source: utils::http::Error, urn: Urn, }, - #[snafu(display( - "the entity information send by DataHub for the tag with the URN {urn:?} must contain tag properties" - ))] - EntityResponseMustContainTagProperties { urn: Urn }, - - #[snafu(display( - "the entity information send by DataHub for the user with the URN {urn:?} must contain user information" - ))] - EntityResponseMustContainUserInfo { urn: Urn }, - - #[snafu(display( - "the entity information send by DataHub for the group with the URN {urn:?} must contain group information" - ))] - EntityResponseMustContainGroupInfo { urn: Urn }, + #[snafu(display("DataHub returned GraphQL errors for URN {urn:?}: {messages}"))] + GraphQlErrors { messages: String, urn: Urn }, } impl http_error::Error for Error { @@ -84,67 +67,80 @@ impl http_error::Error for Error { Self::ConfigureTls { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::ConstructHttpClient { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::BuildDataHubEndpoint { .. } => StatusCode::BAD_REQUEST, - Self::QueryForUrn { .. } => StatusCode::INTERNAL_SERVER_ERROR, - Self::EntityResponseMustContainTagProperties { .. } => { - StatusCode::INTERNAL_SERVER_ERROR - } - Self::EntityResponseMustContainUserInfo { .. } => StatusCode::INTERNAL_SERVER_ERROR, - Self::EntityResponseMustContainGroupInfo { .. } => StatusCode::INTERNAL_SERVER_ERROR, + Self::ExecuteGraphQlQuery { .. } => StatusCode::INTERNAL_SERVER_ERROR, + Self::GraphQlErrors { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } } -/// This struct combines the CRD configuration with credentials loaded from the filesystem. -/// Credentials and the HTTP client are initialized once at startup and stored internally. -pub struct ResolvedDataHubBackend { - config: v1alpha1::DataHubBackend, - client_id: String, - client_secret: String, - http_client: reqwest::Client, +/// A DataHub URN, e.g. `urn:li:corpuser:alice` or `urn:li:tag:pii`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct Urn(pub String); - tag_cache: Cache, - user_cache: Cache, - group_cache: Cache, +impl Display for Urn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +/// The information we return for a single resource. Assembled from a single DataHub GraphQL query +/// that resolves the resource's tags and owners (with their user/group details) server-side. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct DataHubResourceInfoResponse { urn: Urn, - owners: BTreeMap, tags: Vec, + + /// Owners grouped by their ownership type URN, e.g. + /// `urn:li:ownershipType:__system__technical_owner`. DataHub has no fixed set of ownership + /// types — users can define custom ones — so this is an open map, not an enum. + owners: BTreeMap, } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, PartialOrd, Ord, strum::EnumIter)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Default)] #[serde(rename_all = "camelCase")] -pub enum OwnerType { - BusinessOwner, - TechnicalOwner, - DataSteward, - None, +pub struct Owners { + /// Human-readable name of the ownership type, e.g. "Technical Owner". [`None`] if DataHub did + /// not resolve a name for the type. + ownership_type_name: Option, + users: Vec, + groups: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct DataHubResourceInfoResponseOwners { - users: Vec, - groups: Vec, +pub struct Tag { + urn: Urn, + name: String, } -impl From<&AspectsOwnershipValueOwnerType> for OwnerType { - fn from(value: &AspectsOwnershipValueOwnerType) -> Self { - match value { - AspectsOwnershipValueOwnerType::Urn { type_urn } => match type_urn { - OwnerTypeUrn::BusinessOwner => Self::BusinessOwner, - OwnerTypeUrn::TechnicalOwner => Self::TechnicalOwner, - OwnerTypeUrn::DataSteward => Self::DataSteward, - OwnerTypeUrn::None => Self::None, - }, - AspectsOwnershipValueOwnerType::Raw { - type_: RawOwnerType::TechnicalOwner, - } => Self::TechnicalOwner, - } - } +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct User { + urn: Urn, + full_name: Option, + display_name: String, + email: Option, + active: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Group { + urn: Urn, + display_name: String, + description: Option, +} + +/// This struct combines the CRD configuration with credentials loaded from the filesystem. +/// Credentials, the HTTP client and the GraphQL endpoint are initialized once at startup and stored +/// internally. +pub struct ResolvedDataHubBackend { + config: v1alpha1::DataHubBackend, + client_id: String, + client_secret: String, + http_client: reqwest::Client, + graphql_url: Url, } impl ResolvedDataHubBackend { @@ -152,7 +148,6 @@ impl ResolvedDataHubBackend { #[instrument(skip_all)] pub async fn resolve( config: v1alpha1::DataHubBackend, - cache: &stackable_opa_operator::crd::cache::Cache, credentials_dir: &Path, ) -> Result { let client_id_path = credentials_dir.join("clientId"); @@ -175,211 +170,95 @@ impl ResolvedDataHubBackend { .context(ConfigureTlsSnafu)?; let http_client = client_builder.build().context(ConstructHttpClientSnafu)?; - let tag_cache = Cache::builder() - .time_to_live(*cache.entry_time_to_live) - .name("tag-info") - .build(); - let user_cache = Cache::builder() - .time_to_live(*cache.entry_time_to_live) - .name("user-info") - .build(); - let group_cache = Cache::builder() - .time_to_live(*cache.entry_time_to_live) - .name("group-info") - .build(); + let schema = if config.tls.uses_tls() { + "https" + } else { + "http" + }; + let port = config + .port + .unwrap_or(if config.tls.uses_tls() { 443 } else { 80 }); + let graphql = format!( + "{schema}://{hostname}:{port}/api/graphql", + hostname = config.hostname + ); + let graphql_url = Url::parse(&graphql) + .with_context(|_| BuildDataHubEndpointSnafu { endpoint: graphql })?; Ok(Self { config, client_id, client_secret, http_client, - tag_cache, - user_cache, - group_cache, - }) - } -} - -impl ResourceInfoBackend for ResolvedDataHubBackend { - type Response = DataHubResourceInfoResponse; - - // The individual URN lookups already run concurrently via `try_join_all`. On top of that we - // overlap the coarse-grained groups as well: tags are fetched at the same time as the owners, - // all owner types are fetched concurrently, and within each owner type the user and group - // lookups run at the same time. - // - // TODO: This is unbounded concurrency. For entities with many owners/tags this can burst a lot - // of simultaneous requests at DataHub. If that becomes a problem, bound the concurrency (e.g. - // via `futures::stream::StreamExt::buffer_unordered` or a shared `tokio::sync::Semaphore`). - #[instrument(skip(self))] - async fn get_resource_info( - &self, - request: &ResourceInfoRequest, - ) -> Result { - let urn = urn_for_request(request, &self.config.env); - let entity_response: DataHubEntityResponse = self.query_entity(&urn).await?; - let entity_response = &entity_response; - - // Fetch the tags of the entity. - let tags_fut = async move { - let tag_urns = entity_response.tag_urns(); - try_join_all( - tag_urns - .iter() - .map(|urn| async { self.query_tag(urn).await }), - ) - .await - .context(GetDataHubUserSnafu) - }; - - // Fetch the owners of every type. - let owners_fut = try_join_all(OwnerType::iter().map(|owner_type| async move { - let (user_urns, users_without_urn, group_urns) = - entity_response.owners_for_type(&owner_type); - - let (mut users, groups) = futures::try_join!( - try_join_all( - user_urns - .iter() - .map(|urn| async { self.query_user(urn).await }), - ), - try_join_all( - group_urns - .iter() - .map(|urn| async { self.query_group(urn).await }), - ), - ) - .context(GetDataHubUserSnafu)?; - - users.extend( - users_without_urn - .iter() - .map(|username| AspectsCorpUserInfoValue { - full_name: None, - display_name: username.trim_start_matches("urn:li:corpuser").to_owned(), - email: None, - active: true, - data_hub_user: false, - }), - ); - - Ok::<_, GetResourceInfoError>(( - owner_type, - DataHubResourceInfoResponseOwners { users, groups }, - )) - })); - - let (tags, owners) = futures::try_join!(tags_fut, owners_fut)?; - - Ok(DataHubResourceInfoResponse { - urn, - tags, - owners: owners.into_iter().collect(), + graphql_url, }) } -} -impl ResolvedDataHubBackend { + /// Fetches a resource, its tags and its owners (including the owning users' and groups' details) + /// in a single DataHub GraphQL query. DataHub resolves the referenced tag/user/group entities + /// server-side, so there is no client-side fan-out. #[instrument(skip(self))] - async fn query_entity(&self, urn: &Urn) -> Result { - let Self { - config: - DataHubBackend { - hostname, - port, - tls, - .. - }, - client_id, - client_secret, - http_client, - .. - } = &self; - - let schema = if tls.uses_tls() { "https" } else { "http" }; - let port = port.unwrap_or(if tls.uses_tls() { 443 } else { 80 }); - - let entity = format!( - "{schema}://{hostname}:{port}/entitiesV2/{url_encoded_urn}", - url_encoded_urn = urlencoding::encode(&urn.0) - ); - let entity_url = - Url::parse(&entity).with_context(|_| BuildDataHubEndpointSnafu { endpoint: entity })?; - - trace!(%entity_url, "Sending request to DataHub's entity API"); + async fn query_entity(&self, urn: &Urn) -> Result { + trace!(%urn, %self.graphql_url, "Sending GraphQL query to DataHub"); - let entity = send_json_request( - http_client - .get(entity_url.clone()) + let response: graphql::GraphQlResponse = send_json_request( + self.http_client + .post(self.graphql_url.clone()) // DataHub's system authenticator strips the leading "Basic " and compares the rest // VERBATIM against ":" — it is NOT standard RFC 7617 Basic auth. So do // NOT use reqwest's .basic_auth(), which base64-encodes the credentials; set the // header manually so the value goes out unencoded. .header( reqwest::header::AUTHORIZATION, - format!("Basic {client_id}:{client_secret}"), - ), + format!("Basic {}:{}", self.client_id, self.client_secret), + ) + .json(&graphql::request(urn)), ) .await - .with_context(|_| QueryForUrnSnafu { urn: urn.clone() })?; + .with_context(|_| ExecuteGraphQlQuerySnafu { urn: urn.clone() })?; + + // GraphQL reports genuine problems (unresolvable URNs, resolver failures) in `errors` while + // still returning `200 OK`. These are actual errors — not a missing resource — so we fail + // the request rather than returning incomplete data. + if !response.errors.is_empty() { + let messages = response + .errors + .iter() + .map(|error| error.message.as_str()) + .collect::>() + .join("; "); + return GraphQlErrorsSnafu { + messages, + urn: urn.clone(), + } + .fail(); + } - debug!(%urn, %entity_url, "Fetched entity from DataHub"); + // A resource that is unknown to DataHub is reported as a null entity (with no errors). It + // simply has no metadata, so we return an empty response rather than failing the request. + let Some(entity) = response.data.and_then(|data| data.entity) else { + debug!(%urn, "DataHub returned no entity; responding with empty resource info"); + return Ok(graphql::Entity::default()); + }; + + debug!(%urn, "Fetched entity from DataHub via GraphQL"); trace!(?entity, "DataHub entity payload"); Ok(entity) } +} - #[instrument(skip(self))] - async fn query_tag(&self, urn: &Urn) -> Result> { - self.tag_cache - .try_get_with_by_ref(urn, async { - let tag = self - .query_entity(urn) - .await? - .tag_properties() - .with_context(|| EntityResponseMustContainTagPropertiesSnafu { - urn: urn.clone(), - })? - .name - .clone(); - debug!(%urn, %tag, "Fetched tag from DataHub"); - - Ok(tag) - }) - .await - } +impl ResourceInfoBackend for ResolvedDataHubBackend { + type Response = DataHubResourceInfoResponse; #[instrument(skip(self))] - async fn query_user(&self, urn: &Urn) -> Result> { - self.user_cache - .try_get_with_by_ref(urn, async { - let user = self - .query_entity(urn) - .await? - .user_info() - .with_context(|| EntityResponseMustContainUserInfoSnafu { urn: urn.clone() })? - .clone(); - debug!(%urn, ?user, "Fetched user from DataHub"); - - Ok(user) - }) - .await - } + async fn get_resource_info( + &self, + request: &ResourceInfoRequest, + ) -> Result { + let urn = urn_for_request(request, &self.config.env); + let entity = self.query_entity(&urn).await?; - #[instrument(skip(self))] - async fn query_group(&self, urn: &Urn) -> Result> { - self.group_cache - .try_get_with_by_ref(urn, async { - let group = self - .query_entity(urn) - .await? - .group_info() - .with_context(|| EntityResponseMustContainGroupInfoSnafu { urn: urn.clone() })? - .clone(); - debug!(%urn, ?group, "Fetched group from DataHub"); - - Ok(group) - }) - .await + Ok(entity.into_response(urn)) } } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs index 4afdce37..3ca4eadc 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use crate::{ api::{ResourceInfoRequest, ResourceInfoRequestResource}, - backend::data_hub::upstream_api::Urn, + backend::data_hub::Urn, }; pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> Urn { diff --git a/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs b/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs deleted file mode 100644 index 0d547251..00000000 --- a/rust/resource-info-fetcher/src/backend/data_hub/upstream_api.rs +++ /dev/null @@ -1,236 +0,0 @@ -use std::fmt::{self, Display}; - -use serde::{Deserialize, Serialize}; - -use crate::backend::data_hub::OwnerType; - -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Urn(pub String); - -impl Display for Urn { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Tag(pub String); - -impl Display for Tag { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DataHubEntityResponse { - aspects: Aspects, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Aspects { - global_tags: Option, - ownership: Option, - tag_properties: Option, - corp_user_info: Option, - corp_group_info: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsGlobalTags { - value: AspectsGlobalTagsValue, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsGlobalTagsValue { - tags: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsGlobalTagsValueTag { - #[serde(rename = "tag")] - tag_urn: Urn, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsOwnership { - value: AspectsOwnershipValue, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsOwnershipValue { - owners: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsOwnershipValueOwner { - owner: String, - - #[serde(flatten)] - type_: AspectsOwnershipValueOwnerType, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(untagged)] -pub enum AspectsOwnershipValueOwnerType { - Urn { - #[serde(rename = "typeUrn")] - type_urn: OwnerTypeUrn, - }, - Raw { - #[serde(rename = "type")] - type_: RawOwnerType, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum OwnerTypeUrn { - #[serde(rename = "urn:li:ownershipType:__system__business_owner")] - BusinessOwner, - #[serde(rename = "urn:li:ownershipType:__system__technical_owner")] - TechnicalOwner, - #[serde(rename = "urn:li:ownershipType:__system__data_steward")] - DataSteward, - #[serde(rename = "urn:li:ownershipType:__system__none")] - None, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum RawOwnerType { - #[serde(rename = "TECHNICAL_OWNER")] - TechnicalOwner, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsTagProperties { - value: AspectsTagPropertiesValue, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsTagPropertiesValue { - pub name: Tag, - color_hex: String, - description: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsCorpUserInfo { - value: AspectsCorpUserInfoValue, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsCorpUserInfoValue { - pub full_name: Option, - pub display_name: String, - pub email: Option, - pub active: bool, - - /// We default this field to `true`, so that we set it during deserialization from DataHub. When - /// "faking" a user that doesn't exist in DataHub, we set it explicitly to `false`. - #[serde(default = "true_default")] - pub data_hub_user: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsCorpGroupInfo { - value: AspectsCorpGroupInfoValue, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AspectsCorpGroupInfoValue { - display_name: String, - description: String, -} - -fn true_default() -> bool { - true -} - -impl DataHubEntityResponse { - pub fn tag_urns(&self) -> Vec { - self.aspects - .global_tags - .iter() - .flat_map(|global_tag| &global_tag.value.tags) - .map(|tag| tag.tag_urn.clone()) - .collect() - } - - /// Returns - /// - /// 1. The list of owner user URNs that are known to DataHub - /// 3. The list of owner users that are not know to DataHub - /// 2. The list of owner group URNs - pub fn owners_for_type(&self, owner_type: &OwnerType) -> (Vec, Vec, Vec) { - let owners = self - .aspects - .ownership - .iter() - .flat_map(|owner| &owner.value.owners) - .filter(|owner| &OwnerType::from(&owner.type_) == owner_type); - - let user_urns = owners - .clone() - .filter_map(|owner| match &owner.type_ { - AspectsOwnershipValueOwnerType::Urn { type_urn } - if owner.owner.starts_with("urn:li:corpuser:") => - { - Some(Urn(owner.owner.to_owned())) - } - _ => None, - }) - .collect(); - let users_without_urn = owners - .clone() - .filter_map(|owner| match &owner.type_ { - AspectsOwnershipValueOwnerType::Raw { .. } => Some(owner.owner.to_owned()), - AspectsOwnershipValueOwnerType::Urn { .. } => None, - }) - .collect(); - let group_urns = owners - .clone() - .filter_map(|owner| match &owner.type_ { - AspectsOwnershipValueOwnerType::Urn { type_urn } - if owner.owner.starts_with("urn:li:corpGroup:") => - { - Some(Urn(owner.owner.to_owned())) - } - _ => None, - }) - .collect(); - - (user_urns, users_without_urn, group_urns) - } - - pub fn tag_properties(&self) -> Option<&AspectsTagPropertiesValue> { - self.aspects - .tag_properties - .as_ref() - .map(|properties| &properties.value) - } - - pub fn user_info(&self) -> Option<&AspectsCorpUserInfoValue> { - self.aspects.corp_user_info.as_ref().map(|info| &info.value) - } - - pub fn group_info(&self) -> Option<&AspectsCorpGroupInfoValue> { - self.aspects - .corp_group_info - .as_ref() - .map(|info| &info.value) - } -} diff --git a/rust/resource-info-fetcher/src/main.rs b/rust/resource-info-fetcher/src/main.rs index 544bc1dd..fe124425 100644 --- a/rust/resource-info-fetcher/src/main.rs +++ b/rust/resource-info-fetcher/src/main.rs @@ -84,13 +84,12 @@ enum StartupError { /// contains both the configuration and the resolved credentials. async fn resolve_backend( backend: v1alpha1::Backend, - cache: &stackable_opa_operator::crd::cache::Cache, credentials_dir: &Path, ) -> Result { match backend { v1alpha1::Backend::DataHub(config) => { let resolved = - backend::data_hub::ResolvedDataHubBackend::resolve(config, cache, credentials_dir) + backend::data_hub::ResolvedDataHubBackend::resolve(config, credentials_dir) .await .context(ResolveDataHubBackendSnafu)?; Ok(ResolvedBackend::DataHub(resolved)) @@ -132,8 +131,7 @@ async fn main() -> Result<(), StartupError> { let config: v1alpha1::Config = read_config_file(&args.config) .with_context(|_| ParseConfigFileSnafu { path: args.config })?; - let backend = - Arc::new(resolve_backend(config.backend, &config.cache, &args.credentials_dir).await?); + let backend = Arc::new(resolve_backend(config.backend, &args.credentials_dir).await?); let resource_info_cache = { Cache::builder() .name("resource-info") From 551c2f66e1e152ee1f8124368e79e7fad1c0197e Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 21 Jul 2026 09:09:32 +0200 Subject: [PATCH 08/13] feat: Also fetch domains and data products --- .../src/backend/data_hub/graphql.rs | 115 ++++++++++++++++-- .../src/backend/data_hub/mod.rs | 25 ++++ 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/rust/resource-info-fetcher/src/backend/data_hub/graphql.rs b/rust/resource-info-fetcher/src/backend/data_hub/graphql.rs index 25925e98..95c23509 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/graphql.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/graphql.rs @@ -8,26 +8,42 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; -use crate::backend::data_hub::{DataHubResourceInfoResponse, Group, Owners, Tag, Urn, User}; +use crate::backend::data_hub::{ + DataHubResourceInfoResponse, DataProduct, Domain, Group, Owners, Tag, Urn, User, +}; /// A single query covering every entity kind we build URNs for. We use the generic `entity(urn:)` /// resolver plus per-type inline fragments, because a request can target a dataset (Trino table or /// Kafka topic), a container (Trino catalog or schema), a chart or a dashboard. -const RESOURCE_INFO_QUERY: &str = r" +const RESOURCE_INFO_QUERY: &str = r#" query ResourceInfo($urn: String!) { entity(urn: $urn) { ...ResourceInfo } } fragment ResourceInfo on Entity { - ... on Dataset { tags { ...Tags } ownership { ...Owners } } - ... on Container { tags { ...Tags } ownership { ...Owners } } - ... on Chart { tags { ...Tags } ownership { ...Owners } } - ... on Dashboard { tags { ...Tags } ownership { ...Owners } } + # DataHub has no direct "dataProduct" field on assets; membership is a graph edge that points from + # the data product to its assets. From the asset's side it is therefore an INCOMING relationship. + dataProducts: relationships(input: {types: ["DataProductContains"], direction: INCOMING, count: 10}) { + relationships { ...DataProduct } + } + ... on Dataset { tags { ...Tags } ownership { ...Owners } domain { ...Domain } } + ... on Container { tags { ...Tags } ownership { ...Owners } domain { ...Domain } } + ... on Chart { tags { ...Tags } ownership { ...Owners } domain { ...Domain } } + ... on Dashboard { tags { ...Tags } ownership { ...Owners } domain { ...Domain } } } fragment Tags on GlobalTags { tags { tag { urn properties { name } } } } +fragment Domain on DomainAssociation { + domain { urn properties { name description } } +} +fragment DataProduct on EntityRelationship { + entity { + urn + ... on DataProduct { properties { name description } } + } +} fragment Owners on Ownership { owners { owner { @@ -39,7 +55,7 @@ fragment Owners on Ownership { type } } -"; +"#; /// Builds the request body for the [`RESOURCE_INFO_QUERY`], parameterized by the entity's URN. pub fn request(urn: &Urn) -> GraphQlRequest<'_> { @@ -86,9 +102,12 @@ pub struct ResponseData { /// concrete entity type. [`Default`] yields the "no metadata" entity used when DataHub does not /// return an entity for a URN. #[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct Entity { tags: Option, ownership: Option, + domain: Option, + data_products: Option, } #[derive(Debug, Deserialize)] @@ -112,6 +131,47 @@ struct TagProperties { name: String, } +#[derive(Debug, Deserialize)] +struct DomainAssociation { + domain: DomainNode, +} + +#[derive(Debug, Deserialize)] +struct DomainNode { + urn: Urn, + properties: Option, +} + +#[derive(Debug, Deserialize)] +struct DomainProperties { + name: String, + description: Option, +} + +/// The result of the `DataProductContains` relationship query. Each relationship's related entity is +/// a data product the resource belongs to. +#[derive(Debug, Deserialize)] +struct EntityRelationships { + relationships: Vec, +} + +#[derive(Debug, Deserialize)] +struct EntityRelationship { + entity: DataProductNode, +} + +#[derive(Debug, Deserialize)] +struct DataProductNode { + urn: Urn, + properties: Option, +} + +#[derive(Debug, Deserialize)] +struct DataProductProperties { + name: String, + description: Option, +} + #[derive(Debug, Deserialize)] struct Ownership { owners: Vec, @@ -189,6 +249,39 @@ impl Entity { }) .collect(); + let domain = self.domain.map(|association| { + let DomainNode { urn, properties } = association.domain; + // A domain without a properties aspect has no name; fall back to its URN. + let (name, description) = match properties { + Some(properties) => (properties.name, properties.description), + None => (urn.0.clone(), None), + }; + Domain { + urn, + name, + description, + } + }); + + let data_products = self + .data_products + .into_iter() + .flat_map(|relationships| relationships.relationships) + .map(|relationship| { + let DataProductNode { urn, properties } = relationship.entity; + // A data product without a properties aspect has no name; fall back to its URN. + let (name, description) = match properties { + Some(properties) => (properties.name, properties.description), + None => (urn.0.clone(), None), + }; + DataProduct { + urn, + name, + description, + } + }) + .collect(); + let mut owners: BTreeMap = BTreeMap::new(); for owner in self .ownership @@ -210,7 +303,13 @@ impl Entity { } } - DataHubResourceInfoResponse { urn, tags, owners } + DataHubResourceInfoResponse { + urn, + tags, + domain, + data_products, + owners, + } } } diff --git a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs index d40fcb65..2208766b 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs @@ -91,6 +91,15 @@ pub struct DataHubResourceInfoResponse { urn: Urn, tags: Vec, + /// The domain the resource belongs to, e.g. `urn:li:domain:marketing`. A resource is assigned to + /// at most one domain, so this is an [`Option`] rather than a list. [`None`] if the resource is + /// not assigned to any domain. + domain: Option, + + /// The data products the resource is part of, e.g. `urn:li:dataProduct:orders`. Modelled as a + /// list because DataHub allows an asset to belong to more than one data product. + data_products: Vec, + /// Owners grouped by their ownership type URN, e.g. /// `urn:li:ownershipType:__system__technical_owner`. DataHub has no fixed set of ownership /// types — users can define custom ones — so this is an open map, not an enum. @@ -114,6 +123,22 @@ pub struct Tag { name: String, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Domain { + urn: Urn, + name: String, + description: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DataProduct { + urn: Urn, + name: String, + description: Option, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct User { From a8310f6ede211c17f3f6a27ab168c17edba9cc23 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 21 Jul 2026 09:12:13 +0200 Subject: [PATCH 09/13] RawDataHubUrn -> DataHubUrn --- rust/resource-info-fetcher/src/api.rs | 2 +- .../src/backend/data_hub/resource_to_urn_mapping.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/resource-info-fetcher/src/api.rs b/rust/resource-info-fetcher/src/api.rs index da8702b9..dccfac86 100644 --- a/rust/resource-info-fetcher/src/api.rs +++ b/rust/resource-info-fetcher/src/api.rs @@ -51,7 +51,7 @@ pub enum ResourceInfoRequestResource { topic: String, }, - RawDataHubUrn(String), + DataHubUrn(String), } #[derive(Snafu, Debug)] diff --git a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs index 3ca4eadc..4730c6c9 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/resource_to_urn_mapping.rs @@ -42,7 +42,7 @@ pub fn urn_for_request(request: &ResourceInfoRequest, env: &str) -> Urn { ResourceInfoRequestResource::KafkaTopic { topic } => { format!("urn:li:dataset:(urn:li:dataPlatform:{stacklet},{topic},{env})") } - ResourceInfoRequestResource::RawDataHubUrn(urn) => urn.to_owned(), + ResourceInfoRequestResource::DataHubUrn(urn) => urn.to_owned(), }; Urn(urn) From a25da2b57742b24e8368fb8f11aa583ac55b9c4a Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 21 Jul 2026 10:29:00 +0200 Subject: [PATCH 10/13] refactor: Move user-info-fetcher sidecar creation into separate module --- .../{daemonset.rs => daemonset/mod.rs} | 314 +++++++++--------- .../resource/daemonset/user_info_fetcher.rs | 205 ++++++++++++ 2 files changed, 360 insertions(+), 159 deletions(-) rename rust/operator-binary/src/controller/build/resource/{daemonset.rs => daemonset/mod.rs} (82%) create mode 100644 rust/operator-binary/src/controller/build/resource/daemonset/user_info_fetcher.rs diff --git a/rust/operator-binary/src/controller/build/resource/daemonset.rs b/rust/operator-binary/src/controller/build/resource/daemonset/mod.rs similarity index 82% rename from rust/operator-binary/src/controller/build/resource/daemonset.rs rename to rust/operator-binary/src/controller/build/resource/daemonset/mod.rs index f4324e7a..e291c812 100644 --- a/rust/operator-binary/src/controller/build/resource/daemonset.rs +++ b/rust/operator-binary/src/controller/build/resource/daemonset/mod.rs @@ -17,20 +17,14 @@ use stackable_operator::{ volume::{SecretOperatorVolumeSourceBuilder, VolumeBuilder}, }, }, - commons::{ - secret_class::{ - SecretClassVolume, SecretClassVolumeProvisionParts, SecretClassVolumeScope, - }, - tls_verification::{TlsClientDetails, TlsClientDetailsError}, - }, - crd::authentication::ldap, + commons::secret_class::SecretClassVolumeProvisionParts, k8s_openapi::{ DeepMerge, api::{ apps::v1::{DaemonSet, DaemonSetSpec, DaemonSetUpdateStrategy, RollingUpdateDaemonSet}, core::v1::{ EmptyDirVolumeSource, EnvVarSource, HTTPGetAction, ObjectFieldSelector, Probe, - ResourceRequirements, SecretVolumeSource, ServiceAccount, + ResourceRequirements, ServiceAccount, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -54,11 +48,16 @@ use stackable_operator::{ use super::service::{self, APP_PORT, APP_PORT_NAME}; use crate::{ - controller::{OpaRoleGroupConfig, RoleGroupName, ValidatedCluster, ValidatedOpaConfig, build}, - crd::{Container, DEFAULT_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, user_info_fetcher}, + controller::{ + OpaRoleGroupConfig, RoleGroupName, ValidatedCluster, ValidatedOpaConfig, + build::{self, resource::daemonset::user_info_fetcher::add_user_info_fetcher_sidecar}, + }, + crd::{Container, DEFAULT_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT}, operations::graceful_shutdown::add_graceful_shutdown_config, }; +mod user_info_fetcher; + pub const BUNDLES_ACTIVE_DIR: &str = "/bundles/active"; pub const BUNDLES_INCOMING_DIR: &str = "/bundles/incoming"; pub const BUNDLES_TMP_DIR: &str = "/bundles/tmp"; @@ -132,31 +131,6 @@ pub enum Error { source: crate::operations::graceful_shutdown::Error, }, - #[snafu(display("failed to build volume spec for the User Info Fetcher TLS config"))] - UserInfoFetcherKerberosVolume { - source: stackable_operator::builder::pod::Error, - }, - - #[snafu(display("failed to build volume mount spec for the User Info Fetcher TLS config"))] - UserInfoFetcherKerberosVolumeMount { - source: stackable_operator::builder::pod::container::Error, - }, - - #[snafu(display("failed to convert the User Info Fetcher Kerberos SecretClass into a volume"))] - ConvertUserInfoFetcherKerberosSecretClassVolume { - source: stackable_operator::commons::secret_class::SecretClassVolumeError, - }, - - #[snafu(display( - "failed to build volume or volume mount spec for the User Info Fetcher TLS config" - ))] - UserInfoFetcherTlsVolumeAndMounts { source: TlsClientDetailsError }, - - #[snafu(display( - "failed to build volume or volume mount spec for the User Info Fetcher LDAP config" - ))] - UserInfoFetcherLdapVolumeAndMounts { source: ldap::v1alpha1::Error }, - #[snafu(display("failed to add needed volume"))] AddVolume { source: builder::pod::Error }, @@ -169,6 +143,9 @@ pub enum Error { TlsVolumeBuild { source: builder::pod::volume::SecretOperatorVolumeSourceBuilderError, }, + + #[snafu(display("failed to build User Info Fetcher sidecar"))] + BuildUserInfoFetcherSidecar { source: user_info_fetcher::Error }, } type Result = std::result::Result; @@ -438,129 +415,14 @@ pub fn build_server_rolegroup_daemonset( .context(AddVolumeSnafu)?; } - if let Some(user_info) = &cluster.cluster_config.user_info { - let user_info_fetcher_container_name = container_name(&Container::UserInfoFetcher); - let mut cb_user_info_fetcher = new_container_builder(&user_info_fetcher_container_name); - - cb_user_info_fetcher - .image_from_product_image(resolved_product_image) // inherit the pull policy and pull secrets, and then... - .image(user_info_fetcher_image) // ...override the image - .command(vec!["stackable-opa-user-info-fetcher".to_string()]) - .add_env_var( - "CONFIG", - format!( - "{CONFIG_DIR}/{file}", - file = build::properties::ConfigFileName::UserInfoFetcher - ), - ) - .add_env_var("CREDENTIALS_DIR", USER_INFO_FETCHER_CREDENTIALS_DIR) - .add_volume_mount(CONFIG_VOLUME_NAME.as_ref(), CONFIG_DIR) - .context(AddVolumeMountSnafu)? - .resources(sidecar_resource_requirements()); - add_stackable_rust_cli_env_vars( - &mut cb_user_info_fetcher, - cluster_info, - sidecar_container_log_level(merged_config, &Container::UserInfoFetcher).to_string(), - &Container::UserInfoFetcher, - ); - - match &user_info.backend { - user_info_fetcher::v1alpha2::Backend::None {} => {} - user_info_fetcher::v1alpha2::Backend::ExperimentalXfscAas(_) => {} - user_info_fetcher::v1alpha2::Backend::ActiveDirectory(ad) => { - pb.add_volume( - SecretClassVolume::new( - ad.kerberos_secret_class_name.to_string(), - Some(SecretClassVolumeScope { - pod: false, - node: false, - services: vec![cluster.name.to_string()], - listener_volumes: Vec::new(), - }), - ) - .to_volume( - USER_INFO_FETCHER_KERBEROS_VOLUME_NAME.as_ref(), - // The user-info-fetcher needs both the keytab (private) and the Kerberos config (public). - SecretClassVolumeProvisionParts::PublicPrivate, - ) - .context(ConvertUserInfoFetcherKerberosSecretClassVolumeSnafu)?, - ) - .context(UserInfoFetcherKerberosVolumeSnafu)?; - cb_user_info_fetcher - .add_volume_mount( - USER_INFO_FETCHER_KERBEROS_VOLUME_NAME.as_ref(), - USER_INFO_FETCHER_KERBEROS_DIR, - ) - .context(UserInfoFetcherKerberosVolumeMountSnafu)?; - cb_user_info_fetcher.add_env_var( - "KRB5_CONFIG", - format!("{USER_INFO_FETCHER_KERBEROS_DIR}/krb5.conf"), - ); - cb_user_info_fetcher.add_env_var( - "KRB5_CLIENT_KTNAME", - format!("{USER_INFO_FETCHER_KERBEROS_DIR}/keytab"), - ); - cb_user_info_fetcher.add_env_var("KRB5CCNAME", "MEMORY:".to_string()); - ad.tls - .add_volumes_and_mounts(&mut pb, vec![&mut cb_user_info_fetcher]) - .context(UserInfoFetcherTlsVolumeAndMountsSnafu)?; - } - user_info_fetcher::v1alpha2::Backend::Keycloak(keycloak) => { - pb.add_volume( - VolumeBuilder::new(USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref()) - .secret(SecretVolumeSource { - secret_name: Some(keycloak.client_credentials_secret.to_string()), - ..Default::default() - }) - .build(), - ) - .context(AddVolumeSnafu)?; - cb_user_info_fetcher - .add_volume_mount( - USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref(), - USER_INFO_FETCHER_CREDENTIALS_DIR, - ) - .context(AddVolumeMountSnafu)?; - keycloak - .tls - .add_volumes_and_mounts(&mut pb, vec![&mut cb_user_info_fetcher]) - .context(UserInfoFetcherTlsVolumeAndMountsSnafu)?; - } - user_info_fetcher::v1alpha2::Backend::Entra(entra) => { - pb.add_volume( - VolumeBuilder::new(USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref()) - .secret(SecretVolumeSource { - secret_name: Some(entra.client_credentials_secret.to_string()), - ..Default::default() - }) - .build(), - ) - .context(AddVolumeSnafu)?; - cb_user_info_fetcher - .add_volume_mount( - USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref(), - USER_INFO_FETCHER_CREDENTIALS_DIR, - ) - .context(AddVolumeMountSnafu)?; - - TlsClientDetails { - tls: entra.tls.clone(), - } - .add_volumes_and_mounts(&mut pb, vec![&mut cb_user_info_fetcher]) - .context(UserInfoFetcherTlsVolumeAndMountsSnafu)?; - } - user_info_fetcher::v1alpha2::Backend::OpenLdap(openldap) => { - // Reuse the logic from the LDAP `AuthenticationProvider` which handles - // volume mounting of TLS secrets and LDAP bind credentials - openldap - .to_ldap_provider() - .add_volumes_and_mounts(&mut pb, vec![&mut cb_user_info_fetcher]) - .context(UserInfoFetcherLdapVolumeAndMountsSnafu)?; - } - } - - pb.add_container(cb_user_info_fetcher.build()); - } + add_user_info_fetcher_sidecar( + &mut pb, + cluster, + merged_config, + user_info_fetcher_image, + cluster_info, + ) + .context(BuildUserInfoFetcherSidecarSnafu)?; // The Vector logging config was validated up-front (see `ValidatedLogging`); a `Some` here means // the Vector agent is enabled and the aggregator discovery ConfigMap name is valid. @@ -850,7 +712,8 @@ mod tests { use stackable_operator::{ commons::networking::DomainName, k8s_openapi::{ - api::core::v1::ServiceAccount, apimachinery::pkg::apis::meta::v1::ObjectMeta, + api::core::v1::{Container, ServiceAccount}, + apimachinery::pkg::apis::meta::v1::ObjectMeta, }, }; @@ -1128,4 +991,137 @@ mod tests { .contains("stackable-opa-bundle-builder &") ); } + + fn uif_container(ds: &DaemonSet) -> Container { + ds.spec + .as_ref() + .unwrap() + .template + .spec + .as_ref() + .unwrap() + .containers + .iter() + .find(|c| c.name == "user-info-fetcher") + .expect("the user-info-fetcher container should exist") + .clone() + } + + fn env_var(container: &Container, name: &str) -> String { + container + .env + .as_ref() + .expect("the container should have env vars") + .iter() + .find(|e| e.name == name) + .unwrap_or_else(|| panic!("env var {name} should be set")) + .value + .clone() + .unwrap_or_else(|| panic!("env var {name} should have a literal value")) + } + + fn mount_path(container: &Container, volume_name: &str) -> String { + container + .volume_mounts + .as_ref() + .expect("the container should have volume mounts") + .iter() + .find(|m| m.name == volume_name) + .unwrap_or_else(|| panic!("volume mount {volume_name} should exist")) + .mount_path + .clone() + } + + #[test] + fn user_info_fetcher_container_has_expected_command_and_config_wiring() { + let ds = build(&validated_cluster_from_spec(json!({ + "image": { "productVersion": "1.2.3" }, + "clusterConfig": { + "userInfo": { + "backend": { + "experimentalXfscAas": { + "hostname": "aas.default.svc.cluster.local", + "port": 5000, + } + } + } + }, + "servers": { "roleGroups": { "default": {} } }, + }))); + + let uif = uif_container(&ds); + assert_eq!( + uif.command, + Some(vec!["stackable-opa-user-info-fetcher".to_owned()]) + ); + // The sidecar reads its config from the shared config volume, and looks for backend + // credentials in a fixed directory (populated by the backend-specific arms below). + assert_eq!( + env_var(&uif, "CONFIG"), + "/stackable/config/user-info-fetcher.json" + ); + assert_eq!(env_var(&uif, "CREDENTIALS_DIR"), "/stackable/credentials"); + assert_eq!(mount_path(&uif, "config"), "/stackable/config"); + } + + #[test] + fn user_info_fetcher_active_directory_backend_mounts_kerberos_and_sets_krb5_env() { + let ds = build(&validated_cluster_from_spec(json!({ + "image": { "productVersion": "1.2.3" }, + "clusterConfig": { + "userInfo": { + "backend": { + "experimentalActiveDirectory": { + "ldapServer": "ad.example.com", + "baseDistinguishedName": "dc=example,dc=com", + "kerberosSecretClassName": "kerberos", + } + } + } + }, + "servers": { "roleGroups": { "default": {} } }, + }))); + + // A Kerberos secret volume is provisioned and mounted for the sidecar. + assert!(volume_names(&ds).contains(&"kerberos".to_owned())); + let uif = uif_container(&ds); + assert_eq!(mount_path(&uif, "kerberos"), "/stackable/kerberos"); + // The krb5 client must find the config and keytab, and keep tickets in memory only. + assert_eq!( + env_var(&uif, "KRB5_CONFIG"), + "/stackable/kerberos/krb5.conf" + ); + assert_eq!( + env_var(&uif, "KRB5_CLIENT_KTNAME"), + "/stackable/kerberos/keytab" + ); + assert_eq!(env_var(&uif, "KRB5CCNAME"), "MEMORY:"); + } + + #[test] + fn user_info_fetcher_keycloak_backend_mounts_client_credentials_secret() { + let ds = build(&validated_cluster_from_spec(json!({ + "image": { "productVersion": "1.2.3" }, + "clusterConfig": { + "userInfo": { + "backend": { + "keycloak": { + "hostname": "keycloak.example.com", + "clientCredentialsSecret": "keycloak-credentials", + "adminRealm": "master", + "userRealm": "my-realm", + } + } + } + }, + "servers": { "roleGroups": { "default": {} } }, + }))); + + // The client credentials secret is projected into the sidecar's credentials dir. + assert!(volume_names(&ds).contains(&"credentials".to_owned())); + assert_eq!( + mount_path(&uif_container(&ds), "credentials"), + "/stackable/credentials" + ); + } } diff --git a/rust/operator-binary/src/controller/build/resource/daemonset/user_info_fetcher.rs b/rust/operator-binary/src/controller/build/resource/daemonset/user_info_fetcher.rs new file mode 100644 index 00000000..8f754e58 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/daemonset/user_info_fetcher.rs @@ -0,0 +1,205 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + self, + pod::{PodBuilder, volume::VolumeBuilder}, + }, + commons::{ + secret_class::{ + SecretClassVolume, SecretClassVolumeProvisionParts, SecretClassVolumeScope, + }, + tls_verification::{TlsClientDetails, TlsClientDetailsError}, + }, + crd::authentication::ldap, + k8s_openapi::api::core::v1::SecretVolumeSource, + utils::cluster_info::KubernetesClusterInfo, + v2::builder::pod::container::new_container_builder, +}; + +use crate::{ + controller::{ + ValidatedCluster, ValidatedOpaConfig, + build::{ + self, + resource::daemonset::{ + CONFIG_DIR, CONFIG_VOLUME_NAME, USER_INFO_FETCHER_CREDENTIALS_DIR, + USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME, USER_INFO_FETCHER_KERBEROS_DIR, + USER_INFO_FETCHER_KERBEROS_VOLUME_NAME, add_stackable_rust_cli_env_vars, + container_name, sidecar_container_log_level, sidecar_resource_requirements, + }, + }, + }, + crd::{Container, user_info_fetcher}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build volume spec for the User Info Fetcher TLS config"))] + KerberosVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to build volume mount spec for the User Info Fetcher TLS config"))] + KerberosVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("failed to convert the User Info Fetcher Kerberos SecretClass into a volume"))] + ConvertKerberosSecretClassVolume { + source: stackable_operator::commons::secret_class::SecretClassVolumeError, + }, + + #[snafu(display( + "failed to build volume or volume mount spec for the User Info Fetcher TLS config" + ))] + TlsVolumeAndMounts { source: TlsClientDetailsError }, + + #[snafu(display( + "failed to build volume or volume mount spec for the User Info Fetcher LDAP config" + ))] + LdapVolumeAndMounts { source: ldap::v1alpha1::Error }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { source: builder::pod::Error }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: builder::pod::container::Error, + }, +} + +type Result = std::result::Result; + +pub fn add_user_info_fetcher_sidecar( + pb: &mut PodBuilder, + cluster: &ValidatedCluster, + merged_config: &ValidatedOpaConfig, + user_info_fetcher_image: &str, + cluster_info: &KubernetesClusterInfo, +) -> Result<()> { + if let Some(user_info) = &cluster.cluster_config.user_info { + let user_info_fetcher_container_name = container_name(&Container::UserInfoFetcher); + let mut cb_user_info_fetcher = new_container_builder(&user_info_fetcher_container_name); + + cb_user_info_fetcher + .image_from_product_image(&cluster.image) // inherit the pull policy and pull secrets, and then... + .image(user_info_fetcher_image) // ...override the image + .command(vec!["stackable-opa-user-info-fetcher".to_string()]) + .add_env_var( + "CONFIG", + format!( + "{CONFIG_DIR}/{file}", + file = build::properties::ConfigFileName::UserInfoFetcher + ), + ) + .add_env_var("CREDENTIALS_DIR", USER_INFO_FETCHER_CREDENTIALS_DIR) + .add_volume_mount(CONFIG_VOLUME_NAME.as_ref(), CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .resources(sidecar_resource_requirements()); + add_stackable_rust_cli_env_vars( + &mut cb_user_info_fetcher, + cluster_info, + sidecar_container_log_level(merged_config, &Container::UserInfoFetcher).to_string(), + &Container::UserInfoFetcher, + ); + + match &user_info.backend { + user_info_fetcher::v1alpha2::Backend::None {} => {} + user_info_fetcher::v1alpha2::Backend::ExperimentalXfscAas(_) => {} + user_info_fetcher::v1alpha2::Backend::ActiveDirectory(ad) => { + pb.add_volume( + SecretClassVolume::new( + ad.kerberos_secret_class_name.to_string(), + Some(SecretClassVolumeScope { + pod: false, + node: false, + services: vec![cluster.name.to_string()], + listener_volumes: Vec::new(), + }), + ) + .to_volume( + USER_INFO_FETCHER_KERBEROS_VOLUME_NAME.as_ref(), + // The user-info-fetcher needs both the keytab (private) and the Kerberos config (public). + SecretClassVolumeProvisionParts::PublicPrivate, + ) + .context(ConvertKerberosSecretClassVolumeSnafu)?, + ) + .context(KerberosVolumeSnafu)?; + cb_user_info_fetcher + .add_volume_mount( + USER_INFO_FETCHER_KERBEROS_VOLUME_NAME.as_ref(), + USER_INFO_FETCHER_KERBEROS_DIR, + ) + .context(KerberosVolumeMountSnafu)?; + cb_user_info_fetcher.add_env_var( + "KRB5_CONFIG", + format!("{USER_INFO_FETCHER_KERBEROS_DIR}/krb5.conf"), + ); + cb_user_info_fetcher.add_env_var( + "KRB5_CLIENT_KTNAME", + format!("{USER_INFO_FETCHER_KERBEROS_DIR}/keytab"), + ); + cb_user_info_fetcher.add_env_var("KRB5CCNAME", "MEMORY:".to_string()); + ad.tls + .add_volumes_and_mounts(pb, vec![&mut cb_user_info_fetcher]) + .context(TlsVolumeAndMountsSnafu)?; + } + user_info_fetcher::v1alpha2::Backend::Keycloak(keycloak) => { + pb.add_volume( + VolumeBuilder::new(USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref()) + .secret(SecretVolumeSource { + secret_name: Some(keycloak.client_credentials_secret.to_string()), + ..Default::default() + }) + .build(), + ) + .context(AddVolumeSnafu)?; + cb_user_info_fetcher + .add_volume_mount( + USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref(), + USER_INFO_FETCHER_CREDENTIALS_DIR, + ) + .context(AddVolumeMountSnafu)?; + keycloak + .tls + .add_volumes_and_mounts(pb, vec![&mut cb_user_info_fetcher]) + .context(TlsVolumeAndMountsSnafu)?; + } + user_info_fetcher::v1alpha2::Backend::Entra(entra) => { + pb.add_volume( + VolumeBuilder::new(USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref()) + .secret(SecretVolumeSource { + secret_name: Some(entra.client_credentials_secret.to_string()), + ..Default::default() + }) + .build(), + ) + .context(AddVolumeSnafu)?; + cb_user_info_fetcher + .add_volume_mount( + USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref(), + USER_INFO_FETCHER_CREDENTIALS_DIR, + ) + .context(AddVolumeMountSnafu)?; + + TlsClientDetails { + tls: entra.tls.clone(), + } + .add_volumes_and_mounts(pb, vec![&mut cb_user_info_fetcher]) + .context(TlsVolumeAndMountsSnafu)?; + } + user_info_fetcher::v1alpha2::Backend::OpenLdap(openldap) => { + // Reuse the logic from the LDAP `AuthenticationProvider` which handles + // volume mounting of TLS secrets and LDAP bind credentials + openldap + .to_ldap_provider() + .add_volumes_and_mounts(pb, vec![&mut cb_user_info_fetcher]) + .context(LdapVolumeAndMountsSnafu)?; + } + } + + pb.add_container(cb_user_info_fetcher.build()); + } + + Ok(()) +} From f5b5bf9f12714aea4bd75af9c73f6907559d4c90 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 21 Jul 2026 11:24:08 +0200 Subject: [PATCH 11/13] Re-add Tiltfile --- Tiltfile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Tiltfile diff --git a/Tiltfile b/Tiltfile new file mode 100644 index 00000000..1859de01 --- /dev/null +++ b/Tiltfile @@ -0,0 +1,54 @@ +# Load the metadata first, so that we immediately get access to the operator name +meta = read_json('nix/meta.json') +operator_name = meta['operator']['name'] + +# If tilt_options.json exists read it and load the default_registry, default_operator_repository, +# and default_product_repository value from it +settings = read_json('tilt_options.json', default={}) +registry = settings.get('default_registry', 'oci.stackable.tech') + +# Construct the operator image repository. It uses the "sandbox" instead of the "sdp" namespace to +# separate "testing/local" images from official (published) images. +operator_repository = settings.get('default_operator_repository', registry + '/' + 'sandbox') +operator_image_name = operator_repository + '/' + operator_name + +# For the product image, we want to use the images in the "sdp" namespace, because "sandbox" doesn't +# contain those images (by default). +product_repository = settings.get('default_product_repository', registry + '/' + 'sdp') +# Configure default registry either read from config file above, or with default value of +# "oci.stackable.tech" +default_registry(registry) + +custom_build( + operator_image_name, + 'make regenerate-nix && nix-build . -A docker --argstr dockerName "' + operator_image_name + '" && ./result/load-image | docker load', + deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], + ignore=['*.~undo-tree~'], + # ignore=['result*', 'Cargo.nix', 'target', *.yaml], + outputs_image_ref_to='result/ref', +) + +# We need to set the correct image annotation on the operator Deployment to use e.g. +# oci.stackable.tech/sandbox/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of +# oci.stackable.tech/sandbox/opa-operator:0.0.0-dev (which does not exist) +k8s_kind('Deployment', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') +k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') + +# Optionally specify a custom Helm values file to be passed to the Helm deployment below. +# This file can for example be used to set custom telemetry options (like log level) which is not +# supported by helm(set). +helm_values = settings.get('helm_values', None) + +helm_override_operator_image_repository = 'image.repository=' + operator_repository +helm_override_product_image_repository = 'image.productRepository=' + product_repository + +k8s_yaml(helm( + 'deploy/helm/' + operator_name, + name=operator_name, + namespace="stackable-operators", + set=[ + helm_override_operator_image_repository, + helm_override_product_image_repository, + ], + values=helm_values, +)) From 2cfd78d66b4331521a4c9279a488794eac93d351 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 22 Jul 2026 10:01:50 +0200 Subject: [PATCH 12/13] Implement RIF into operator --- Cargo.nix | 702 ++++++++++++------ crate-hashes.json | 18 +- .../usage-guide/resource-info-fetcher.adoc | 120 +++ .../pages/usage-guide/user-info-fetcher.adoc | 2 +- extra/crds.yaml | 532 +++++++++++++ .../src/controller/build/properties/mod.rs | 3 + .../build/properties/resource_info_fetcher.rs | 18 + .../controller/build/resource/config_map.rs | 15 +- .../build/resource/daemonset/mod.rs | 33 +- .../daemonset/resource_info_fetcher.rs | 108 +++ rust/operator-binary/src/controller/mod.rs | 3 +- .../src/controller/validate.rs | 1 + rust/operator-binary/src/crd/mod.rs | 9 + .../src/crd/resource_info_fetcher/mod.rs | 10 +- rust/operator-binary/src/main.rs | 3 +- rust/operator-binary/src/opa_controller.rs | 2 + rust/regorule-library/src/lib.rs | 14 +- .../regorule-library/src/resourceinfo/v1.rego | 44 ++ .../src/backend/data_hub/mod.rs | 49 +- rust/resource-info-fetcher/src/main.rs | 4 +- .../00-patch-ns.yaml.j2 | 9 + .../data-hub-resource-info/01-assert.yaml.j2 | 10 + ...tor-aggregator-discovery-configmap.yaml.j2 | 9 + .../data-hub-resource-info/02-assert.yaml | 12 + .../02-install-datahub-prerequisites.yaml.j2 | 31 + .../02_datahub-prerequisites-values.yaml | 64 ++ .../data-hub-resource-info/03-assert.yaml | 9 + .../03-install-datahub.yaml.j2 | 12 + .../03_datahub-values.yaml | 54 ++ .../data-hub-resource-info/04-assert.yaml | 13 + .../04-load-sample-data.yaml | 98 +++ .../data-hub-resource-info/10-assert.yaml | 8 + .../10-install-opa.yaml.j2 | 65 ++ .../data-hub-resource-info/20-assert.yaml | 15 + .../data-hub-resource-info/20-run-test.yaml | 42 ++ .../data-hub-resource-info/test-regorule.py | 99 +++ tests/test-definition.yaml | 12 + 37 files changed, 1982 insertions(+), 270 deletions(-) create mode 100644 docs/modules/opa/pages/usage-guide/resource-info-fetcher.adoc create mode 100644 rust/operator-binary/src/controller/build/properties/resource_info_fetcher.rs create mode 100644 rust/operator-binary/src/controller/build/resource/daemonset/resource_info_fetcher.rs create mode 100644 rust/regorule-library/src/resourceinfo/v1.rego create mode 100644 tests/templates/kuttl/data-hub-resource-info/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/data-hub-resource-info/01-assert.yaml.j2 create mode 100644 tests/templates/kuttl/data-hub-resource-info/01-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/data-hub-resource-info/02-assert.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/02-install-datahub-prerequisites.yaml.j2 create mode 100644 tests/templates/kuttl/data-hub-resource-info/02_datahub-prerequisites-values.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/03-assert.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/03-install-datahub.yaml.j2 create mode 100644 tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/04-assert.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/10-assert.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/10-install-opa.yaml.j2 create mode 100644 tests/templates/kuttl/data-hub-resource-info/20-assert.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/20-run-test.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/test-regorule.py diff --git a/Cargo.nix b/Cargo.nix index fa46693a..cf78941a 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -43,6 +43,16 @@ rec { # You can override the features with # workspaceMembers."${crateName}".build.override { features = [ "default" "feature1" ... ]; }. workspaceMembers = { + "info-fetcher-commons" = rec { + packageId = "info-fetcher-commons"; + build = internal.buildRustCrateWithFeatures { + packageId = "info-fetcher-commons"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "stackable-opa-bundle-builder" = rec { packageId = "stackable-opa-bundle-builder"; build = internal.buildRustCrateWithFeatures { @@ -73,6 +83,16 @@ rec { # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; }; + "stackable-opa-resource-info-fetcher" = rec { + packageId = "stackable-opa-resource-info-fetcher"; + build = internal.buildRustCrateWithFeatures { + packageId = "stackable-opa-resource-info-fetcher"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "stackable-opa-user-info-fetcher" = rec { packageId = "stackable-opa-user-info-fetcher"; build = internal.buildRustCrateWithFeatures { @@ -380,9 +400,9 @@ rec { }; "anyhow" = rec { crateName = "anyhow"; - version = "1.0.103"; + version = "1.0.104"; edition = "2021"; - sha256 = "1wsav2g6vxcvf2c0fv3jhxfr55l0p2g8nygy7rmmvcsfwgi8ahra"; + sha256 = "0w34jjcm02p5g9kvsjr1dvpw0zs2fi7igi6nr414fkm5gz85w2ik"; authors = [ "David Tolnay " ]; @@ -559,7 +579,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "visit-mut" ]; } ]; @@ -567,9 +587,9 @@ rec { }; "async-trait" = rec { crateName = "async-trait"; - version = "0.1.89"; + version = "0.1.91"; edition = "2021"; - sha256 = "1fsxxmz3rzx1prn1h3rs7kyjhkap60i7xvi0ldapkvbb14nssdch"; + sha256 = "1v3cm8mzg66037wm392p1vsdx0lq8bid6y2ivr7z03lpfx0xqdmf"; procMacro = true; libName = "async_trait"; authors = [ @@ -586,7 +606,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 3.0.2"; usesDefaultFeatures = false; features = [ "clone-impls" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -619,10 +639,10 @@ rec { }; "aws-lc-rs" = rec { crateName = "aws-lc-rs"; - version = "1.17.1"; + version = "1.17.3"; edition = "2021"; - links = "aws_lc_rs_1_17_1_sys"; - sha256 = "1b9j3rrl3bxi27y9qwfi5hm8h337q1hj4ab03jdxvrf7gy9xhhj3"; + links = "aws_lc_rs_1_17_3_sys"; + sha256 = "1wbj1n78iqsf38xd2q93isjkb1iyaf7akm3wrji8ri6s33dbbg80"; libName = "aws_lc_rs"; authors = [ "AWS-LibCrypto" @@ -655,10 +675,10 @@ rec { }; "aws-lc-sys" = rec { crateName = "aws-lc-sys"; - version = "0.42.0"; + version = "0.43.0"; edition = "2021"; - links = "aws_lc_0_42_0"; - sha256 = "0i0l6q3xf0p5v1raxd718ccxl06dviws9kpwyh97ll1im4fyp73d"; + links = "aws_lc_0_43_0"; + sha256 = "0k12q9axgpzhqj5q5ics2m302fnbr4pp4pipi9kn5zknril32423"; build = "builder/main.rs"; libName = "aws_lc_sys"; authors = [ @@ -1034,7 +1054,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "cexpr"; @@ -1086,7 +1106,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "extra-traits" "visit-mut" ]; } ]; @@ -1160,11 +1180,11 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "bitflags 2.13.0" = rec { + "bitflags 2.13.1" = rec { crateName = "bitflags"; - version = "2.13.0"; + version = "2.13.1"; edition = "2021"; - sha256 = "1y239gpvl061rfvav7jds8mjs42kmwi39is7yx5d1qw3hvp8nf5l"; + sha256 = "1nl76mpykmwmb8rq1l5vw1azdh1wvxdrnsk4sy3rdrzx01nvg25m"; authors = [ "The Rust Project Developers" ]; @@ -1256,9 +1276,9 @@ rec { }; "bytes" = rec { crateName = "bytes"; - version = "1.12.0"; + version = "1.12.1"; edition = "2021"; - sha256 = "14xmxm8imyvw675bsgyadmzm9k63js1sdqh7099p0hlj2p9zbqwa"; + sha256 = "017z19dpg4f942h051m7bpnzcgng042hhcpd7bmg7bjjqd42lrgw"; authors = [ "Carl Lerche " "Sean McArthur " @@ -1272,9 +1292,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.66"; + version = "1.3.0"; edition = "2018"; - sha256 = "15nr9bpbcinb9z7zvr1d70xyivv8969v490001qdjywrjg3wmmpm"; + sha256 = "1f27b93qhs65bjq04ljwgwxf8xq2qbba4j1k99cv9d9qav88i5f8"; authors = [ "Alex Crichton " ]; @@ -1452,10 +1472,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.6.1"; + version = "4.6.2"; edition = "2024"; crateBin = []; - sha256 = "0lcf88l7vlg796rrqr7wipbbmfa5sgsgx4211b7xmxxv8dz13nqx"; + sha256 = "04ah42lfd7imr2mqj7xh0xgk3k5imb63hpgnhwvnphzmljfry1fx"; dependencies = [ { name = "clap_builder"; @@ -1494,9 +1514,9 @@ rec { }; "clap_builder" = rec { crateName = "clap_builder"; - version = "4.6.0"; + version = "4.6.2"; edition = "2024"; - sha256 = "17q6np22yxhh5y5v53y4l31ps3hlaz45mvz2n2nicr7n3c056jki"; + sha256 = "12sl6fyj6w2djxj0lsc1lkj1h3wpx74fjhb37izvaf65vjpji5ph"; dependencies = [ { name = "anstream"; @@ -1552,7 +1572,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -1884,7 +1904,7 @@ rec { } { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "bytes"; @@ -2094,7 +2114,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "extra-traits" ]; } ]; @@ -2125,7 +2145,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; @@ -2235,7 +2255,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "extra-traits" ]; } ]; @@ -2254,7 +2274,7 @@ rec { dependencies = [ { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } ]; features = { @@ -2281,7 +2301,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "visit-mut" ]; } ]; @@ -2358,7 +2378,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -2452,7 +2472,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; buildDependencies = [ @@ -2549,7 +2569,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -2725,13 +2745,13 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -2945,7 +2965,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -3105,9 +3125,9 @@ rec { }; "fastrand" = rec { crateName = "fastrand"; - version = "2.4.1"; + version = "2.5.0"; edition = "2018"; - sha256 = "1mnqxxnxvd69ma9mczabpbbsgwlhd6l78yv3vd681453a9s247wz"; + sha256 = "08q2r30y62winysimnlpbvw9kiwn0rmdlidqlmzd6z90mv764z6s"; authors = [ "Stjepan Glavina " ]; @@ -3332,9 +3352,9 @@ rec { }; "futures" = rec { crateName = "futures"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "0b9q86r5ar18v5xjiyqn7sb8sa32xv98qqnfz779gl7ns7lpw54b"; + sha256 = "066j5aqz8an05xh4hn5ljdnjn80z3g335v4grx4gaifr57wg3358"; dependencies = [ { name = "futures-channel"; @@ -3394,9 +3414,9 @@ rec { }; "futures-channel" = rec { crateName = "futures-channel"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "07fcyzrmbmh7fh4ainilf1s7gnwvnk07phdq77jkb9fpa2ffifq7"; + sha256 = "1bn5hlhfkl1sgypmiachaqcgwmr6wmjal7dyhfyb1zkazvs90996"; libName = "futures_channel"; dependencies = [ { @@ -3422,9 +3442,9 @@ rec { }; "futures-core" = rec { crateName = "futures-core"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "07bbvwjbm5g2i330nyr1kcvjapkmdqzl4r6mqv75ivvjaa0m0d3y"; + sha256 = "1iqdbvcdlplfr2g43h7xrfkv2sg5p1a26x8acz1xgxl07i3hrm9c"; libName = "futures_core"; features = { "default" = [ "std" ]; @@ -3435,9 +3455,9 @@ rec { }; "futures-executor" = rec { crateName = "futures-executor"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "17aplz3ns74qn7a04qg7qlgsdx5iwwwkd4jvdfra6hl3h4w9rwms"; + sha256 = "0n3lpkmcfrsnh40i4armn040gnqbpd257hz5qs46zipjr6f8fm37"; libName = "futures_executor"; dependencies = [ { @@ -3465,9 +3485,9 @@ rec { }; "futures-io" = rec { crateName = "futures-io"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "063pf5m6vfmyxj74447x8kx9q8zj6m9daamj4hvf49yrg9fs7jyf"; + sha256 = "0yjx13qdm9b2p4w00ddw85k6yccnnmqrlrrz8yfmi5jg7jmfqxs5"; libName = "futures_io"; features = { "default" = [ "std" ]; @@ -3476,9 +3496,9 @@ rec { }; "futures-macro" = rec { crateName = "futures-macro"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "0ys4b1lk7s0bsj29pv42bxsaavalch35rprp64s964p40c1bfdg8"; + sha256 = "02xiyd5y1nk9b805aympj4wq2czgvxnhcml9w9xkc665d3g3qv9d"; procMacro = true; libName = "futures_macro"; dependencies = [ @@ -3492,7 +3512,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -3500,9 +3520,9 @@ rec { }; "futures-sink" = rec { crateName = "futures-sink"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "14q8ml7hn5a6gyy9ri236j28kh0svqmrk4gcg0wh26rkazhm95y3"; + sha256 = "01z38z344hpryw84b6r0rbwcb669d8pyvl2szg10aqwx96n1hi73"; libName = "futures_sink"; features = { "default" = [ "std" ]; @@ -3512,9 +3532,9 @@ rec { }; "futures-task" = rec { crateName = "futures-task"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "14s3vqf8llz3kjza33vn4ixg6kwxp61xrysn716h0cwwsnri2xq3"; + sha256 = "02f1y1yvjg1cv998zkgl1706pi9y4fyc9045l1hlmyqyhclfscdj"; libName = "futures_task"; features = { "default" = [ "std" ]; @@ -3524,9 +3544,9 @@ rec { }; "futures-util" = rec { crateName = "futures-util"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "1mn60lw5kh32hz9isinjlpw34zx708fk5q1x0m40n6g6jq9a971q"; + sha256 = "1anyg40j5www5l22r2jbn1birsafz4q1w9qmcjk4vqzwasi90ym7"; libName = "futures_util"; dependencies = [ { @@ -3596,7 +3616,10 @@ rec { "io-compat" = [ "io" "compat" "tokio-io" "libc" ]; "libc" = [ "dep:libc" ]; "memchr" = [ "dep:memchr" ]; - "portable-atomic" = [ "futures-core/portable-atomic" ]; + "portable-atomic" = [ "futures-core/portable-atomic" "portable_atomic_crate" ]; + "portable-atomic-alloc" = [ "portable-atomic-util/alloc" "portable-atomic" ]; + "portable-atomic-util" = [ "dep:portable-atomic-util" ]; + "portable_atomic_crate" = [ "dep:portable_atomic_crate" ]; "sink" = [ "futures-sink" ]; "slab" = [ "dep:slab" ]; "spin" = [ "dep:spin" ]; @@ -3856,7 +3879,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "libc"; @@ -4227,9 +4250,9 @@ rec { }; "http-body" = rec { crateName = "http-body"; - version = "1.0.1"; + version = "1.1.0"; edition = "2018"; - sha256 = "111ir5k2b9ihz5nr9cz7cwm7fnydca7dx4hc7vr16scfzghxrzhy"; + sha256 = "0b5wj0rdj8p03k20q8x0jy249amg2db919fnmh7zcrgf2clqyana"; libName = "http_body"; authors = [ "Carl Lerche " @@ -4250,9 +4273,9 @@ rec { }; "http-body-util" = rec { crateName = "http-body-util"; - version = "0.1.3"; + version = "0.1.4"; edition = "2018"; - sha256 = "0jm6jv4gxsnlsi1kzdyffjrj8cfr3zninnxpw73mvkxy4qzdj8dh"; + sha256 = "1wizkqx9a75x8v5lm7cawpammz8sfvd7cngnkp34wkcfl3b1zx79"; libName = "http_body_util"; authors = [ "Carl Lerche " @@ -5241,6 +5264,68 @@ rec { } ]; + }; + "info-fetcher-commons" = rec { + crateName = "info-fetcher-commons"; + version = "0.0.0-dev"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/info-fetcher-commons; }; + libName = "info_fetcher_commons"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "axum"; + packageId = "axum"; + } + { + name = "hyper"; + packageId = "hyper"; + } + { + name = "native-tls"; + packageId = "native-tls"; + } + { + name = "reqwest"; + packageId = "reqwest"; + usesDefaultFeatures = false; + features = [ "json" "form" "query" "native-tls" "charset" "http2" "system-proxy" ]; + } + { + name = "rustls-pki-types"; + packageId = "rustls-pki-types"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "snafu"; + packageId = "snafu 0.9.1"; + } + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "webhook" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + ]; + }; "ipnet" = rec { crateName = "ipnet"; @@ -5372,9 +5457,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.31"; + version = "0.2.34"; edition = "2021"; - sha256 = "0d6nvffabb1v34x2s2a3xjca4zny1dfdhqm3xy0wyl77rchn3znc"; + sha256 = "05kfvhd4rp0a44qipkzlag547j5s3zxs46ql1pibf3mq8yax1171"; authors = [ "Andrew Gallant " ]; @@ -5384,6 +5469,12 @@ rec { packageId = "defmt"; optional = true; } + { + name = "jiff-core"; + packageId = "jiff-core"; + rename = "jcore"; + usesDefaultFeatures = false; + } { name = "jiff-static"; packageId = "jiff-static"; @@ -5438,16 +5529,16 @@ rec { } ]; features = { - "alloc" = [ "serde_core?/alloc" "portable-atomic-util/alloc" "defmt?/alloc" ]; + "alloc" = [ "jcore/alloc" "serde_core?/alloc" "portable-atomic-util/alloc" "defmt?/alloc" ]; "default" = [ "std" "tz-system" "tz-fat" "tzdb-bundle-platform" "tzdb-zoneinfo" "tzdb-concatenated" "perf-inline" ]; - "defmt" = [ "dep:defmt" ]; + "defmt" = [ "dep:defmt" "jcore/defmt" ]; "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; - "logging" = [ "dep:log" ]; + "logging" = [ "dep:log" "jcore/logging" ]; "serde" = [ "dep:serde_core" ]; "static" = [ "static-tz" "jiff-static?/tzdb" ]; "static-tz" = [ "dep:jiff-static" ]; - "std" = [ "alloc" "log?/std" "serde_core?/std" ]; - "tz-fat" = [ "jiff-static?/tz-fat" ]; + "std" = [ "alloc" "jcore/std" "log?/std" "serde_core?/std" ]; + "tz-fat" = [ "jcore/tz-fat" "jiff-static?/tz-fat" ]; "tz-system" = [ "std" "dep:windows-link" ]; "tzdb-bundle-always" = [ "dep:jiff-tzdb" "alloc" ]; "tzdb-bundle-platform" = [ "dep:jiff-tzdb-platform" "alloc" ]; @@ -5456,17 +5547,47 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "perf-inline" "serde" "std" "tz-fat" "tz-system" "tzdb-bundle-platform" "tzdb-concatenated" "tzdb-zoneinfo" ]; }; + "jiff-core" = rec { + crateName = "jiff-core"; + version = "0.1.0"; + edition = "2021"; + sha256 = "02axx56pkh2w4bw5rp94qlvcpwzd3n2w2025fnikvrgg762aiv3z"; + libName = "jiff_core"; + authors = [ + "Andrew Gallant " + ]; + dependencies = [ + { + name = "defmt"; + packageId = "defmt"; + optional = true; + } + ]; + features = { + "alloc" = [ "defmt?/alloc" ]; + "default" = [ "std" "tz-fat" ]; + "defmt" = [ "dep:defmt" ]; + "logging" = [ "dep:log" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" "tz-fat" ]; + }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.31"; + version = "0.2.34"; edition = "2021"; - sha256 = "1lpdrmzd5yafwzniw0xi5gac4rz0vccsja1qrprjim32ysbyhrg1"; + sha256 = "1pp9w3jn2l3zkqn35j3mfgzl4xwh9ddb0z37vhagkym6nxva0g9j"; procMacro = true; libName = "jiff_static"; authors = [ "Andrew Gallant " ]; dependencies = [ + { + name = "jiff-core"; + packageId = "jiff-core"; + rename = "jcore"; + } { name = "proc-macro2"; packageId = "proc-macro2"; @@ -5477,7 +5598,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -5487,9 +5608,9 @@ rec { }; "jiff-tzdb" = rec { crateName = "jiff-tzdb"; - version = "0.1.7"; + version = "0.1.8"; edition = "2021"; - sha256 = "14353p1g4234ww11f6jqy51fkrp31dqijaj0hwsjng59y5yj8hk1"; + sha256 = "07hl9sgzfb9as1x0n5bjk1qxishzcriapy9xa481y8xd6acx6aql"; libName = "jiff_tzdb"; libPath = "lib.rs"; authors = [ @@ -5602,7 +5723,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } ]; devDependencies = [ @@ -5625,9 +5746,9 @@ rec { }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; - version = "1.0.4"; + version = "1.0.5"; edition = "2021"; - sha256 = "1ymdpqawc3nxrlr6izwpw22h5msa16wqjbkqldijhrxvqhh76fk3"; + sha256 = "0psx7dab3nax9hjjgvpk33x3n8ryjfnj7ykr3gw8pqdz844y08j2"; libName = "jsonpath_rust"; authors = [ "BorisZhguchev " @@ -5651,7 +5772,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } ]; @@ -5750,7 +5871,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; libName = "k8s_version"; authors = [ @@ -6074,7 +6195,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } { name = "tokio"; @@ -6240,7 +6361,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } ]; devDependencies = [ @@ -6298,7 +6419,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -6392,7 +6513,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } { name = "tokio"; @@ -6537,7 +6658,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } { name = "tokio"; @@ -6656,7 +6777,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "bytes"; @@ -6889,6 +7010,28 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "md5" = rec { + crateName = "md5"; + version = "0.8.1"; + edition = "2021"; + sha256 = "032pi1dmpk8aimpc8p5rxw07wx7w3j8q4ah3iwyxz9n66a3qvfvy"; + authors = [ + "Daniel McKenna " + "Ivan Ukhov " + "Kamal Ahmad " + "Konstantin Stepanov " + "Lukas Kalbertodt " + "Nathan Musoke " + "Scott Mabin " + "Tony Arcieri " + "Wim de With " + "Yosef Dinerstein " + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "memchr" = rec { crateName = "memchr"; version = "2.8.3"; @@ -6968,9 +7111,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.2.1"; + version = "1.2.2"; edition = "2021"; - sha256 = "1nkggmrlnjs93w8rja4lvjj4aml1xqahgimv1h0p7d373kvhmg82"; + sha256 = "09y4b7gc42ymgssshh8sz6gs3y5r8bbigqaw2c4snh6fy5qmrmih"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -7066,7 +7209,7 @@ rec { "quanta" = [ "dep:quanta" ]; "unstable-debug-counters" = [ "future" ]; }; - resolvedDefaultFeatures = [ "default" "future" ]; + resolvedDefaultFeatures = [ "future" ]; }; "native-tls" = rec { crateName = "native-tls"; @@ -7244,7 +7387,7 @@ rec { } { name = "rand"; - packageId = "rand 0.8.6"; + packageId = "rand 0.8.7"; optional = true; usesDefaultFeatures = false; } @@ -7263,7 +7406,7 @@ rec { devDependencies = [ { name = "rand"; - packageId = "rand 0.8.6"; + packageId = "rand 0.8.7"; features = [ "small_rng" ]; } ]; @@ -7430,7 +7573,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "cfg-if"; @@ -7481,7 +7624,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -7568,7 +7711,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; optional = true; usesDefaultFeatures = false; } @@ -7738,7 +7881,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; usesDefaultFeatures = false; } { @@ -7938,14 +8081,14 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; optional = true; usesDefaultFeatures = false; features = [ "std" "std_rng" "small_rng" "os_rng" "thread_rng" ]; } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; usesDefaultFeatures = false; } { @@ -8329,7 +8472,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -8389,7 +8532,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "parsing" "printing" "clone-impls" "proc-macro" "full" "visit-mut" ]; } @@ -8485,9 +8628,9 @@ rec { }; "portable-atomic" = rec { crateName = "portable-atomic"; - version = "1.13.1"; + version = "1.14.0"; edition = "2018"; - sha256 = "0j8vlar3n5acyigq8q6f4wjx3k3s5yz0rlpqrv76j73gi5qr8fn3"; + sha256 = "1hyfma9n2cs2ibazpfwrbv61zwg7cv86g0pr5yjkg07qgr4xa81x"; libName = "portable_atomic"; features = { "critical-section" = [ "dep:critical-section" ]; @@ -8594,7 +8737,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "full" ]; } @@ -8607,7 +8750,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "clone-impls" "extra-traits" "parsing" "printing" "visit-mut" ]; } @@ -8642,9 +8785,9 @@ rec { }; "proc-macro2" = rec { crateName = "proc-macro2"; - version = "1.0.106"; + version = "1.0.107"; edition = "2021"; - sha256 = "0d09nczyaj67x4ihqr5p7gxbkz38gxhk4asc0k8q23g9n85hzl4g"; + sha256 = "1nb6ly8kp65f724kj73ippc7lvydss24sm2vagk6qpklpg4pwplq"; libName = "proc_macro2"; authors = [ "David Tolnay " @@ -8777,7 +8920,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -8813,9 +8956,9 @@ rec { }; "quote" = rec { crateName = "quote"; - version = "1.0.46"; + version = "1.0.47"; edition = "2021"; - sha256 = "0s034glrlav8nzqy2yskqzv52ncy82k126sm2jk5j1vs1iylbg6z"; + sha256 = "00ch0yyzvv6s671ik0kcsbw8nigdaj2g3fr61kcahwx48aqlvgqz"; authors = [ "David Tolnay " ]; @@ -8855,11 +8998,11 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; - "rand 0.8.6" = rec { + "rand 0.8.7" = rec { crateName = "rand"; - version = "0.8.6"; + version = "0.8.7"; edition = "2018"; - sha256 = "12kd4rljn86m00rcaz4c1rcya4mb4gk5ig6i8xq00a8wjgxfr82w"; + sha256 = "06iaf16fr0z8zly7anmn8ky0p80xnx9yv0gdcm30fwn9vqmigxi2"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -8889,11 +9032,11 @@ rec { }; resolvedDefaultFeatures = [ "rand_chacha" "std_rng" ]; }; - "rand 0.9.4" = rec { + "rand 0.9.5" = rec { crateName = "rand"; - version = "0.9.4"; + version = "0.9.5"; edition = "2021"; - sha256 = "1sknbxgs6nfg0nxdd7689lwbyr2i4vaswchrv4b34z8vpc3azia4"; + sha256 = "0hbvllk8g28mqjld6hqmckk69w296qpzg95whm3didsyg46ivvxr"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -9046,7 +9189,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } ]; features = { @@ -9058,9 +9201,9 @@ rec { }; "ref-cast" = rec { crateName = "ref-cast"; - version = "1.0.25"; + version = "1.0.26"; edition = "2021"; - sha256 = "0zdzc34qjva9xxgs889z5iz787g81hznk12zbk4g2xkgwq530m7k"; + sha256 = "0vdra0766jcc2czzqwhql41kkfyajdnai1pbkjxbq8vr7mvqyvi1"; libName = "ref_cast"; authors = [ "David Tolnay " @@ -9075,9 +9218,9 @@ rec { }; "ref-cast-impl" = rec { crateName = "ref-cast-impl"; - version = "1.0.25"; + version = "1.0.26"; edition = "2021"; - sha256 = "1nkhn1fklmn342z5c4mzfzlxddv3x8yhxwwk02cj06djvh36065p"; + sha256 = "0g70ff9an5i97cw9kijgzqrqydz7smcfic2zyydddizfbxl874ic"; procMacro = true; libName = "ref_cast_impl"; authors = [ @@ -9094,16 +9237,16 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 3.0.2"; } ]; }; "regex" = rec { crateName = "regex"; - version = "1.12.4"; + version = "1.13.1"; edition = "2021"; - sha256 = "1fm6si2xpmhwqflabdqsakc0qkq718wx2ljl37nbj75fb5vjnagi"; + sha256 = "1391a0a4100ik8cp7l577p3ip3haqq03rd9c5vdr7vcfdixj687h"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -9159,9 +9302,9 @@ rec { }; "regex-automata" = rec { crateName = "regex-automata"; - version = "0.4.14"; + version = "0.4.16"; edition = "2021"; - sha256 = "13xf7hhn4qmgfh784llcp2kzrvljd13lb2b1ca0mwnf15w9d87bf"; + sha256 = "1b8ihxq99g3hr8mr37bvhib4bfn8rlmpmp0wjg2q1j50plvdpkwg"; libName = "regex_automata"; authors = [ "The Rust Project Developers" @@ -9751,7 +9894,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; usesDefaultFeatures = false; } { @@ -9848,9 +9991,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.41"; + version = "0.23.42"; edition = "2021"; - sha256 = "07vbs2935a7xjqqvy8w3ndzmmw8dg769d9zcgdg7k6sdccjv34kb"; + sha256 = "0f619dq1izpl40glcqgfjbqzpmwg8g5iffjx4429sh4v06mzqm1w"; dependencies = [ { name = "aws-lc-rs"; @@ -10167,13 +10310,13 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -10284,7 +10427,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "core-foundation"; @@ -10354,9 +10497,9 @@ rec { }; "serde" = rec { crateName = "serde"; - version = "1.0.228"; + version = "1.0.229"; edition = "2021"; - sha256 = "17mf4hhjxv5m90g42wmlbc61hdhlm6j9hwfkpcnd72rpgzm993ls"; + sha256 = "1fp04fq4a79bpm61xz1zy0pbz4kpc7d771zii1k3inmszq55jj21"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -10496,9 +10639,9 @@ rec { }; "serde_core" = rec { crateName = "serde_core"; - version = "1.0.228"; + version = "1.0.229"; edition = "2021"; - sha256 = "1bb7id2xwx8izq50098s5j2sqrrvk31jbbrjqygyan6ask3qbls1"; + sha256 = "0j1ajiha76h3nmd976il9li6975k121xa7jb39ws8n0yqp4s5p37"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -10523,9 +10666,9 @@ rec { }; "serde_derive" = rec { crateName = "serde_derive"; - version = "1.0.228"; + version = "1.0.229"; edition = "2021"; - sha256 = "0y8xm7fvmr2kjcd029g9fijpndh8csv5m20g4bd76w8qschg4h6m"; + sha256 = "0j4k63i7h1bikxwz2c89ig0hrwbnl9mz1czn85xx99x5cc9dg9g7"; procMacro = true; authors = [ "Erick Tryzelaar " @@ -10546,7 +10689,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 3.0.2"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -10578,7 +10721,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -10587,9 +10730,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.150"; + version = "1.0.151"; edition = "2021"; - sha256 = "1ffgfhy9kndjnrz8lmy95pr758p2zk8dxv6yi99x0vkkni24w0g8"; + sha256 = "051zww7lvpw147vvwss1ng6w587qyrkzg75fvj08q2dfrmgbahf8"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -10732,9 +10875,9 @@ rec { }; "sha1" = rec { crateName = "sha1"; - version = "0.10.6"; + version = "0.10.7"; edition = "2018"; - sha256 = "1fnnxlfg08xhkmwf2ahv634as30l1i3xhlhkvxflmasi5nd85gz3"; + sha256 = "1f632d529qzz95yrprr632w1fxqkrv6b6jksjc11vnzl049lay59"; authors = [ "RustCrypto Developers" ]; @@ -10761,10 +10904,8 @@ rec { } ]; features = { - "asm" = [ "sha1-asm" ]; "default" = [ "std" ]; "oid" = [ "digest/oid" ]; - "sha1-asm" = [ "dep:sha1-asm" ]; "std" = [ "digest/std" ]; }; resolvedDefaultFeatures = [ "default" "std" ]; @@ -10918,9 +11059,9 @@ rec { }; "simd-adler32" = rec { crateName = "simd-adler32"; - version = "0.3.9"; + version = "0.3.10"; edition = "2018"; - sha256 = "0532ysdwcvzyp2bwpk8qz0hijplcdwpssr5gy5r7qwqqy5z5qgbh"; + sha256 = "1sny4y2qa5mwyxx5x59ln2p02vsdh92004njlslnx98imjc9489s"; libName = "simd_adler32"; authors = [ "Marvin Countryman " @@ -11107,7 +11248,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -11143,7 +11284,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -11153,9 +11294,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.6.4"; + version = "0.6.5"; edition = "2021"; - sha256 = "0ldyp5rhba15spwxj1n94xh7sjks1398c3vwpwkxkd1087nwzlaj"; + sha256 = "1m7diygswpvlpvrxd6ap169nxgax014jr8220nqlr3bzyb3y5lf3"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -11255,7 +11396,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; libName = "stackable_certs"; authors = [ @@ -11291,7 +11432,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; } { name = "rand_core"; @@ -11531,6 +11672,103 @@ rec { "Stackable GmbH " ]; + }; + "stackable-opa-resource-info-fetcher" = rec { + crateName = "stackable-opa-resource-info-fetcher"; + version = "0.0.0-dev"; + edition = "2024"; + crateBin = [ + { + name = "stackable-opa-resource-info-fetcher"; + path = "src/main.rs"; + requiredFeatures = [ ]; + } + ]; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/resource-info-fetcher; }; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "axum"; + packageId = "axum"; + } + { + name = "clap"; + packageId = "clap"; + } + { + name = "futures"; + packageId = "futures"; + } + { + name = "hyper"; + packageId = "hyper"; + } + { + name = "info-fetcher-commons"; + packageId = "info-fetcher-commons"; + } + { + name = "md5"; + packageId = "md5"; + } + { + name = "moka"; + packageId = "moka"; + usesDefaultFeatures = false; + features = [ "future" ]; + } + { + name = "reqwest"; + packageId = "reqwest"; + usesDefaultFeatures = false; + features = [ "json" "form" "query" "native-tls" "charset" "http2" "system-proxy" ]; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "snafu"; + packageId = "snafu 0.9.1"; + } + { + name = "stackable-opa-operator"; + packageId = "stackable-opa-operator"; + } + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "webhook" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "url"; + packageId = "url"; + } + ]; + buildDependencies = [ + { + name = "built"; + packageId = "built"; + features = [ "chrono" "git2" ]; + } + ]; + }; "stackable-opa-user-info-fetcher" = rec { crateName = "stackable-opa-user-info-fetcher"; @@ -11572,6 +11810,10 @@ rec { name = "hyper"; packageId = "hyper"; } + { + name = "info-fetcher-commons"; + packageId = "info-fetcher-commons"; + } { name = "krb5"; packageId = "krb5"; @@ -11584,6 +11826,7 @@ rec { { name = "moka"; packageId = "moka"; + usesDefaultFeatures = false; features = [ "future" ]; } { @@ -11671,7 +11914,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; libName = "stackable_operator"; authors = [ @@ -11752,7 +11995,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; } { name = "regex"; @@ -11870,7 +12113,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11892,7 +12135,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; @@ -11905,7 +12148,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; libName = "stackable_shared"; authors = [ @@ -11986,7 +12229,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; libName = "stackable_telemetry"; authors = [ @@ -12096,7 +12339,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; libName = "stackable_versioned"; authors = [ @@ -12146,7 +12389,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -12201,7 +12444,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; @@ -12214,7 +12457,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz"; + sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; }; libName = "stackable_webhook"; authors = [ @@ -12273,7 +12516,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; } { name = "serde"; @@ -12396,7 +12639,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "parsing" ]; } ]; @@ -12460,11 +12703,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "full" "parsing" "printing" "proc-macro" "quote" ]; }; - "syn 2.0.118" = rec { + "syn 2.0.119" = rec { crateName = "syn"; - version = "2.0.118"; + version = "2.0.119"; edition = "2021"; - sha256 = "08hlbc32lqd5d67p26ck7chg0rkclsw9as6f96vfn4s2j1zyb6hv"; + sha256 = "15vjy620l91a3q4n4f4gzhnflmdr6pnm38v2m6cpk86i8av32a47"; authors = [ "David Tolnay " ]; @@ -12493,6 +12736,39 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "fold" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; + "syn 3.0.2" = rec { + crateName = "syn"; + version = "3.0.2"; + edition = "2021"; + sha256 = "18w7g5b9c585jw2rgvhygqdli8hq7w2jcds4h05lgz5plbbdc1x2"; + authors = [ + "David Tolnay " + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + usesDefaultFeatures = false; + } + { + name = "quote"; + packageId = "quote"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "unicode-ident"; + packageId = "unicode-ident"; + } + ]; + features = { + "default" = [ "derive" "parsing" "printing" "clone-impls" "proc-macro" ]; + "printing" = [ "dep:quote" ]; + "proc-macro" = [ "proc-macro2/proc-macro" "quote?/proc-macro" ]; + "test" = [ "syn-test-suite/all-features" ]; + }; + resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; + }; "sync_wrapper" = rec { crateName = "sync_wrapper"; version = "1.0.2"; @@ -12536,7 +12812,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "derive" "parsing" "printing" "clone-impls" "visit" "extra-traits" ]; } @@ -12559,7 +12835,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "core-foundation"; @@ -12698,18 +12974,18 @@ rec { ]; }; - "thiserror 2.0.18" = rec { + "thiserror 2.0.19" = rec { crateName = "thiserror"; - version = "2.0.18"; + version = "2.0.19"; edition = "2021"; - sha256 = "1i7vcmw9900bvsmay7mww04ahahab7wmr8s925xc083rpjybb222"; + sha256 = "1ngwxsjsa64v1n7vb90h2b0i3fqk1piwaf0z6fqdacqfhjc3b909"; authors = [ "David Tolnay " ]; dependencies = [ { name = "thiserror-impl"; - packageId = "thiserror-impl 2.0.18"; + packageId = "thiserror-impl 2.0.19"; } ]; features = { @@ -12738,16 +13014,16 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; }; - "thiserror-impl 2.0.18" = rec { + "thiserror-impl 2.0.19" = rec { crateName = "thiserror-impl"; - version = "2.0.18"; + version = "2.0.19"; edition = "2021"; - sha256 = "1mf1vrbbimj1g6dvhdgzjmn6q09yflz2b92zs1j9n3k7cxzyxi7b"; + sha256 = "1ka10pqy1g8zy5al9m8yadg30jp8hx0q80j8awmd8131yw6gxjs3"; procMacro = true; libName = "thiserror_impl"; authors = [ @@ -12764,16 +13040,16 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 3.0.2"; } ]; }; "thread_local" = rec { crateName = "thread_local"; - version = "1.1.9"; + version = "1.1.10"; edition = "2021"; - sha256 = "1191jvl8d63agnq06pcnarivf63qzgpws5xa33hgc92gjjj4c0pn"; + sha256 = "0w20g2pfdcp8pz3gds0bzksv6mxk802szca8qlr3701jdm69rn8s"; authors = [ "Amanieu d'Antras " ]; @@ -12788,9 +13064,9 @@ rec { }; "time" = rec { crateName = "time"; - version = "0.3.53"; + version = "0.3.54"; edition = "2024"; - sha256 = "0l4aans0kv47y53736cjs0pnvdz91iyywrkqbrxk6cmrvknsmpqq"; + sha256 = "0i12170vw516jprmbv385krw75nyn7kwfp48nqybgfpnkximw79y"; authors = [ "Jacob Pratt " "Time contributors" @@ -12871,9 +13147,9 @@ rec { }; "time-macros" = rec { crateName = "time-macros"; - version = "0.2.31"; + version = "0.2.32"; edition = "2024"; - sha256 = "0pq8y9bm1zr008dmjs62qdfwsigv2kwkga5sj0d4jvk625qvhcf4"; + sha256 = "11gdd3b81mj8i0h114qfjjzm8j2rz2mhr9byr0ksjbldli196s3y"; procMacro = true; libName = "time_macros"; authors = [ @@ -12976,7 +13252,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "parsing" ]; } ]; @@ -12988,9 +13264,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.52.3"; + version = "1.53.0"; edition = "2021"; - sha256 = "1zpzazypkg61sw91na1m85x5s4rsjym335fwwhwm1hcs70dz1iwg"; + sha256 = "1vpzc93iaiaqk90jh54vqji8fawiiksk4cwh8qyns1xy5pavr26r"; authors = [ "Tokio Contributors " ]; @@ -13109,9 +13385,9 @@ rec { }; "tokio-macros" = rec { crateName = "tokio-macros"; - version = "2.7.0"; + version = "2.7.1"; edition = "2021"; - sha256 = "15m4f37mdafs0gg36sh0rskm1i768lb7zmp8bw67kaxr3avnqniq"; + sha256 = "1fj2h3gysqzwqchyhcyyvslwdj7qjgyzlc20d6sajwqf949sya33"; procMacro = true; libName = "tokio_macros"; authors = [ @@ -13128,7 +13404,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -13624,7 +13900,7 @@ rec { } { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "bytes"; @@ -13749,7 +14025,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "bytes"; @@ -13924,7 +14200,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.18"; + packageId = "thiserror 2.0.19"; } { name = "time"; @@ -13966,7 +14242,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ]; } @@ -14459,9 +14735,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.23.4"; + version = "1.24.0"; edition = "2021"; - sha256 = "0lws65rrqncssdz1rk8g8ww7xg6k4d3l6avzkslzwni78llag05z"; + sha256 = "0faj5x0zgri8m3i8dv9qgyhiwqwdyhbl2g351cp3iin4ynk26fdz"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -14735,7 +15011,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "visit" "visit-mut" "full" "extra-traits" ]; } { @@ -16161,7 +16437,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "parsing" "proc-macro" "printing" "full" "clone-impls" ]; } @@ -16188,7 +16464,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "parsing" "proc-macro" "printing" "full" "clone-impls" ]; } @@ -17274,7 +17550,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "fold" ]; } { @@ -17286,9 +17562,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.53"; + version = "0.8.54"; edition = "2021"; - sha256 = "1qcdv45iz4499bafwcnflvzf7adfvbnvgfc5w8cx8mk12d9n0wkm"; + sha256 = "06cxymy8i9q9a93xdins9ayakx9b1nc2arb7qdfd03ssf05brjxp"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -17322,9 +17598,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.53"; + version = "0.8.54"; edition = "2021"; - sha256 = "0wgxvsnv44x5xdli270xh085458m76dkl0iqjpa3624hry9gs527"; + sha256 = "1xb292dhgb0d4fs05cdj2s0v3srmk7bajv94sdc7631dnnvigs72"; procMacro = true; libName = "zerocopy_derive"; authors = [ @@ -17342,14 +17618,14 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "visit" ]; } ]; @@ -17398,7 +17674,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "fold" ]; } { @@ -17452,7 +17728,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "extra-traits" "visit" ]; } ]; @@ -17565,7 +17841,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -17573,9 +17849,9 @@ rec { }; "zmij" = rec { crateName = "zmij"; - version = "1.0.21"; + version = "1.0.23"; edition = "2021"; - sha256 = "1amb5i6gz7yjb0dnmz5y669674pqmwbj44p4yfxfv2ncgvk8x15q"; + sha256 = "06zwri21nnrl34rwinmvbciap8yk1mrl8qfg9pff7lgspc56sri9"; authors = [ "David Tolnay " ]; diff --git a/crate-hashes.json b/crate-hashes.json index 960f1c12..4e88c144 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,14 +1,14 @@ { "git+https://github.com/stackabletech/krb5-rs.git?tag=v0.1.0#krb5-sys@0.1.0": "148zr0q04163hpirkrff5q7cbxqgwzzxh0091zr4g23x7l64jh39", "git+https://github.com/stackabletech/krb5-rs.git?tag=v0.1.0#krb5@0.1.0": "148zr0q04163hpirkrff5q7cbxqgwzzxh0091zr4g23x7l64jh39", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#k8s-version@0.1.3": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-certs@0.4.1": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator-derive@0.3.1": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator@0.113.3": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-shared@0.1.2": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-telemetry@0.6.5": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned-macros@0.11.1": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned@0.11.1": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-webhook@0.9.2": "0hgvzbf2qwf9yk5qvk0bsb1jjjdkngprlki1jaqhh005z4jxk5fz", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#k8s-version@0.1.3": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-certs@0.4.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator-derive@0.3.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator@0.113.3": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-shared@0.1.2": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-telemetry@0.6.5": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned-macros@0.11.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned@0.11.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-webhook@0.9.2": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/docs/modules/opa/pages/usage-guide/resource-info-fetcher.adoc b/docs/modules/opa/pages/usage-guide/resource-info-fetcher.adoc new file mode 100644 index 00000000..0053adf5 --- /dev/null +++ b/docs/modules/opa/pages/usage-guide/resource-info-fetcher.adoc @@ -0,0 +1,120 @@ += Resource info fetcher +:description: Resource Info Fetcher for OPA retrieves data from backends like DataHub. Integrate extra resource details into Rego rules for enhanced policy management. + +The _Resource info fetcher_ allows for additional information to be obtained from the configured backend (for example, DataHub). +You can then write Rego rules for OpenPolicyAgent which make an HTTP request to the Resource info fetcher and make use of the additional information returned for the resource, such as Trino tables or Kafka topic. + +You can enable the Resource info fetcher sidecar as follows: + +[source,yaml] +---- +apiVersion: opa.stackable.tech/v1alpha2 +kind: OpaCluster +metadata: + name: opa +spec: + image: + productVersion: 1.0.0 + clusterConfig: + resourceInfo: # <1> + backend: + dataHub: + hostname: datahub-gms.my-namespace.svc.cluster.local + port: 8080 + tls: + verification: + server: + caCert: + secretClass: tls # <2> + credentialsSecretName: resource-info-fetcher-credentials # <3> + env: PROD # <4> + cache: # optional, enabled by default + entryTimeToLive: 60s # optional, defaults to 60s + servers: + roleGroups: + default: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: resource-info-fetcher-credentials +stringData: + token: # <3> +---- + +<1> Enable the `resource-info-fetcher` sidecar +<2> Enable TLS verification using the CA from the `tls` SecretClass. +<3> Authenticate to DataHub with a Personal Access Token (PAT) read from the specified Secret. The Secret must have a `token` entry. See the DataHub documentation on https://docs.datahub.com/docs/authentication/personal-access-tokens[Personal Access Tokens] for how to create one, and make sure https://docs.datahub.com/docs/authentication/introducing-metadata-service-authentication[Metadata Service Authentication] is enabled on your DataHub. +<4> The DataHub environment (fabric) to query, e.g. `PROD` or `DEV`. Defaults to `PROD`. + +Currently the following backends are supported: + +* xref:#backend-data-hub[] + +[#backends] +== Backends + +The initial version is tightly modeled to the only supported backend, namely DataHub. +The API might change as soon as we add support for other backends. + + +[#backend-data-hub] +=== DataHub + +DataHub is currently only supported backend, so the general documentation from above applies. + +== Resource info fetcher API + +Resource information can be retrieved from regorules using the functions `TODO` in `data.stackable.opa.resourceinfo.v1`. + +An example of the returned structure: + +[source,json] +---- +{ + "dataProducts": [], + "domain": null, + "owners": { + "urn:li:ownershipType:__system__business_owner": { + "groups": [ + { + "description": "Customer Service/Analytics (mirrored from the Keycloak demo realm)", + "displayName": "Customer Service/Analytics", + "urn": "urn:li:corpGroup:customer-service-analytics" + } + ], + "ownershipTypeName": "Business Owner", + "users": [] + } + }, + "tags": [ + { + "name": "PII", + "urn": "urn:li:tag:PII" + } + ], + "urn": "urn:li:container:c8531e5a52cacf56768d0bf77ca8787c" +} +---- + +=== Debug request + +To debug the resource-info-fetcher you can `curl` it's API for a given resource. +To achieve this shell into the `opa` container and execute + +[source,bash] +---- +curl --header 'Content-Type: application/json' -d '{"stacklet": "trino", "trinoSchema": {"catalog": "lakehouse", "schema": "customer_analytics"}}' localhost:9477/resource | jq + +curl --header 'Content-Type: application/json' -d '{"stacklet": "", "dataHubUrn": "urn:li:chart:(superset,1)"}' localhost:9477/resource | jq +---- + +=== Rego rule library + +The HTTP API exposed by the resource-info-fetcher can be called directly using the rego function `http.send`. +However, we provide a convenience rego rule library, which we ship with `OpaClusters` by default. + +[source,rego] +---- +TODO +---- diff --git a/docs/modules/opa/pages/usage-guide/user-info-fetcher.adoc b/docs/modules/opa/pages/usage-guide/user-info-fetcher.adoc index 7eb9b212..da570a8d 100644 --- a/docs/modules/opa/pages/usage-guide/user-info-fetcher.adoc +++ b/docs/modules/opa/pages/usage-guide/user-info-fetcher.adoc @@ -46,7 +46,7 @@ stringData: <1> Enable the `user-info-fetcher` sidecar <2> Enable TLS verification using the CA from the `tls` SecretClass. -<3> Obtain Keycloak API credentials from the specified secret. The Secret must have `clientId` and `clientSecret` entries. +<3> Obtain Keycloak API credentials from the specified Secret. The Secret must have `clientId` and `clientSecret` entries. <4> Refer to the applicable realm in your Keycloak server. Currently the following backends are supported: diff --git a/extra/crds.yaml b/extra/crds.yaml index 529722d5..13c92585 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -29,6 +29,7 @@ spec: clusterConfig: default: listenerClass: cluster-internal + resourceInfo: null tls: null userInfo: null description: Global OPA cluster configuration that applies to all roles and role groups. @@ -52,6 +53,111 @@ spec: - external-unstable - external-stable type: string + resourceInfo: + description: |- + Configures how to fetch additional metadata about resource information from a data + catalog. + + Data catalog could e.g. be DataHub and resources could be Trino catalogs, schemas, + tables or Kafka topic etc. + nullable: true + properties: + backend: + description: The backend directory service to use. + oneOf: + - required: + - dataHub + properties: + dataHub: + description: Backend that fetches resource information from DataHub. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing a DataHub Personal Access Token (PAT) that is authorized + to read resource metadata. + + Must contain the field `token`. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + env: + default: PROD + description: The env in DataHub, defaults to `PROD` + type: string + hostname: + description: Hostname of DataHub + type: string + port: + description: Port of DataHub. If TLS is used defaults to `443`, otherwise to `80`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - credentialsSecretName + - hostname + type: object + type: object + cache: + default: + entryTimeToLive: 1m + description: Caching configuration. + properties: + entryTimeToLive: + default: 1m + description: How long responses should be cached for. + type: string + type: object + required: + - backend + type: object tls: description: |- TLS encryption settings for the OPA server. @@ -925,6 +1031,86 @@ spec: description: Configuration per logger type: object type: object + resource-info-fetcher: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object user-info-fetcher: anyOf: - required: @@ -1546,6 +1732,86 @@ spec: description: Configuration per logger type: object type: object + resource-info-fetcher: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object user-info-fetcher: anyOf: - required: @@ -1960,6 +2226,7 @@ spec: clusterConfig: default: listenerClass: cluster-internal + resourceInfo: null tls: null userInfo: null description: Global OPA cluster configuration that applies to all roles and role groups. @@ -1983,6 +2250,111 @@ spec: - external-unstable - external-stable type: string + resourceInfo: + description: |- + Configures how to fetch additional metadata about resource information from a data + catalog. + + Data catalog could e.g. be DataHub and resources could be Trino catalogs, schemas, + tables or Kafka topic etc. + nullable: true + properties: + backend: + description: The backend directory service to use. + oneOf: + - required: + - dataHub + properties: + dataHub: + description: Backend that fetches resource information from DataHub. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing a DataHub Personal Access Token (PAT) that is authorized + to read resource metadata. + + Must contain the field `token`. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + env: + default: PROD + description: The env in DataHub, defaults to `PROD` + type: string + hostname: + description: Hostname of DataHub + type: string + port: + description: Port of DataHub. If TLS is used defaults to `443`, otherwise to `80`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - credentialsSecretName + - hostname + type: object + type: object + cache: + default: + entryTimeToLive: 1m + description: Caching configuration. + properties: + entryTimeToLive: + default: 1m + description: How long responses should be cached for. + type: string + type: object + required: + - backend + type: object tls: description: |- TLS encryption settings for the OPA server. @@ -2856,6 +3228,86 @@ spec: description: Configuration per logger type: object type: object + resource-info-fetcher: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object user-info-fetcher: anyOf: - required: @@ -3477,6 +3929,86 @@ spec: description: Configuration per logger type: object type: object + resource-info-fetcher: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object user-info-fetcher: anyOf: - required: diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 44d83b46..7f7b21dd 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -2,6 +2,7 @@ pub mod config_json; pub mod product_logging; +pub mod resource_info_fetcher; pub mod user_info_fetcher; /// The names of the config files assembled into the rolegroup `ConfigMap`. @@ -14,6 +15,8 @@ pub enum ConfigFileName { ConfigJson, #[strum(serialize = "user-info-fetcher.json")] UserInfoFetcher, + #[strum(serialize = "resource-info-fetcher.json")] + ResourceInfoFetcher, } #[cfg(test)] diff --git a/rust/operator-binary/src/controller/build/properties/resource_info_fetcher.rs b/rust/operator-binary/src/controller/build/properties/resource_info_fetcher.rs new file mode 100644 index 00000000..85dbf28a --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/resource_info_fetcher.rs @@ -0,0 +1,18 @@ +//! Builds the OPA `resource-info-fetcher.json` file. + +use snafu::{ResultExt, Snafu}; + +use crate::crd::resource_info_fetcher; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to serialize resource info fetcher configuration"))] + SerializeResourceInfoFetcherConfig { source: serde_json::Error }, +} + +type Result = std::result::Result; + +/// Serializes the resource-info-fetcher configuration into the `resource-info-fetcher.json` file content. +pub fn build(resource_info: &resource_info_fetcher::v1alpha1::Config) -> Result { + serde_json::to_string_pretty(resource_info).context(SerializeResourceInfoFetcherConfigSnafu) +} diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index bc9aee78..9e3f0878 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -9,7 +9,9 @@ use stackable_operator::{ use crate::controller::{ OpaRoleGroupConfig, RoleGroupName, ValidatedCluster, - build::properties::{ConfigFileName, config_json, product_logging, user_info_fetcher}, + build::properties::{ + ConfigFileName, config_json, product_logging, resource_info_fetcher, user_info_fetcher, + }, }; #[derive(Snafu, Debug)] @@ -20,6 +22,11 @@ pub enum Error { #[snafu(display("failed to build user-info-fetcher.json"))] BuildUserInfoFetcher { source: user_info_fetcher::Error }, + #[snafu(display("failed to build resource-info-fetcher.json"))] + BuildResourceInfoFetcher { + source: resource_info_fetcher::Error, + }, + #[snafu(display("failed to assemble ConfigMap for role group {role_group}"))] Assemble { source: stackable_operator::builder::configmap::Error, @@ -63,6 +70,12 @@ pub fn build_rolegroup_config_map( user_info_fetcher::build(user_info).context(BuildUserInfoFetcherSnafu)?, ); } + if let Some(resource_info) = &cluster.cluster_config.resource_info { + cm_builder.add_data( + ConfigFileName::ResourceInfoFetcher.to_string(), + resource_info_fetcher::build(resource_info).context(BuildResourceInfoFetcherSnafu)?, + ); + } if rolegroup_config.config.logging.vector_container.is_some() { cm_builder.add_data( diff --git a/rust/operator-binary/src/controller/build/resource/daemonset/mod.rs b/rust/operator-binary/src/controller/build/resource/daemonset/mod.rs index e291c812..c10a0f11 100644 --- a/rust/operator-binary/src/controller/build/resource/daemonset/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/daemonset/mod.rs @@ -50,12 +50,19 @@ use super::service::{self, APP_PORT, APP_PORT_NAME}; use crate::{ controller::{ OpaRoleGroupConfig, RoleGroupName, ValidatedCluster, ValidatedOpaConfig, - build::{self, resource::daemonset::user_info_fetcher::add_user_info_fetcher_sidecar}, + build::{ + self, + resource::daemonset::{ + resource_info_fetcher::add_resource_info_fetcher_sidecar, + user_info_fetcher::add_user_info_fetcher_sidecar, + }, + }, }, crd::{Container, DEFAULT_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT}, operations::graceful_shutdown::add_graceful_shutdown_config, }; +mod resource_info_fetcher; mod user_info_fetcher; pub const BUNDLES_ACTIVE_DIR: &str = "/bundles/active"; @@ -67,10 +74,13 @@ const CONFIG_DIR: &str = "/stackable/config"; stackable_operator::constant!(LOG_VOLUME_NAME: VolumeName = "log"); stackable_operator::constant!(BUNDLES_VOLUME_NAME: VolumeName = "bundles"); const BUNDLES_DIR: &str = "/bundles"; -stackable_operator::constant!(USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME: VolumeName = "credentials"); +stackable_operator::constant!(USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME: VolumeName = "user-info-fetcher-credentials"); +// As UIF and RIF run in two different containers, the directories won't clash const USER_INFO_FETCHER_CREDENTIALS_DIR: &str = "/stackable/credentials"; stackable_operator::constant!(USER_INFO_FETCHER_KERBEROS_VOLUME_NAME: VolumeName = "kerberos"); const USER_INFO_FETCHER_KERBEROS_DIR: &str = "/stackable/kerberos"; +stackable_operator::constant!(RESOURCE_INFO_FETCHER_CREDENTIALS_VOLUME_NAME: VolumeName = "resource-info-fetcher-credentials"); +const RESOURCE_INFO_FETCHER_CREDENTIALS_DIR: &str = "/stackable/credentials"; stackable_operator::constant!(TLS_VOLUME_NAME: VolumeName = "tls"); const TLS_STORE_DIR: &str = "/stackable/tls"; @@ -146,6 +156,11 @@ pub enum Error { #[snafu(display("failed to build User Info Fetcher sidecar"))] BuildUserInfoFetcherSidecar { source: user_info_fetcher::Error }, + + #[snafu(display("failed to build Resource Info Fetcher sidecar"))] + BuildResourceInfoFetcherSidecar { + source: resource_info_fetcher::Error, + }, } type Result = std::result::Result; @@ -222,6 +237,7 @@ pub fn build_server_rolegroup_daemonset( role_group: &OpaRoleGroupConfig, opa_bundle_builder_image: &str, user_info_fetcher_image: &str, + resource_info_fetcher_image: &str, service_account: &ServiceAccount, cluster_info: &KubernetesClusterInfo, ) -> Result { @@ -423,6 +439,14 @@ pub fn build_server_rolegroup_daemonset( cluster_info, ) .context(BuildUserInfoFetcherSidecarSnafu)?; + add_resource_info_fetcher_sidecar( + &mut pb, + cluster, + merged_config, + resource_info_fetcher_image, + cluster_info, + ) + .context(BuildResourceInfoFetcherSidecarSnafu)?; // The Vector logging config was validated up-front (see `ValidatedLogging`); a `Some` here means // the Vector agent is enabled and the aggregator discovery ConfigMap name is valid. @@ -749,6 +773,7 @@ mod tests { role_group, "bundle-builder-image", "user-info-fetcher-image", + "resource-info-fetcher-image", &service_account(), &cluster_info(), ) @@ -1118,9 +1143,9 @@ mod tests { }))); // The client credentials secret is projected into the sidecar's credentials dir. - assert!(volume_names(&ds).contains(&"credentials".to_owned())); + assert!(volume_names(&ds).contains(&"user-info-fetcher-credentials".to_owned())); assert_eq!( - mount_path(&uif_container(&ds), "credentials"), + mount_path(&uif_container(&ds), "user-info-fetcher-credentials"), "/stackable/credentials" ); } diff --git a/rust/operator-binary/src/controller/build/resource/daemonset/resource_info_fetcher.rs b/rust/operator-binary/src/controller/build/resource/daemonset/resource_info_fetcher.rs new file mode 100644 index 00000000..93886b3a --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/daemonset/resource_info_fetcher.rs @@ -0,0 +1,108 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + self, + pod::{PodBuilder, volume::VolumeBuilder}, + }, + commons::tls_verification::TlsClientDetailsError, + k8s_openapi::api::core::v1::SecretVolumeSource, + utils::cluster_info::KubernetesClusterInfo, + v2::builder::pod::container::new_container_builder, +}; + +use crate::{ + controller::{ + ValidatedCluster, ValidatedOpaConfig, + build::{ + self, + resource::daemonset::{ + CONFIG_DIR, CONFIG_VOLUME_NAME, RESOURCE_INFO_FETCHER_CREDENTIALS_DIR, + RESOURCE_INFO_FETCHER_CREDENTIALS_VOLUME_NAME, + add_stackable_rust_cli_env_vars, container_name, sidecar_container_log_level, + sidecar_resource_requirements, + }, + }, + }, + crd::{Container, resource_info_fetcher}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display( + "failed to build volume or volume mount spec for the Resource Info Fetcher TLS config" + ))] + TlsVolumeAndMounts { source: TlsClientDetailsError }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { source: builder::pod::Error }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: builder::pod::container::Error, + }, +} + +type Result = std::result::Result; + +pub fn add_resource_info_fetcher_sidecar( + pb: &mut PodBuilder, + cluster: &ValidatedCluster, + merged_config: &ValidatedOpaConfig, + resource_info_fetcher_image: &str, + cluster_info: &KubernetesClusterInfo, +) -> Result<()> { + if let Some(resource_info) = &cluster.cluster_config.resource_info { + let rif_container_name = container_name(&Container::ResourceInfoFetcher); + let mut cb_rif = new_container_builder(&rif_container_name); + + cb_rif + .image_from_product_image(&cluster.image) // inherit the pull policy and pull secrets, and then... + .image(resource_info_fetcher_image) // ...override the image + .command(vec!["stackable-opa-resource-info-fetcher".to_string()]) + .add_env_var( + "CONFIG", + format!( + "{CONFIG_DIR}/{file}", + file = build::properties::ConfigFileName::ResourceInfoFetcher + ), + ) + .add_env_var("CREDENTIALS_DIR", RESOURCE_INFO_FETCHER_CREDENTIALS_DIR) + .add_volume_mount(CONFIG_VOLUME_NAME.as_ref(), CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .resources(sidecar_resource_requirements()); + add_stackable_rust_cli_env_vars( + &mut cb_rif, + cluster_info, + sidecar_container_log_level(merged_config, &Container::ResourceInfoFetcher).to_string(), + &Container::ResourceInfoFetcher, + ); + + match &resource_info.backend { + resource_info_fetcher::v1alpha1::Backend::DataHub(data_hub) => { + pb.add_volume( + VolumeBuilder::new(RESOURCE_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref()) + .secret(SecretVolumeSource { + secret_name: Some(data_hub.credentials_secret_name.to_string()), + ..Default::default() + }) + .build(), + ) + .context(AddVolumeSnafu)?; + cb_rif + .add_volume_mount( + RESOURCE_INFO_FETCHER_CREDENTIALS_VOLUME_NAME.as_ref(), + RESOURCE_INFO_FETCHER_CREDENTIALS_DIR, + ) + .context(AddVolumeMountSnafu)?; + data_hub + .tls + .add_volumes_and_mounts(pb, vec![&mut cb_rif]) + .context(TlsVolumeAndMountsSnafu)?; + } + } + + pb.add_container(cb_rif.build()); + } + + Ok(()) +} diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 31336579..b39c3054 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -31,7 +31,7 @@ use stackable_operator::{ use crate::{ crd::{ APP_NAME, OPERATOR_NAME, OpaConfig, OpaConfigOverrides, OpaRole, OpaStorageConfig, - user_info_fetcher, v1alpha2, + resource_info_fetcher, user_info_fetcher, v1alpha2, }, opa_controller::OPA_CONTROLLER_NAME, }; @@ -236,6 +236,7 @@ impl KubeResource for ValidatedCluster { /// raw `OpaCluster` to render config (except for owner references). pub struct ValidatedClusterConfig { pub user_info: Option, + pub resource_info: Option, pub tls: Option, pub listener_class: v1alpha2::CurrentlySupportedListenerClasses, } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index aef9c278..9698e950 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -231,6 +231,7 @@ pub fn validate( image, ValidatedClusterConfig { user_info: opa.spec.cluster_config.user_info.clone(), + resource_info: opa.spec.cluster_config.resource_info.clone(), tls: opa.spec.cluster_config.tls.clone(), listener_class: opa.spec.cluster_config.listener_class.clone(), }, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index fb6af2da..29ed7771 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -121,6 +121,14 @@ pub mod versioned { #[serde(default)] pub user_info: Option, + /// Configures how to fetch additional metadata about resource information from a data + /// catalog. + /// + /// Data catalog could e.g. be DataHub and resources could be Trino catalogs, schemas, + /// tables or Kafka topics etc. + #[serde(default)] + pub resource_info: Option, + /// TLS encryption settings for the OPA server. /// When configured, OPA will use HTTPS (port 8443) instead of HTTP (port 8081). /// Clients must connect using HTTPS and trust the certificates provided by the configured SecretClass. @@ -202,6 +210,7 @@ pub enum Container { BundleBuilder, Opa, UserInfoFetcher, + ResourceInfoFetcher, } // NOTE (@Techassi): This struct can currently NOT be versioned because it is used via Role which diff --git a/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs b/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs index 57e76154..885132db 100644 --- a/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs +++ b/rust/operator-binary/src/crd/resource_info_fetcher/mod.rs @@ -1,11 +1,13 @@ -use crate::crd::cache::Cache; use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::{networking::HostName, tls_verification::TlsClientDetails}, schemars::{self, JsonSchema}, + v2::types::kubernetes::SecretName, versioned::versioned, }; +use crate::crd::cache::Cache; + #[versioned(version(name = "v1alpha1"))] pub mod versioned { #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -39,6 +41,12 @@ pub mod versioned { #[serde(flatten)] pub tls: TlsClientDetails, + /// Name of a Secret containing a DataHub Personal Access Token (PAT) that is authorized + /// to read resource metadata. + /// + /// Must contain the field `token`. + pub credentials_secret_name: SecretName, + /// The env in DataHub, defaults to `PROD` #[serde(default = "default_data_hub_env")] pub env: String, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index d4af4eb2..0116d62d 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -167,7 +167,8 @@ async fn main() -> anyhow::Result<()> { Arc::new(opa_controller::Ctx { client: client.clone(), opa_bundle_builder_image: operator_image.clone(), - user_info_fetcher_image: operator_image, + user_info_fetcher_image: operator_image.clone(), + resource_info_fetcher_image: operator_image, operator_environment, cluster_info: kubernetes_cluster_info, }), diff --git a/rust/operator-binary/src/opa_controller.rs b/rust/operator-binary/src/opa_controller.rs index 2e74f7c2..38a00da9 100644 --- a/rust/operator-binary/src/opa_controller.rs +++ b/rust/operator-binary/src/opa_controller.rs @@ -47,6 +47,7 @@ pub struct Ctx { pub client: stackable_operator::client::Client, pub opa_bundle_builder_image: String, pub user_info_fetcher_image: String, + pub resource_info_fetcher_image: String, pub cluster_info: KubernetesClusterInfo, pub operator_environment: OperatorEnvironmentOptions, } @@ -228,6 +229,7 @@ pub async fn reconcile_opa( rolegroup, &ctx.opa_bundle_builder_image, &ctx.user_info_fetcher_image, + &ctx.resource_info_fetcher_image, &rbac_sa, &ctx.cluster_info, ) diff --git a/rust/regorule-library/src/lib.rs b/rust/regorule-library/src/lib.rs index b032dade..d222e555 100644 --- a/rust/regorule-library/src/lib.rs +++ b/rust/regorule-library/src/lib.rs @@ -1,4 +1,10 @@ -pub const REGORULES: &[(&str, &str)] = &[( - "stackable/opa/userinfo/v1.rego", - include_str!("userinfo/v1.rego"), -)]; +pub const REGORULES: &[(&str, &str)] = &[ + ( + "stackable/opa/userinfo/v1.rego", + include_str!("userinfo/v1.rego"), + ), + ( + "stackable/opa/resourceinfo/v1.rego", + include_str!("resourceinfo/v1.rego"), + ), +]; diff --git a/rust/regorule-library/src/resourceinfo/v1.rego b/rust/regorule-library/src/resourceinfo/v1.rego new file mode 100644 index 00000000..a7bbfe4f --- /dev/null +++ b/rust/regorule-library/src/resourceinfo/v1.rego @@ -0,0 +1,44 @@ +package stackable.opa.resourceinfo.v1 + +# Trino catalog +resourceInfoTrinoCatalog(stacklet, catalog) := resourceInfoJson( + {"stacklet": stacklet, "trinoCatalog": {"catalog": catalog}} +) + +# Trino schema +resourceInfoTrinoSchema(stacklet, catalog, schema) := resourceInfoJson( + {"stacklet": stacklet, "trinoSchema": {"catalog": catalog, "schema": schema}} +) + +# Trino table +resourceInfoTrinoTable(stacklet, catalog, schema, table) := resourceInfoJson( + {"stacklet": stacklet, "trinoTable": {"catalog": catalog, "schema": schema, "table": table}} +) + +# Superset chart +resourceInfoSupersetChart(stacklet, id) := resourceInfoJson( + {"stacklet": stacklet, "supersetChart": {"id": id}} +) + +# Superset dashboard +resourceInfoSupersetDashboard(stacklet, id) := resourceInfoJson( + {"stacklet": stacklet, "supersetDashboard": {"id": id}} +) + +# Kafka topic +resourceInfoKafkaTopic(stacklet, topic) := resourceInfoJson( + {"stacklet": stacklet, "kafkaTopic": {"topic": topic}} +) + +# Raw DataHub urn +resourceInfoDataHubUrn(urn) := resourceInfoJson( + {"stacklet": "dummy", "dataHubUrn": urn} +) + +resourceInfoJson(json) := http.send({ + "method": "POST", + "url": "http://127.0.0.1:9477/resource", + "body": json, + "headers": {"Content-Type": "application/json"}, + "raise_error": true +}).body diff --git a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs index 2208766b..73eb6dd5 100644 --- a/rust/resource-info-fetcher/src/backend/data_hub/mod.rs +++ b/rust/resource-info-fetcher/src/backend/data_hub/mod.rs @@ -25,14 +25,8 @@ mod resource_to_urn_mapping; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to read client ID from {path:?}"))] - ReadClientId { - source: std::io::Error, - path: String, - }, - - #[snafu(display("failed to read client secret from {path:?}"))] - ReadClientSecret { + #[snafu(display("failed to read DataHub token from {path:?}"))] + ReadToken { source: std::io::Error, path: String, }, @@ -62,8 +56,7 @@ pub enum Error { impl http_error::Error for Error { fn status_code(&self) -> StatusCode { match self { - Self::ReadClientId { .. } => StatusCode::SERVICE_UNAVAILABLE, - Self::ReadClientSecret { .. } => StatusCode::SERVICE_UNAVAILABLE, + Self::ReadToken { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::ConfigureTls { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::ConstructHttpClient { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::BuildDataHubEndpoint { .. } => StatusCode::BAD_REQUEST, @@ -162,8 +155,7 @@ pub struct Group { /// internally. pub struct ResolvedDataHubBackend { config: v1alpha1::DataHubBackend, - client_id: String, - client_secret: String, + token: String, http_client: reqwest::Client, graphql_url: Url, } @@ -175,19 +167,16 @@ impl ResolvedDataHubBackend { config: v1alpha1::DataHubBackend, credentials_dir: &Path, ) -> Result { - let client_id_path = credentials_dir.join("clientId"); - let client_secret_path = credentials_dir.join("clientSecret"); + let token_path = credentials_dir.join("token"); - let client_id = tokio::fs::read_to_string(&client_id_path) - .await - .with_context(|_| ReadClientIdSnafu { - path: client_id_path.display().to_string(), - })?; - let client_secret = tokio::fs::read_to_string(&client_secret_path) + // Trim trailing whitespace/newlines so the value is safe to use in an HTTP header. + let token = tokio::fs::read_to_string(&token_path) .await - .with_context(|_| ReadClientSecretSnafu { - path: client_secret_path.display().to_string(), - })?; + .with_context(|_| ReadTokenSnafu { + path: token_path.display().to_string(), + })? + .trim() + .to_owned(); let mut client_builder = ClientBuilder::new(); client_builder = utils::tls::configure_reqwest(&config.tls, client_builder) @@ -212,8 +201,7 @@ impl ResolvedDataHubBackend { Ok(Self { config, - client_id, - client_secret, + token, http_client, graphql_url, }) @@ -229,14 +217,9 @@ impl ResolvedDataHubBackend { let response: graphql::GraphQlResponse = send_json_request( self.http_client .post(self.graphql_url.clone()) - // DataHub's system authenticator strips the leading "Basic " and compares the rest - // VERBATIM against ":" — it is NOT standard RFC 7617 Basic auth. So do - // NOT use reqwest's .basic_auth(), which base64-encodes the credentials; set the - // header manually so the value goes out unencoded. - .header( - reqwest::header::AUTHORIZATION, - format!("Basic {}:{}", self.client_id, self.client_secret), - ) + // Authenticate with a DataHub Personal Access Token (a bearer JWT). DataHub's + // Metadata Service Authentication verifies it and resolves it to the token's actor. + .bearer_auth(&self.token) .json(&graphql::request(urn)), ) .await diff --git a/rust/resource-info-fetcher/src/main.rs b/rust/resource-info-fetcher/src/main.rs index fe124425..97d4cd1d 100644 --- a/rust/resource-info-fetcher/src/main.rs +++ b/rust/resource-info-fetcher/src/main.rs @@ -25,7 +25,7 @@ pub mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } -pub const APP_NAME: &str = "opa-user-info-fetcher"; +pub const APP_NAME: &str = "opa-resource-info-fetcher"; #[derive(clap::Parser)] pub struct Args { @@ -74,7 +74,7 @@ enum StartupError { #[snafu(display("failed to run server"))] RunServer { source: std::io::Error }, - #[snafu(display("failed to resolve Keycloak backend"))] + #[snafu(display("failed to resolve DataHub backend"))] ResolveDataHubBackend { source: backend::data_hub::Error }, } diff --git a/tests/templates/kuttl/data-hub-resource-info/00-patch-ns.yaml.j2 b/tests/templates/kuttl/data-hub-resource-info/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/data-hub-resource-info/01-assert.yaml.j2 b/tests/templates/kuttl/data-hub-resource-info/01-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/01-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/data-hub-resource-info/01-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/data-hub-resource-info/01-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/01-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/data-hub-resource-info/02-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/02-assert.yaml new file mode 100644 index 00000000..19771a3c --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/02-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# OpenSearch + Kafka can be slow to become ready on a cold cluster (image pulls, JVM start). +timeout: 900 +commands: + # Wait for the stateful prerequisites to be ready before installing DataHub itself. + # (Pod/StatefulSet names come from the `prerequisites` release; adjust if a chart bump + # renames them.) + - script: kubectl -n $NAMESPACE rollout status --timeout=590s statefulset/opensearch-cluster-master + - script: kubectl -n $NAMESPACE rollout status --timeout=590s statefulset/prerequisites-mysql + - script: kubectl -n $NAMESPACE rollout status --timeout=590s statefulset/prerequisites-kafka-controller diff --git a/tests/templates/kuttl/data-hub-resource-info/02-install-datahub-prerequisites.yaml.j2 b/tests/templates/kuttl/data-hub-resource-info/02-install-datahub-prerequisites.yaml.j2 new file mode 100644 index 00000000..b8a1edee --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/02-install-datahub-prerequisites.yaml.j2 @@ -0,0 +1,31 @@ +--- +# Secrets consumed by both the prerequisites chart (MySQL root password) and the +# DataHub chart (token service signing key + system client secret for metadata auth). +# These are the canonical secret names/keys from the datahub-helm quickstart. +apiVersion: v1 +kind: Secret +metadata: + name: mysql-secrets +stringData: + mysql-root-password: datahub +--- +apiVersion: v1 +kind: Secret +metadata: + name: datahub-auth-secrets +stringData: + # In a real deployment these would be strong random values. + token_service_signing_key: "WnEdIeTG/VVCLQqGwC/BAkqyY0k+H8NEAtWGejrBI94=" + system_client_secret: "changemeplease" +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # Release name is `prerequisites` (the datahub chart's defaults point at it); + # the chart itself is named `datahub-prerequisites`. + - script: >- + helm install prerequisites datahub-prerequisites + --namespace $NAMESPACE + --version {{ test_scenario['values']['data-hub-prerequisites'] }} + --repo https://helm.datahubproject.io + --values 02_datahub-prerequisites-values.yaml diff --git a/tests/templates/kuttl/data-hub-resource-info/02_datahub-prerequisites-values.yaml b/tests/templates/kuttl/data-hub-resource-info/02_datahub-prerequisites-values.yaml new file mode 100644 index 00000000..bb611430 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/02_datahub-prerequisites-values.yaml @@ -0,0 +1,64 @@ +# Values for the DataHub "prerequisites" Helm chart (chart 0.3.0). +# +# We keep the chart defaults where they are already minimal: +# - opensearch.enabled: true (Apache-2.0 licensed; used as both search AND graph store) +# - elasticsearch.enabled: false (avoided: Elastic/SSPL license is not Apache-2.0 compatible) +# - neo4j.enabled: false (graph is served by OpenSearch, saves a pod) +# - mysql.enabled: true +# - kafka.enabled: true (Bitnami Kafka in KRaft mode, so no separate ZooKeeper pod) +# - postgresql.enabled: false +# - cp-helm-charts.enabled: false +# +# This is a throwaway test backend: single replica everywhere, no persistence guarantees, +# no HA, small resources. +# +# HEADS UP (verify on first run): the `mysql` and `kafka` subcharts pull Bitnami images. +# Bitnami paywalled its free Docker Hub images in 2025, so the pinned tags may fail to pull. +# If they do, override the image repos to `docker.io/bitnamilegacy/*` here, e.g.: +# mysql: +# image: +# registry: docker.io +# repository: bitnamilegacy/mysql +# kafka: +# image: +# registry: docker.io +# repository: bitnamilegacy/kafka + +opensearch: + singleNode: true + replicas: 1 + # Single-node OpenSearch: no dedicated master election needed. + opensearchJavaOpts: "-Xms512m -Xmx512m" + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 1Gi + +mysql: + auth: + # Root password is read from the `mysql-secrets` Secret created in 02-install-datahub-prerequisites. + # This is the same Secret the datahub chart reads (its default global.sql.datasource.password). + existingSecret: mysql-secrets + database: datahub + primary: + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 512Mi + +kafka: + controller: + replicaCount: 1 + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "1" + memory: 1Gi diff --git a/tests/templates/kuttl/data-hub-resource-info/03-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/03-assert.yaml new file mode 100644 index 00000000..ad6615ba --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/03-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# GMS waits for the system-update bootstrap job (schema + indices) before it goes ready, +# so give this plenty of headroom. +timeout: 900 +commands: + - script: kubectl -n $NAMESPACE rollout status --timeout=890s deployment/datahub-datahub-gms + - script: kubectl -n $NAMESPACE rollout status --timeout=890s deployment/datahub-datahub-frontend diff --git a/tests/templates/kuttl/data-hub-resource-info/03-install-datahub.yaml.j2 b/tests/templates/kuttl/data-hub-resource-info/03-install-datahub.yaml.j2 new file mode 100644 index 00000000..484d3bb6 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/03-install-datahub.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # The chart runs a `datahub-system-update` bootstrap job (creates the MySQL schema and + # the OpenSearch indices) before GMS starts; 03-assert waits for GMS to become ready. + - script: >- + helm install datahub datahub + --namespace $NAMESPACE + --version {{ test_scenario['values']['data-hub'] }} + --repo https://helm.datahubproject.io + --values 03_datahub-values.yaml diff --git a/tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml b/tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml new file mode 100644 index 00000000..01181468 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml @@ -0,0 +1,54 @@ +# Values for the DataHub Helm chart (chart 1.0.3, app v1.6.0). +# +# Installed with release name `datahub`, so the resulting Services are: +# - datahub-datahub-gms:8080 (GMS / metadata service, what the resource-info-fetcher talks to) +# - datahub-datahub-frontend:9002 (React frontend; needed here to mint a Personal Access Token) +# +# The chart's default connection settings already point at a `prerequisites`-named release +# (opensearch / mysql / kafka), so we only override what we need for a minimal, secured test. + +global: + # Use OpenSearch for the graph store too, so we don't need Neo4j. + graph_service_impl: elasticsearch + + datahub: + # THE point of this test: require authentication on every GMS request. + # With this on, GMS returns 401 for unauthenticated calls and only accepts a valid + # bearer token (a DataHub Personal Access Token, minted in 04-load-sample-data). + metadata_service_authentication: + enabled: true + # Signing key + system client secret are read from `datahub-auth-secrets` + # (created in 02-install-datahub-prerequisites). These are the chart's default + # secretRef names/keys; kept explicit for clarity. + systemClientSecret: + secretRef: datahub-auth-secrets + secretKey: system_client_secret + tokenService: + signingKey: + secretRef: datahub-auth-secrets + secretKey: token_service_signing_key + +# --- Trim optional components to keep the test light --- +acryl-datahub-actions: + enabled: false +datahub-ingestion-cron: + enabled: false + +# --- Small resources for the core components --- +datahub-gms: + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + +datahub-frontend: + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/tests/templates/kuttl/data-hub-resource-info/04-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/04-assert.yaml new file mode 100644 index 00000000..cf572412 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/04-assert.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + # Sample data finished loading into GMS. + - script: kubectl -n $NAMESPACE wait --for=condition=complete --timeout=590s job/sample-data-ingest +--- +# The credentials Secret for the resource-info-fetcher exists. +apiVersion: v1 +kind: Secret +metadata: + name: datahub-rif-credentials diff --git a/tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml b/tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml new file mode 100644 index 00000000..3c041a90 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml @@ -0,0 +1,98 @@ +--- +# Ingestion recipe: DataHub's built-in bootstrap "demo-data" (datasets, dashboards, tags, +# users, glossary terms). The token is injected from the environment at run time. +apiVersion: v1 +kind: ConfigMap +metadata: + name: datahub-sample-data-recipe +data: + recipe.yml: | + source: + type: demo-data + config: {} + sink: + type: datahub-rest + config: + server: "http://datahub-datahub-gms:8080" + token: "${DATAHUB_GMS_TOKEN}" +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 900 +commands: + # 1) Mint a DataHub Personal Access Token (PAT). + # GMS has metadata_service_authentication enabled, so we authenticate to the frontend + # with the default bootstrap user (datahub/datahub) to get a session, then call the + # createAccessToken GraphQL mutation. Run inside a throwaway pod because the frontend + # Service is only reachable in-cluster. + - script: | + set -euo pipefail + + # The frontend Deployment reports "available" before GMS finishes bootstrapping the default + # policies that authorize the root user to mint tokens, so early attempts come back empty. + # Retry until a token appears and emit it on a marker line for reliable capture. + kubectl -n "$NAMESPACE" run pat-minter --restart=Never \ + --image=oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev \ + --command -- bash -ceu ' + FE="http://datahub-datahub-frontend:9002" + for i in $(seq 1 60); do + curl -s -c /tmp/cookies.txt -X POST "$FE/logIn" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"datahub\",\"password\":\"datahub\"}" >/dev/null || true + resp="$(curl -s -b /tmp/cookies.txt -X POST "$FE/api/v2/graphql" \ + -H "Content-Type: application/json" \ + -d "{\"query\":\"mutation {createAccessToken(input:{type:PERSONAL,actorUrn:\\\"urn:li:corpuser:datahub\\\",duration:ONE_DAY,name:\\\"resource-info-fetcher-test\\\"}){accessToken}}\"}" || true)" + token="$(echo "$resp" | jq -r ".data.createAccessToken.accessToken // empty" 2>/dev/null || true)" + if [ -n "$token" ]; then echo "PAT_TOKEN:$token"; exit 0; fi + echo "attempt $i: DataHub not ready to mint a PAT yet: $resp" >&2 + sleep 5 + done + echo "gave up waiting for DataHub to mint a PAT" >&2 + exit 1 + ' + kubectl -n "$NAMESPACE" wait --for=jsonpath='{.status.phase}'=Succeeded pod/pat-minter --timeout=360s || true + TOKEN="$(kubectl -n "$NAMESPACE" logs pat-minter | sed -n 's/^PAT_TOKEN://p' | tail -n 1)" + if [ -z "$TOKEN" ]; then + echo "Failed to mint a DataHub PAT; pat-minter logs:" >&2 + kubectl -n "$NAMESPACE" logs pat-minter >&2 || true + kubectl -n "$NAMESPACE" delete pod pat-minter --ignore-not-found + exit 1 + fi + kubectl -n "$NAMESPACE" delete pod pat-minter --ignore-not-found + + # 2) Store the PAT as the Secret the resource-info-fetcher (OPA sidecar) will read. + # A PAT is a single bearer token -> single `token` key (NOT clientId/clientSecret). + kubectl -n "$NAMESPACE" create secret generic datahub-rif-credentials \ + --from-literal=token="$TOKEN" + + # 3) Load the built-in sample data, authenticated with the same PAT. + kubectl -n "$NAMESPACE" delete job sample-data-ingest --ignore-not-found + kubectl -n "$NAMESPACE" apply -f - < URN). + kafkaTopic := resourceinfo.resourceInfoKafkaTopic(input.stacklet, input.topic) + --- + apiVersion: opa.stackable.tech/v1alpha2 + kind: OpaCluster + metadata: + name: test-opa + spec: + image: +{% if test_scenario['values']['opa-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opa-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opa-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opa-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + resourceInfo: + backend: + dataHub: + # GMS Service from the datahub Helm release (plain HTTP in this test). + hostname: datahub-datahub-gms.$NAMESPACE.svc.cluster.local + port: 8080 + # Secret holding the DataHub Personal Access Token (key \`token\`), created in + # 04-load-sample-data. The fetcher sends it as \`Authorization: Bearer\`. + credentialsSecretName: datahub-rif-credentials + env: PROD + cache: + entryTimeToLive: 60s +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: {} + EOF diff --git a/tests/templates/kuttl/data-hub-resource-info/20-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/20-assert.yaml new file mode 100644 index 00000000..8c750c5f --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/20-assert.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# The Job's exit code is the verdict. Wait for it to complete and surface its logs either +# way (on success they show the fetcher response; on failure, the assertion error). +timeout: 310 +commands: + - script: | + if kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=300s job/test-resource-info; then + kubectl -n "$NAMESPACE" logs job/test-resource-info + else + echo "resource-info assertion Job did not complete:" + kubectl -n "$NAMESPACE" logs job/test-resource-info || true + exit 1 + fi diff --git a/tests/templates/kuttl/data-hub-resource-info/20-run-test.yaml b/tests/templates/kuttl/data-hub-resource-info/20-run-test.yaml new file mode 100644 index 00000000..7634c06a --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/20-run-test.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # Ship the assertion script into the cluster as a ConfigMap (keeps test-regorule.py a + # real, lintable file instead of inlining Python into YAML). Idempotent so re-runs are safe. + - script: | + kubectl -n "$NAMESPACE" create configmap test-resource-info-script \ + --from-file=test-regorule.py \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-resource-info +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: test + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + command: + - python + - /scripts/test-regorule.py + - -u + - http://test-opa-server:8081/v1/data/test + volumeMounts: + - name: script + mountPath: /scripts + resources: + requests: + cpu: 200m + memory: 128Mi + limits: + cpu: "1" + memory: 128Mi + volumes: + - name: script + configMap: + name: test-resource-info-script diff --git a/tests/templates/kuttl/data-hub-resource-info/test-regorule.py b/tests/templates/kuttl/data-hub-resource-info/test-regorule.py new file mode 100644 index 00000000..faa3ef65 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/test-regorule.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +import argparse +import json + +import requests + +# Every dataset in DataHub's built-in sample data (loaded in 04-load-sample-data) carries the +# same two owners under the "data owner" ownership type. +SAMPLE_OWNERS = ["urn:li:corpuser:datahub", "urn:li:corpuser:jdoe"] + +# Resources looked up by their raw DataHub URN, with the metadata we expect back. Values were +# captured from a live fetcher response against the sample data. +BY_URN_CASES = [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "tags": ["urn:li:tag:Legacy"], + "owners": SAMPLE_OWNERS, + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", + "tags": [], + "owners": SAMPLE_OWNERS, + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "tags": [], + "owners": SAMPLE_OWNERS, + }, +] + + +def tag_urns(resource): + return sorted(tag["urn"] for tag in resource["tags"]) + + +def owner_urns(resource): + # `owners` is keyed by ownership type; collect the user and group urns underneath each. + urns = [] + for ownership in resource["owners"].values(): + urns += [user["urn"] for user in ownership["users"]] + urns += [group["urn"] for group in ownership["groups"]] + return sorted(urns) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-u", "--url", required=True, help="OPA data API base URL for the 'test' package" + ) + args = parser.parse_args() + + def query(rule, opa_input): + # strict-builtin-errors turns a failing resource-info-fetcher call (e.g. rejected GMS + # authentication) into a 500 rather than a silently undefined result. + response = requests.post( + f"{args.url}/{rule}", + params={"strict-builtin-errors": "true"}, + data=json.dumps({"input": opa_input}), + ) + assert response.status_code == 200, ( + f"{rule}: expected 200 from OPA, got {response.status_code}: {response.text}" + ) + body = response.json() + # A missing 'result' means the rule was undefined - typically an auth/connection problem. + assert "result" in body, f"{rule}: rule did not evaluate: {body}" + return body["result"] + + def assert_resource(resource, urn, expected_tags, expected_owners): + assert resource["urn"] == urn, f"expected urn {urn}, got {resource['urn']}" + assert tag_urns(resource) == sorted(expected_tags), ( + f"{urn}: tags {tag_urns(resource)} != expected {sorted(expected_tags)}" + ) + assert owner_urns(resource) == sorted(expected_owners), ( + f"{urn}: owners {owner_urns(resource)} != expected {sorted(expected_owners)}" + ) + + # 1) Resolve several resources by their raw DataHub URN. + for case in BY_URN_CASES: + print(f"Checking byUrn: {case['urn']}") + resource = query("byUrn", {"urn": case["urn"]}) + assert_resource(resource, case["urn"], case["tags"], case["owners"]) + + # 2) Exercise the Stackable KafkaTopic abstraction: the fetcher maps (stacklet, topic) to a + # DataHub URN and resolves it. It must return the same record as the raw-URN lookup above, + # which proves the mapping in resource_to_urn_mapping.rs is correct end-to-end. + print("Checking kafkaTopic abstraction: kafka / SampleKafkaDataset") + kafka_topic = query("kafkaTopic", {"stacklet": "kafka", "topic": "SampleKafkaDataset"}) + assert_resource( + kafka_topic, + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", + [], + SAMPLE_OWNERS, + ) + + print("Test successful!") + + +if __name__ == "__main__": + main() diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 79a48411..0c910d0e 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -14,6 +14,12 @@ dimensions: - name: keycloak values: - 23.0.1 + - name: data-hub + values: + - 1.0.3 + - name: data-hub-prerequisites + values: + - 0.3.0 - name: openshift values: - "false" @@ -62,6 +68,12 @@ tests: dimensions: - opa-latest - openshift + - name: data-hub-resource-info + dimensions: + - opa-latest + - data-hub + - data-hub-prerequisites + - openshift suites: - name: nightly patch: From 86127a682c3771e5af9d0e77dc23b5c505a84ae0 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 23 Jul 2026 15:34:15 +0200 Subject: [PATCH 13/13] Improve test --- tests/release.yaml | 2 + .../03_datahub-values.yaml | 11 + .../04-load-sample-data.yaml | 2 + .../10-install-opa.yaml.j2 | 9 +- .../data-hub-resource-info/20-assert.yaml | 16 +- .../20-install-trino.yaml.j2 | 93 ++++++ .../data-hub-resource-info/21-assert.yaml | 15 + .../21-scrape-trino.yaml | 148 +++++++++ .../data-hub-resource-info/22-assert.yaml | 14 + .../22-create-metadata.yaml | 47 +++ .../data-hub-resource-info/30-assert.yaml | 15 + .../{20-run-test.yaml => 30-run-test.yaml} | 0 .../create-datahub-metadata.py | 287 ++++++++++++++++++ .../data-hub-resource-info/test-regorule.py | 106 +++++++ tests/test-definition.yaml | 6 + 15 files changed, 754 insertions(+), 17 deletions(-) create mode 100644 tests/templates/kuttl/data-hub-resource-info/20-install-trino.yaml.j2 create mode 100644 tests/templates/kuttl/data-hub-resource-info/21-assert.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/21-scrape-trino.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/22-assert.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/22-create-metadata.yaml create mode 100644 tests/templates/kuttl/data-hub-resource-info/30-assert.yaml rename tests/templates/kuttl/data-hub-resource-info/{20-run-test.yaml => 30-run-test.yaml} (100%) create mode 100644 tests/templates/kuttl/data-hub-resource-info/create-datahub-metadata.py diff --git a/tests/release.yaml b/tests/release.yaml index 6df27b86..ffd00dcd 100644 --- a/tests/release.yaml +++ b/tests/release.yaml @@ -14,3 +14,5 @@ releases: operatorVersion: 0.0.0-dev opa: operatorVersion: 0.0.0-dev + trino: + operatorVersion: 0.0.0-dev diff --git a/tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml b/tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml index 01181468..f3323f26 100644 --- a/tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml +++ b/tests/templates/kuttl/data-hub-resource-info/03_datahub-values.yaml @@ -43,6 +43,9 @@ datahub-gms: limits: cpu: "2" memory: 2Gi + extraEnvs: + - name: DATAHUB_TELEMETRY_ENABLED + value: "false" datahub-frontend: resources: @@ -52,3 +55,11 @@ datahub-frontend: limits: cpu: "1" memory: 1Gi + extraEnvs: + - name: DATAHUB_TELEMETRY_ENABLED + value: "false" + +datahubSystemUpdate: + extraEnvs: + - name: DATAHUB_TELEMETRY_ENABLED + value: "false" diff --git a/tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml b/tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml index 3c041a90..6a4e27e8 100644 --- a/tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml +++ b/tests/templates/kuttl/data-hub-resource-info/04-load-sample-data.yaml @@ -88,6 +88,8 @@ commands: secretKeyRef: name: datahub-rif-credentials key: token + - name: DATAHUB_TELEMETRY_ENABLED + value: "false" volumeMounts: - name: recipe mountPath: /recipe diff --git a/tests/templates/kuttl/data-hub-resource-info/10-install-opa.yaml.j2 b/tests/templates/kuttl/data-hub-resource-info/10-install-opa.yaml.j2 index b817eeaf..62520f05 100644 --- a/tests/templates/kuttl/data-hub-resource-info/10-install-opa.yaml.j2 +++ b/tests/templates/kuttl/data-hub-resource-info/10-install-opa.yaml.j2 @@ -17,14 +17,11 @@ commands: import data.stackable.opa.resourceinfo.v1 as resourceinfo - # Thin, input-driven wrappers - the test data (URNs, resource coordinates) lives in - # test-regorule.py. Each exercises the resource-info-fetcher end-to-end: it must - # authenticate to the (PAT-secured) GMS and return the resource metadata. - - # Resolve a resource by its raw DataHub URN. byUrn := resourceinfo.resourceInfoDataHubUrn(input.urn) - # Resolve a Kafka topic via the Stackable abstraction (stacklet + topic -> URN). + trinoCatalog := resourceinfo.resourceInfoTrinoCatalog(input.stacklet, input.catalog) + trinoSchema := resourceinfo.resourceInfoTrinoSchema(input.stacklet, input.catalog, input.schema) + trinoTable := resourceinfo.resourceInfoTrinoTable(input.stacklet, input.catalog, input.schema, input.table) kafkaTopic := resourceinfo.resourceInfoKafkaTopic(input.stacklet, input.topic) --- apiVersion: opa.stackable.tech/v1alpha2 diff --git a/tests/templates/kuttl/data-hub-resource-info/20-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/20-assert.yaml index 8c750c5f..0aa91c5a 100644 --- a/tests/templates/kuttl/data-hub-resource-info/20-assert.yaml +++ b/tests/templates/kuttl/data-hub-resource-info/20-assert.yaml @@ -1,15 +1,9 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -# The Job's exit code is the verdict. Wait for it to complete and surface its logs either -# way (on success they show the fetcher response; on failure, the assertion error). -timeout: 310 +metadata: + name: install-trino +# TPC-H is built into Trino (no data to load), so we only need the cluster to come up. +timeout: 600 commands: - - script: | - if kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=300s job/test-resource-info; then - kubectl -n "$NAMESPACE" logs job/test-resource-info - else - echo "resource-info assertion Job did not complete:" - kubectl -n "$NAMESPACE" logs job/test-resource-info || true - exit 1 - fi + - script: kubectl -n $NAMESPACE wait --for=condition=available trinoclusters.trino.stackable.tech/my-trino --timeout=590s diff --git a/tests/templates/kuttl/data-hub-resource-info/20-install-trino.yaml.j2 b/tests/templates/kuttl/data-hub-resource-info/20-install-trino.yaml.j2 new file mode 100644 index 00000000..0d578b22 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/20-install-trino.yaml.j2 @@ -0,0 +1,93 @@ +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: trino-users +spec: + provider: + static: + userCredentialsSecret: + name: trino-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: trino-users +type: kubernetes.io/opaque +stringData: + admin: adminadmin + # Used by the DataHub Trino ingestion (21-scrape-trino). + datahub: datahubdatahub + # Test users mirrored 1:1 from the DataHub corpusers created in 22-create-metadata (username == + # DataHub corpuser id), so a later end-to-end OPA-on-Trino authz test can authenticate as them + # and have the RIF resolve their DataHub ownership. Passwords follow the doubled-username + # convention above; they are throwaway test credentials. + alice.turner: alice.turneralice.turner + bob.ramirez: bob.ramirezbob.ramirez + carla.nowak: carla.nowakcarla.nowak + david.okoye: david.okoyedavid.okoye + erin.fischer: erin.fischererin.fischer +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: tpch + labels: + trino: trino +spec: + connector: + tpch: {} +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: tpcds + labels: + trino: trino +spec: + connector: + tpcds: {} +--- +# We need to create the TrinoCluster last, so that the ConfigMaps/Secrets it mounts are already +# existing to prevent unnecessary Pod restarts. +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: my-trino +spec: + image: +{% if test_scenario['values']['trino-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino + authentication: + - authenticationClass: trino-users +# authorization: +# opa: +# configMapName: opa +# package: trino +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + workers: + config: + gracefulShutdownTimeout: 5s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/data-hub-resource-info/21-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/21-assert.yaml new file mode 100644 index 00000000..de4876c8 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/21-assert.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# Metadata-only ingestion of a single schema is quick, but the image is large and the first +# query can wait on a worker; give it headroom. Surface the logs regardless of outcome. +timeout: 600 +commands: + - script: | + if kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=590s job/datahub-ingest-trino; then + kubectl -n "$NAMESPACE" logs job/datahub-ingest-trino --tail=40 + else + echo "Trino ingestion Job did not complete:" + kubectl -n "$NAMESPACE" logs job/datahub-ingest-trino --tail=80 || true + exit 1 + fi diff --git a/tests/templates/kuttl/data-hub-resource-info/21-scrape-trino.yaml b/tests/templates/kuttl/data-hub-resource-info/21-scrape-trino.yaml new file mode 100644 index 00000000..0ea4a61f --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/21-scrape-trino.yaml @@ -0,0 +1,148 @@ +--- +# DataHub ingestion job: scrapes Trino catalog metadata into DataHub GMS so that the +# resource-info-fetcher (RIF) can resolve Trino catalogs/schemas/tables by URN. +# +# Scope is intentionally narrow to keep the run fast and deterministic: +# - `database: tpch` -> only the tpch catalog (the DataHub trino source takes a +# single catalog per run; tpcds is left out). +# - `schema_pattern.allow: sf1` -> only the sf1 schema (TPC-H exposes tiny/sf1/sf100/...; +# sf1 is the smallest real scale factor and has the 8 +# standard tables customer/orders/lineitem/...). +# +# `env: PROD` matters: the SQL ingestion source sets `backcompat_env_as_instance`, so PROD lands +# in the container key's `instance` field. This must match the OpaCluster's `resourceInfo` env +# (10-install-opa) and the URN derivation in resource_to_urn_mapping.rs, or RIF lookups miss. +apiVersion: v1 +kind: ConfigMap +metadata: + name: datahub-trino-recipe +data: + recipe.yaml: | + source: + type: trino + config: + # Must be an FQDN present in Trino's server-cert SAN (see TLS note on the Job below). + # ${TRINO_HOST} is expanded by DataHub from the ingest container's env. + host_port: '${TRINO_HOST}:8443' + database: tpch + username: datahub + password: datahubdatahub + env: PROD + schema_pattern: + allow: ["^sf1$"] + include_views: true + include_tables: true + profiling: + enabled: false + options: + connect_args: + http_scheme: https + sink: + type: datahub-rest + config: + server: 'http://datahub-datahub-gms:8080' + # Same DataHub PAT the RIF sidecar uses, minted in 04-load-sample-data. + token: '${DATAHUB_GMS_TOKEN}' +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: datahub-ingest-trino +spec: + # Trino may briefly reject queries right after the cluster reports available (worker still + # registering); a couple of retries lets the ingestion self-heal. + backoffLimit: 3 + template: + spec: + restartPolicy: OnFailure + initContainers: + # Trino serves HTTPS with a cert signed by the SDP `tls` SecretClass CA. DataHub's trino + # source (v1.6.0) silently ignores `verify: false`, and PYTHONHTTPSVERIFY=0 is overridden + # by the Trino Python client's session-level verify. The reliable fix is to trust the CA: + # extract it from the secret-operator PKCS12 truststore and point Python's HTTP libs at it + # via SSL_CERT_FILE / REQUESTS_CA_BUNDLE below. + - name: extract-ca + image: acryldata/datahub-ingestion:v1.6.0 + # secret-operator writes truststore.p12 as root:0 mode 0640, so reading it needs root. + # This container only reads the store and writes /trust/ca.crt. + securityContext: + runAsUser: 0 + command: + - /bin/bash + - -c + - | + set -euo pipefail + # autoTls issues PKCS12 stores with an EMPTY password and RC2-40-CBC encryption; + # OpenSSL 3.x needs `-legacy` to read RC2. `pass:` == empty password. + openssl pkcs12 -legacy \ + -in /stackable/tls/truststore.p12 \ + -passin pass: \ + -nokeys \ + -out /trust/ca.crt + chmod 0644 /trust/ca.crt + volumeMounts: + - name: tls + mountPath: /stackable/tls + - name: trust + mountPath: /trust + containers: + - name: ingest + image: acryldata/datahub-ingestion:v1.6.0 + command: ["datahub", "ingest", "-c", "/recipe/recipe.yaml"] + env: + # Build the coordinator FQDN (a cert SAN) without templating the recipe: the pod's + # namespace comes from the downward API, then $(POD_NAMESPACE) is interpolated here. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: TRINO_HOST + value: "my-trino-coordinator.$(POD_NAMESPACE).svc.cluster.local" + # Make every Python HTTP request trust the extracted SDP CA. + - name: SSL_CERT_FILE + value: /trust/ca.crt + - name: REQUESTS_CA_BUNDLE + value: /trust/ca.crt + - name: DATAHUB_GMS_TOKEN + valueFrom: + secretKeyRef: + name: datahub-rif-credentials + key: token + - name: DATAHUB_TELEMETRY_ENABLED + value: "false" + volumeMounts: + - name: recipe + mountPath: /recipe + - name: trust + mountPath: /trust + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + volumes: + - name: recipe + configMap: + name: datahub-trino-recipe + - name: trust + emptyDir: {} + # secret-operator ephemeral volume: PKCS12 truststore from the same `tls` CA that signs + # Trino's server cert. No password annotation -> empty PKCS12 password (see extract-ca). + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/format: tls-pkcs12 + secrets.stackable.tech/scope: pod + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem diff --git a/tests/templates/kuttl/data-hub-resource-info/22-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/22-assert.yaml new file mode 100644 index 00000000..72ec8664 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/22-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# Includes a wait for data-product membership edges to index, so allow some headroom. +timeout: 360 +commands: + - script: | + if kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=350s job/datahub-create-metadata; then + kubectl -n "$NAMESPACE" logs job/datahub-create-metadata --tail=40 + else + echo "datahub-create-metadata Job did not complete:" + kubectl -n "$NAMESPACE" logs job/datahub-create-metadata --tail=80 || true + exit 1 + fi diff --git a/tests/templates/kuttl/data-hub-resource-info/22-create-metadata.yaml b/tests/templates/kuttl/data-hub-resource-info/22-create-metadata.yaml new file mode 100644 index 00000000..05b3d574 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/22-create-metadata.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # Ship the metadata-creation script into the cluster as a ConfigMap (keeps + # create-datahub-metadata.py a real, lintable file instead of inlining Python into YAML). + # Idempotent so re-runs are safe. The Trino datasets/containers it targets already exist: + # kuttl only advances here once 21-assert confirmed the ingestion Job completed. + - script: | + kubectl -n "$NAMESPACE" create configmap datahub-metadata-script \ + --from-file=create-datahub-metadata.py \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: datahub-create-metadata +spec: + backoffLimit: 2 + template: + spec: + restartPolicy: Never + containers: + - name: create-metadata + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + command: ["python", "/scripts/create-datahub-metadata.py"] + env: + # Same DataHub PAT the RIF sidecar uses, minted in 04-load-sample-data. + - name: DATAHUB_GMS_TOKEN + valueFrom: + secretKeyRef: + name: datahub-rif-credentials + key: token + volumeMounts: + - name: script + mountPath: /scripts + resources: + requests: + cpu: 200m + memory: 128Mi + limits: + cpu: "1" + memory: 256Mi + volumes: + - name: script + configMap: + name: datahub-metadata-script diff --git a/tests/templates/kuttl/data-hub-resource-info/30-assert.yaml b/tests/templates/kuttl/data-hub-resource-info/30-assert.yaml new file mode 100644 index 00000000..8c750c5f --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/30-assert.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# The Job's exit code is the verdict. Wait for it to complete and surface its logs either +# way (on success they show the fetcher response; on failure, the assertion error). +timeout: 310 +commands: + - script: | + if kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=300s job/test-resource-info; then + kubectl -n "$NAMESPACE" logs job/test-resource-info + else + echo "resource-info assertion Job did not complete:" + kubectl -n "$NAMESPACE" logs job/test-resource-info || true + exit 1 + fi diff --git a/tests/templates/kuttl/data-hub-resource-info/20-run-test.yaml b/tests/templates/kuttl/data-hub-resource-info/30-run-test.yaml similarity index 100% rename from tests/templates/kuttl/data-hub-resource-info/20-run-test.yaml rename to tests/templates/kuttl/data-hub-resource-info/30-run-test.yaml diff --git a/tests/templates/kuttl/data-hub-resource-info/create-datahub-metadata.py b/tests/templates/kuttl/data-hub-resource-info/create-datahub-metadata.py new file mode 100644 index 00000000..20849f28 --- /dev/null +++ b/tests/templates/kuttl/data-hub-resource-info/create-datahub-metadata.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python +"""Populate DataHub with the test users, groups, tags, domains, data products and assignments +that the resource-info-fetcher (RIF) test asserts on. + +Everything is attached to the Trino `tpch.sf1` catalog/schema/tables ingested in 21-scrape-trino. +Writes go to the GMS OpenAPI v3 entity endpoint as UPSERTs (createIfNotExists=false), so the +script is idempotent / re-runnable. async=false makes a rejected write fail loudly. + +The model (see the RIF response contract in rust/resource-info-fetcher/src/backend/data_hub): + - 3 groups own the two halves of the TPC-H model plus the shared reference layer. + - 5 users are members of those groups. + - tag `pii` on customer+supplier, `public` on nation+region (rest untagged for contrast). + - domain Sales (customer/orders/lineitem) and Supply Chain (supplier/part/partsupp). + - data products Order Analytics (Sales) and Supplier 360 (Supply Chain). + - ownership at catalog + schema (technical) and table (business) level; the schema mixes a + group *and* a user owner so the RIF's owner-by-type resolution is fully exercised. +""" + +import os +import sys +import time + +import requests + +GMS = os.environ.get("GMS", "http://datahub-datahub-gms:8080") +TOKEN = os.environ["DATAHUB_GMS_TOKEN"] +SESSION = requests.Session() +SESSION.headers.update({"Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"}) + +BUSINESS = "urn:li:ownershipType:__system__business_owner" +TECHNICAL = "urn:li:ownershipType:__system__technical_owner" + +# Deterministic URNs from the Trino ingestion (platform=trino, env=PROD -> instance=PROD). +# These match resource_to_urn_mapping.rs and were verified against a live DataHub. +CATALOG = "urn:li:container:6a39142fc39af8ec4ec5340eb21c1dee" # tpch +SCHEMA = "urn:li:container:727821ddae4cbef3856d53190f82489c" # tpch.sf1 + + +def dataset(table): + return f"urn:li:dataset:(urn:li:dataPlatform:trino,tpch.sf1.{table},PROD)" + + +def upsert(entity_type, objects): + """UPSERT a list of {urn, :{value:...}} objects to /openapi/v3/entity/.""" + resp = SESSION.post( + f"{GMS}/openapi/v3/entity/{entity_type}", + params={"async": "false", "createIfNotExists": "false"}, + json=objects, + ) + if not resp.ok: + sys.exit(f"ERROR upserting {entity_type}: HTTP {resp.status_code}: {resp.text}") + + +def ownership_aspect(owners): + """Build an `ownership` aspect value from a list of (owner_urn, ownership_type_urn) pairs.""" + owner_types = {} + for owner_urn, type_urn in owners: + owner_types.setdefault(type_urn, []).append(owner_urn) + return { + "value": { + "owners": [ + { + "owner": owner_urn, + "typeUrn": type_urn, + "type": "NONE", + "source": {"type": "MANUAL"}, + } + for owner_urn, type_urn in owners + ], + "ownerTypes": owner_types, + "lastModified": {"actor": "urn:li:corpuser:datahub", "time": 0}, + } + } + + +def wait_for_health(): + for attempt in range(60): + try: + if SESSION.get(f"{GMS}/health", timeout=10).ok: + return + except requests.RequestException: + pass + print(f" GMS not ready yet (attempt {attempt + 1}/60), retrying in 5s") + time.sleep(5) + sys.exit("GMS did not become healthy in time") + + +# --- The test fixture ---------------------------------------------------------------------------- + +# group id -> display name +GROUPS = { + "sales-analytics": "Sales Analytics", + "procurement": "Procurement", + "data-platform": "Data Platform", +} + +# username -> (full name, group id) +USERS = { + "alice.turner": ("Alice Turner", "sales-analytics"), + "bob.ramirez": ("Bob Ramirez", "sales-analytics"), + "carla.nowak": ("Carla Nowak", "procurement"), + "david.okoye": ("David Okoye", "procurement"), + "erin.fischer": ("Erin Fischer", "data-platform"), +} + +# tag id -> display name +TAGS = {"pii": "PII", "public": "Public"} + +# domain id -> (display name, description) +DOMAINS = { + "sales": ("Sales", "Sales and order data"), + "supply-chain": ("Supply Chain", "Suppliers, parts and procurement data"), +} + +# data product id -> (display name, description, domain id, [asset tables]) +DATA_PRODUCTS = { + "order-analytics": ( + "Order Analytics", + "Curated order analytics datasets", + "sales", + ["customer", "orders", "lineitem"], + ), + "supplier-360": ( + "Supplier 360", + "Supplier, part and procurement datasets", + "supply-chain", + ["supplier", "part", "partsupp"], + ), +} + +# table -> (tag ids, domain id or None, [(owner urn, ownership type urn)]) +TABLES = { + "customer": (["pii"], "sales", [("urn:li:corpGroup:sales-analytics", BUSINESS)]), + "orders": ([], "sales", [("urn:li:corpGroup:sales-analytics", BUSINESS)]), + "lineitem": ([], "sales", [("urn:li:corpGroup:sales-analytics", BUSINESS)]), + "supplier": (["pii"], "supply-chain", [("urn:li:corpGroup:procurement", BUSINESS)]), + "part": ([], "supply-chain", [("urn:li:corpGroup:procurement", BUSINESS)]), + "partsupp": ([], "supply-chain", [("urn:li:corpGroup:procurement", BUSINESS)]), + "nation": (["public"], None, [("urn:li:corpGroup:data-platform", TECHNICAL)]), + "region": (["public"], None, [("urn:li:corpGroup:data-platform", TECHNICAL)]), +} + + +def main(): + print("==> Waiting for DataHub GMS to be healthy") + wait_for_health() + + print("==> Groups") + upsert( + "corpgroup", + [ + { + "urn": f"urn:li:corpGroup:{gid}", + "corpGroupInfo": { + "value": { + "displayName": name, + "description": f"{name} team", + "admins": [], + "members": [], + "groups": [], + } + }, + } + for gid, name in GROUPS.items() + ], + ) + + print("==> Users") + upsert( + "corpuser", + [ + { + "urn": f"urn:li:corpuser:{username}", + "corpUserInfo": { + "value": { + "active": True, + "displayName": full_name, + "fullName": full_name, + "email": f"{username}@example.com", + } + }, + "corpUserEditableInfo": {"value": {"email": f"{username}@example.com"}}, + "groupMembership": {"value": {"groups": [f"urn:li:corpGroup:{gid}"]}}, + } + for username, (full_name, gid) in USERS.items() + ], + ) + + print("==> Tags") + upsert( + "tag", + [ + {"urn": f"urn:li:tag:{tid}", "tagProperties": {"value": {"name": name}}} + for tid, name in TAGS.items() + ], + ) + + print("==> Domains") + upsert( + "domain", + [ + { + "urn": f"urn:li:domain:{did}", + "domainProperties": {"value": {"name": name, "description": desc}}, + } + for did, (name, desc) in DOMAINS.items() + ], + ) + + print("==> Data products (assets create the DataProductContains edges the RIF reads)") + upsert( + "dataproduct", + [ + { + "urn": f"urn:li:dataProduct:{pid}", + "dataProductProperties": { + "value": { + "name": name, + "description": desc, + "assets": [{"destinationUrn": dataset(t)} for t in tables], + } + }, + "domains": {"value": {"domains": [f"urn:li:domain:{did}"]}}, + } + for pid, (name, desc, did, tables) in DATA_PRODUCTS.items() + ], + ) + + print("==> Table tags / domains / ownership") + dataset_objects = [] + for table, (tags, did, owners) in TABLES.items(): + obj = {"urn": dataset(table), "ownership": ownership_aspect(owners)} + if tags: + obj["globalTags"] = {"value": {"tags": [{"tag": f"urn:li:tag:{t}"} for t in tags]}} + if did: + obj["domains"] = {"value": {"domains": [f"urn:li:domain:{did}"]}} + dataset_objects.append(obj) + upsert("dataset", dataset_objects) + + print("==> Catalog / schema ownership") + upsert( + "container", + [ + {"urn": CATALOG, "ownership": ownership_aspect([("urn:li:corpGroup:data-platform", TECHNICAL)])}, + # The schema deliberately mixes a group and a user owner (both technical). + { + "urn": SCHEMA, + "ownership": ownership_aspect( + [ + ("urn:li:corpGroup:data-platform", TECHNICAL), + ("urn:li:corpuser:erin.fischer", TECHNICAL), + ] + ), + }, + ], + ) + + # DataProductContains edges are graph-indexed asynchronously (unlike the aspects above), so + # wait until at least one asset per data product resolves the incoming edge. This makes the + # metadata state deterministic for the RIF assertions in the next step. + print("==> Waiting for data-product membership edges to index") + for pid, (_, _, _, tables) in DATA_PRODUCTS.items(): + probe = dataset(tables[0]) + query = ( + '{ entity(urn: "%s") { relationships(input: {types: ["DataProductContains"], ' + 'direction: INCOMING, count: 10}) { total } } }' % probe + ) + for attempt in range(24): + resp = SESSION.post(f"{GMS}/api/graphql", json={"query": query}, timeout=20) + total = ( + resp.json().get("data", {}).get("entity", {}).get("relationships", {}).get("total", 0) + if resp.ok + else 0 + ) + if total: + print(f" {pid}: edge indexed") + break + print(f" {pid}: not indexed yet (attempt {attempt + 1}/24), retrying in 5s") + time.sleep(5) + else: + sys.exit(f"data product {pid} membership edge never indexed") + + print("==> Successfully created all DataHub users, groups, tags, domains, data products") + + +if __name__ == "__main__": + main() diff --git a/tests/templates/kuttl/data-hub-resource-info/test-regorule.py b/tests/templates/kuttl/data-hub-resource-info/test-regorule.py index faa3ef65..d10f7a2a 100644 --- a/tests/templates/kuttl/data-hub-resource-info/test-regorule.py +++ b/tests/templates/kuttl/data-hub-resource-info/test-regorule.py @@ -29,6 +29,70 @@ ] +BUSINESS = "urn:li:ownershipType:__system__business_owner" +TECHNICAL = "urn:li:ownershipType:__system__technical_owner" + +# Trino resources resolved via the Stackable abstraction (stacklet + coordinates -> URN). The +# metadata is created in 22-create-metadata. These exercise the full RIF response - tags, domain, +# data products and owners grouped by ownership type - across the catalog/schema/table levels, and +# prove the container-URN derivation in resource_to_urn_mapping.rs (catalog/schema URNs are opaque +# GUIDs, so a wrong derivation would resolve to a different - empty - entity). +TRINO_CASES = [ + { + "rule": "trinoCatalog", + "input": {"stacklet": "trino", "catalog": "tpch"}, + "urn": "urn:li:container:6a39142fc39af8ec4ec5340eb21c1dee", + "tags": [], + "domain": None, + "data_products": [], + "owners": {TECHNICAL: {"users": [], "groups": ["urn:li:corpGroup:data-platform"]}}, + }, + { + "rule": "trinoSchema", + "input": {"stacklet": "trino", "catalog": "tpch", "schema": "sf1"}, + "urn": "urn:li:container:727821ddae4cbef3856d53190f82489c", + "tags": [], + "domain": None, + "data_products": [], + # The schema deliberately mixes a group and a user owner (both technical). + "owners": { + TECHNICAL: { + "users": ["urn:li:corpuser:erin.fischer"], + "groups": ["urn:li:corpGroup:data-platform"], + } + }, + }, + { + "rule": "trinoTable", + "input": {"stacklet": "trino", "catalog": "tpch", "schema": "sf1", "table": "customer"}, + "urn": "urn:li:dataset:(urn:li:dataPlatform:trino,tpch.sf1.customer,PROD)", + "tags": ["urn:li:tag:pii"], + "domain": "urn:li:domain:sales", + "data_products": ["urn:li:dataProduct:order-analytics"], + "owners": {BUSINESS: {"users": [], "groups": ["urn:li:corpGroup:sales-analytics"]}}, + }, + { + "rule": "trinoTable", + "input": {"stacklet": "trino", "catalog": "tpch", "schema": "sf1", "table": "supplier"}, + "urn": "urn:li:dataset:(urn:li:dataPlatform:trino,tpch.sf1.supplier,PROD)", + "tags": ["urn:li:tag:pii"], + "domain": "urn:li:domain:supply-chain", + "data_products": ["urn:li:dataProduct:supplier-360"], + "owners": {BUSINESS: {"users": [], "groups": ["urn:li:corpGroup:procurement"]}}, + }, + { + # A reference table: tagged public, no domain, in no data product. + "rule": "trinoTable", + "input": {"stacklet": "trino", "catalog": "tpch", "schema": "sf1", "table": "nation"}, + "urn": "urn:li:dataset:(urn:li:dataPlatform:trino,tpch.sf1.nation,PROD)", + "tags": ["urn:li:tag:public"], + "domain": None, + "data_products": [], + "owners": {TECHNICAL: {"users": [], "groups": ["urn:li:corpGroup:data-platform"]}}, + }, +] + + def tag_urns(resource): return sorted(tag["urn"] for tag in resource["tags"]) @@ -42,6 +106,26 @@ def owner_urns(resource): return sorted(urns) +def domain_urn(resource): + # `domain` is optional (at most one per resource); None when unassigned. + return resource["domain"]["urn"] if resource["domain"] else None + + +def data_product_urns(resource): + return sorted(data_product["urn"] for data_product in resource["dataProducts"]) + + +def owners_by_type(resource): + # Reshape `owners` to {ownershipTypeUrn: {"users": [...], "groups": [...]}} for comparison. + return { + type_urn: { + "users": sorted(user["urn"] for user in bucket["users"]), + "groups": sorted(group["urn"] for group in bucket["groups"]), + } + for type_urn, bucket in resource["owners"].items() + } + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -92,6 +176,28 @@ def assert_resource(resource, urn, expected_tags, expected_owners): SAMPLE_OWNERS, ) + # 3) Exercise the Trino catalog/schema/table abstractions against the metadata from + # 22-create-metadata, checking the full RIF response (tags, domain, data products, owners). + for case in TRINO_CASES: + print(f"Checking {case['rule']}: {case['input']}") + resource = query(case["rule"], case["input"]) + assert resource["urn"] == case["urn"], ( + f"{case['rule']}: expected urn {case['urn']}, got {resource['urn']}" + ) + assert tag_urns(resource) == sorted(case["tags"]), ( + f"{case['urn']}: tags {tag_urns(resource)} != expected {sorted(case['tags'])}" + ) + assert domain_urn(resource) == case["domain"], ( + f"{case['urn']}: domain {domain_urn(resource)} != expected {case['domain']}" + ) + assert data_product_urns(resource) == sorted(case["data_products"]), ( + f"{case['urn']}: data products {data_product_urns(resource)} " + f"!= expected {sorted(case['data_products'])}" + ) + assert owners_by_type(resource) == case["owners"], ( + f"{case['urn']}: owners {owners_by_type(resource)} != expected {case['owners']}" + ) + print("Test successful!") diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 0c910d0e..8391a825 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -11,6 +11,11 @@ dimensions: # To use a custom image, add a comma and the full name after the product version # 0.67.1,oci.stackable.tech/sdp/opa:0.67.1-stackable0.0.0-dev - 1.16.2 + - name: trino-latest + values: + # To use a custom image, add a comma and the full name after the product version + # 481,oci.stackable.tech/sdp/trino:481-stackable0.0.0-dev + - "481" - name: keycloak values: - 23.0.1 @@ -71,6 +76,7 @@ tests: - name: data-hub-resource-info dimensions: - opa-latest + - trino-latest - data-hub - data-hub-prerequisites - openshift