diff --git a/Cargo.lock b/Cargo.lock index e9065d8f..db26bc29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2012,6 +2012,7 @@ dependencies = [ "dotenvy", "embed-resource", "futures-util", + "gateway", "gateway-config", "gateway-config-ui", "gateway-local", @@ -6071,6 +6072,7 @@ name = "shared-protocol" version = "0.3.0" dependencies = [ "async-trait", + "bytes", "futures-util", "gateway-config", "reqwest 0.12.28", diff --git a/Cargo.toml b/Cargo.toml index 9892cf57..ef7f533a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ repository = "https://github.com/cppalliance/promptforge" [workspace.dependencies] base64 = "0.22" +bytes = "1" promptforge = { path = "crates/promptforge", version = "0.3.0" } promptforge-core = { path = "crates/promptforge-core", version = "0.3.0" } promptforge-core-support = { path = "crates/promptforge-core-support", version = "0.3.0" } diff --git a/crates/gateway-config-ui/ui/src/components/settings-registry.ts b/crates/gateway-config-ui/ui/src/components/settings-registry.ts index 8f464ee9..a604f59f 100644 --- a/crates/gateway-config-ui/ui/src/components/settings-registry.ts +++ b/crates/gateway-config-ui/ui/src/components/settings-registry.ts @@ -160,6 +160,15 @@ function capabilities(): SettingDef[] { default: false, visibleWhen: (ctx) => ctx.value("thinking") !== "never" && ctx.value("thinking") != null, }, + { + key: "voices", + label: "Voices", + help: "The voices the model offers for speech synthesis.", + section: "capabilities", + type: "chips", + default: [], + visibleWhen: (ctx) => ctx.value("kind") === "speech", + }, ]; } diff --git a/crates/gateway-config-ui/ui/src/views/model-detail.test.mjs b/crates/gateway-config-ui/ui/src/views/model-detail.test.mjs index 0e09e405..630492ed 100644 --- a/crates/gateway-config-ui/ui/src/views/model-detail.test.mjs +++ b/crates/gateway-config-ui/ui/src/views/model-detail.test.mjs @@ -77,8 +77,8 @@ test("the local detail pane renders the registry sections with the model's value const kindSelect = root.querySelector(".field-row[data-key='kind'] .select"); assert.deepEqual( dropdownValues(root, "kind"), - ["chat", "embedding", "classifier"], - "the header kind dropdown offers the three model kinds", + ["chat", "embedding", "classifier", "speech"], + "the header kind dropdown offers the four model kinds", ); assert.equal(kindSelect.value, "chat"); @@ -442,6 +442,41 @@ test("Save PUTs the edited payload with untouched secrets redacted, then the pen ); }); +test("picking the speech kind reveals the voices chips and Save PUTs them", async () => { + const stub = fixtureStub(); + const { dom, root } = await bootApp({ key: "k", stub }); + navigate(dom, "#/remote/gpt-remote"); + await settle(); + + assert.equal( + root.querySelector(".field-row[data-key='voices']"), + null, + "a chat model hides the voices field", + ); + + dropdownValues(root, "kind"); + root.querySelector(".field-row[data-key='kind'] [data-value='speech']").click(); + await settle(); + + const voicesRow = root.querySelector(".field-row[data-key='voices']"); + assert.ok(voicesRow, "the speech kind reveals the voices field"); + const chips = voicesRow.querySelector(".chip-input input"); + chips.value = "nova"; + chips.dispatchEvent(new dom.window.KeyboardEvent("keydown", { key: "Enter" })); + await settle(); + + root.querySelector(".detail-save").click(); + await settle(); + + const put = stub.calls.find( + (call) => call.url.endsWith("/admin/config") && call.init.method === "PUT", + ); + assert.ok(put, "Save PUTs /admin/config"); + const model = JSON.parse(put.init.body).model.find((entry) => entry.name === "gpt-remote"); + assert.equal(model.kind, "speech", "the picked kind carries into the payload"); + assert.deepEqual(model.voices, ["nova"], "the added chip carries into the payload"); +}); + test("deleting a model confirms, PUTs the config without it, and returns to the list", async () => { const stub = fixtureStub(); const { dom, root } = await bootApp({ key: "k", stub }); diff --git a/crates/gateway-config-ui/ui/src/views/models-view.ts b/crates/gateway-config-ui/ui/src/views/models-view.ts index 63510fe0..570e9ec4 100644 --- a/crates/gateway-config-ui/ui/src/views/models-view.ts +++ b/crates/gateway-config-ui/ui/src/views/models-view.ts @@ -644,7 +644,7 @@ export function createModelsView(deps: ModelsViewDeps): ModelsView { help: "The workload this model serves.", section: "header", type: "dropdown", - options: ["chat", "embedding", "classifier"], + options: ["chat", "embedding", "classifier", "speech"], default: "chat", }), ); diff --git a/crates/gateway-config/src/config.rs b/crates/gateway-config/src/config.rs index ef5c6664..947192f9 100644 --- a/crates/gateway-config/src/config.rs +++ b/crates/gateway-config/src/config.rs @@ -531,8 +531,8 @@ impl fmt::Display for ToolDialect { } } -/// The workload a model serves: chat completions, embeddings, or -/// classification. +/// The workload a model serves: chat completions, embeddings, +/// classification, or speech synthesis. /// /// The kind scopes which configuration fields are meaningful: chat-only /// fields (for example `thinking`, `default_max_tokens`, @@ -550,6 +550,8 @@ pub enum ModelKind { Embedding, /// Classification / reranking. Classifier, + /// Speech synthesis (`POST /v1/audio/speech`). + Speech, } impl fmt::Display for ModelKind { @@ -558,6 +560,7 @@ impl fmt::Display for ModelKind { ModelKind::Chat => "chat", ModelKind::Embedding => "embedding", ModelKind::Classifier => "classifier", + ModelKind::Speech => "speech", }; f.write_str(spelling) } @@ -569,7 +572,8 @@ impl fmt::Display for ModelKind { /// reaches it. They are flattened into `[[model]]` and `[[local_model]]`, /// validated at load, and surfaced verbatim on `GET /v1/models` so clients /// can shape requests before sending them. The effort knobs are chat-only -/// and require a `thinking` mode other than `never`. +/// and require a `thinking` mode other than `never`; the `voices` list is +/// speech-only. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[non_exhaustive] pub struct Capabilities { @@ -599,6 +603,10 @@ pub struct Capabilities { /// chat kind only. Defaults to false. #[serde(default)] adaptive_thinking: bool, + /// The voices the model offers for speech synthesis; speech kind only. + /// Empty means the model exposes no fixed voice list. + #[serde(default)] + voices: Vec, } /// One model name and the backend it resolves to. diff --git a/crates/gateway-config/src/config/accessors.rs b/crates/gateway-config/src/config/accessors.rs index c72f67f1..9560fb70 100644 --- a/crates/gateway-config/src/config/accessors.rs +++ b/crates/gateway-config/src/config/accessors.rs @@ -880,7 +880,7 @@ impl ModelConfig { } /// Returns the workload this model serves: chat (the default), - /// embedding, or classifier. + /// embedding, classifier, or speech. /// /// # Examples /// ``` @@ -1221,7 +1221,7 @@ impl LocalModelConfig { } /// Returns the workload this model serves: chat (the default), - /// embedding, or classifier. + /// embedding, classifier, or speech. /// /// # Examples /// ``` @@ -1922,6 +1922,45 @@ impl Capabilities { pub fn adaptive_thinking(&self) -> bool { self.adaptive_thinking } + + /// Returns the voices the model offers for speech synthesis (empty when + /// the model exposes no fixed voice list). + /// + /// # Examples + /// ``` + /// # use gateway_config::Config; + /// # let toml = r#" + /// # config-version = 2 + /// # [server] + /// # bind = "127.0.0.1:8080" + /// # api_key = "secret" + /// # + /// # [[endpoint]] + /// # id = "e" + /// # protocol = "openai" + /// # base_url = "http://127.0.0.1:9" + /// # api_key = "" + /// # + /// # [[model]] + /// # name = "m" + /// # kind = "speech" + /// # description = "a model" + /// # context = 8192 + /// # upstream = "u" + /// # endpoints = ["e"] + /// # voices = ["alloy", "nova"] + /// # "#; + /// let config = Config::from_toml_str(toml)?; + /// assert_eq!( + /// config.models()[0].capabilities().voices(), + /// ["alloy", "nova"] + /// ); + /// # Ok::<(), gateway_config::ConfigError>(()) + /// ``` + #[must_use] + pub fn voices(&self) -> &[String] { + &self.voices + } } impl ToolsConfig { /// Returns the web-search tool configuration, or `None` when no diff --git a/crates/gateway-config/src/config/tests/serialize.rs b/crates/gateway-config/src/config/tests/serialize.rs index 7d6ef323..9eb79c9c 100644 --- a/crates/gateway-config/src/config/tests/serialize.rs +++ b/crates/gateway-config/src/config/tests/serialize.rs @@ -230,6 +230,7 @@ fn enums_round_trip_with_their_toml_spellings() { check(ModelKind::Chat, "chat"); check(ModelKind::Embedding, "embedding"); check(ModelKind::Classifier, "classifier"); + check(ModelKind::Speech, "speech"); check(SttRole::Interim, "interim"); check(SttRole::Final, "final"); } @@ -244,6 +245,7 @@ fn capabilities_round_trip_through_json() { effort_levels: vec!["low".to_owned(), "high".to_owned()], default_effort: Some("low".to_owned()), adaptive_thinking: true, + voices: vec!["alloy".to_owned()], }; let json = serde_json::to_value(&capabilities).expect("serializes"); let back: Capabilities = serde_json::from_value(json).expect("deserializes"); diff --git a/crates/gateway-config/src/config/tests/validation.rs b/crates/gateway-config/src/config/tests/validation.rs index e654b544..0c4cd2f3 100644 --- a/crates/gateway-config/src/config/tests/validation.rs +++ b/crates/gateway-config/src/config/tests/validation.rs @@ -1863,3 +1863,148 @@ fn rejects_nonchat_model_with_capability_effort_fields() { } } } + +#[test] +fn parses_speech_model_with_voices() { + // The speech kind and its voice catalog parse and re-serialize verbatim. + let toml = catalog_with_model_kind("speech", "voices = [\"alloy\", \"nova\"]"); + let config = Config::from_toml_str(&toml).unwrap(); + let model = &config.models()[0]; + assert_eq!(model.kind().to_string(), "speech"); + let json = serde_json::to_value(model).expect("serializes"); + assert_eq!(json["kind"], "speech"); + assert_eq!(json["voices"], serde_json::json!(["alloy", "nova"])); + + let toml = catalog_with_local_model_kind("speech", "voices = [\"alloy\"]"); + let config = Config::from_toml_str(&toml).unwrap(); + let json = serde_json::to_value(&config.local_models()[0]).expect("serializes"); + assert_eq!(json["kind"], "speech"); + assert_eq!(json["voices"], serde_json::json!(["alloy"])); +} + +#[test] +fn accepts_speech_model_with_empty_or_absent_voices() { + // An empty list stays valid; the route skips the voice check for it. + let toml = catalog_with_model_kind("speech", "voices = []"); + assert!(Config::parse_toml(&toml).is_ok()); + let toml = catalog_with_model_kind("speech", ""); + assert!(Config::parse_toml(&toml).is_ok()); + let toml = catalog_with_local_model_kind("speech", ""); + assert!(Config::parse_toml(&toml).is_ok()); +} + +#[test] +fn rejects_chat_only_fields_on_speech_models() { + // Speech is a non-chat kind: the chat-only discipline covers it. + for (field, extra) in [ + ("thinking", "thinking = \"always\""), + ("default_max_tokens", "default_max_tokens = 1024"), + ("tool_dialect", "tool_dialect = \"gemma3_tool_code\""), + ("effort_levels", "effort_levels = [\"low\"]"), + ("default_effort", "default_effort = \"low\""), + ("adaptive_thinking", "adaptive_thinking = true"), + ] { + let toml = catalog_with_model_kind("speech", extra); + match Config::parse_toml(&toml) { + Err(ConfigError::Validation(message)) => { + assert!( + message.contains(field), + "expected the error to name {field}: {message}" + ); + } + other => panic!("expected a validation error for {field}, got {other:?}"), + } + } + for (field, extra) in [ + ("thinking", "thinking = \"always\""), + ("chat_template_file", "chat_template_file = \"q.jinja\""), + ("effort_levels", "effort_levels = [\"low\"]"), + ("adaptive_thinking", "adaptive_thinking = true"), + ( + "speculative", + "[local_model.speculative]\ntype = \"draft-mtp\"\nsource = \"/models/d.gguf\"\nsha256 = \"b52f438017efaec5debf1c0d8be690571e212a07c312f1102bbce927258cfc32\"\ndraft_max = 7", + ), + ( + "multimodal_projector", + "[local_model.multimodal_projector]\nsource = \"/models/p.gguf\"\nsha256 = \"b52f438017efaec5debf1c0d8be690571e212a07c312f1102bbce927258cfc32\"", + ), + ] { + let toml = catalog_with_local_model_kind("speech", extra); + match Config::parse_toml(&toml) { + Err(ConfigError::Validation(message)) => { + assert!( + message.contains(field), + "expected the error to name {field}: {message}" + ); + } + other => panic!("expected a validation error for local {field}, got {other:?}"), + } + } +} + +#[test] +fn rejects_voices_on_non_speech_models() { + // `voices` is speech-only, symmetric with the chat-only discipline. + for kind in ["chat", "embedding", "classifier"] { + let toml = catalog_with_model_kind(kind, "voices = [\"alloy\"]"); + match Config::parse_toml(&toml) { + Err(ConfigError::Validation(message)) => { + assert!( + message.contains("voices"), + "expected the error to name voices: {message}" + ); + } + other => panic!("expected a validation error for kind {kind}, got {other:?}"), + } + let toml = catalog_with_local_model_kind(kind, "voices = [\"alloy\"]"); + match Config::parse_toml(&toml) { + Err(ConfigError::Validation(message)) => { + assert!( + message.contains("voices"), + "expected the error to name voices: {message}" + ); + } + other => panic!("expected a local validation error for kind {kind}, got {other:?}"), + } + } +} + +#[test] +fn rejects_empty_voice_entries() { + for extra in ["voices = [\"\"]", "voices = [\"alloy\", \"\", \"nova\"]"] { + let toml = catalog_with_model_kind("speech", extra); + match Config::parse_toml(&toml) { + Err(ConfigError::Validation(message)) => { + assert!( + message.contains("voices"), + "expected the error to name voices: {message}" + ); + } + other => panic!("expected a validation error for {extra:?}, got {other:?}"), + } + let toml = catalog_with_local_model_kind("speech", extra); + assert!( + matches!(Config::parse_toml(&toml), Err(ConfigError::Validation(_))), + "expected local_model {extra:?} to be rejected" + ); + } +} + +#[test] +fn rejects_duplicate_voices() { + let toml = catalog_with_model_kind("speech", "voices = [\"alloy\", \"nova\", \"alloy\"]"); + match Config::parse_toml(&toml) { + Err(ConfigError::Validation(message)) => { + assert!( + message.contains("alloy"), + "expected the error to name the duplicate: {message}" + ); + } + other => panic!("expected a validation error, got {other:?}"), + } + let toml = catalog_with_local_model_kind("speech", "voices = [\"alloy\", \"alloy\"]"); + assert!( + matches!(Config::parse_toml(&toml), Err(ConfigError::Validation(_))), + "expected local_model duplicate voices to be rejected" + ); +} diff --git a/crates/gateway-config/src/config/validate.rs b/crates/gateway-config/src/config/validate.rs index dc93ff4a..dc36d9c9 100644 --- a/crates/gateway-config/src/config/validate.rs +++ b/crates/gateway-config/src/config/validate.rs @@ -667,7 +667,8 @@ impl Config { /// /// `default_effort` requires a non-empty `effort_levels` and must name a /// listed level; the effort knobs are meaningless on a model that never -/// thinks; and `max_output` must fit the context window. +/// thinks; `max_output` must fit the context window; and `voices` entries +/// must be non-empty and unique. fn validate_capabilities( label: &str, name: &str, @@ -701,10 +702,24 @@ fn validate_capabilities( "{label} {name} max_output {max_output} exceeds context {context}" ))); } + let mut seen_voices = HashSet::new(); + for voice in &capabilities.voices { + if voice.is_empty() { + return Err(ConfigError::Validation(format!( + "{label} {name} voices entries must not be empty" + ))); + } + if !seen_voices.insert(voice.as_str()) { + return Err(ConfigError::Validation(format!( + "{label} {name} lists duplicate voice {voice}" + ))); + } + } Ok(()) } -/// Reject chat-only fields on a non-chat model kind. +/// Reject chat-only fields on a non-chat model kind and the speech-only +/// `voices` list on a non-speech kind. /// /// `thinking` and the capability effort knobs (`effort_levels`, /// `default_effort`, `adaptive_thinking`) are chat-only on every model type; @@ -719,6 +734,11 @@ fn validate_kind_scope( capabilities: &Capabilities, extra: &[(&str, bool)], ) -> Result<(), ConfigError> { + if kind != ModelKind::Speech && !capabilities.voices.is_empty() { + return Err(ConfigError::Validation(format!( + "{kind} {label} {name} must not set voices (speech-only)" + ))); + } if kind == ModelKind::Chat { return Ok(()); } diff --git a/crates/gateway-local/src/error.rs b/crates/gateway-local/src/error.rs index 25cdb7c2..1ae3e004 100644 --- a/crates/gateway-local/src/error.rs +++ b/crates/gateway-local/src/error.rs @@ -3,6 +3,8 @@ use std::io; use std::path::PathBuf; +use gateway_config::ModelKind; + /// A failure while downloading, verifying, or launching a local model. #[derive(Debug, thiserror::Error)] #[non_exhaustive] @@ -16,6 +18,17 @@ pub enum LocalError { arch: String, }, + /// The model's kind has no local `llama-server` launch mode. + /// + /// Speech models configure through `kind = "speech"` but no local speech + /// runtime exists yet. A kind added to `ModelKind` after the launch-mode + /// mapping lands here too, failing loudly instead of launching as chat. + #[error("local {kind} models are not yet supported")] + UnsupportedKind { + /// The model kind that cannot launch locally. + kind: ModelKind, + }, + /// Building the HTTP client failed. #[error("build HTTP client")] HttpClient(#[source] reqwest::Error), diff --git a/crates/gateway-local/src/runtime.rs b/crates/gateway-local/src/runtime.rs index f2aa787a..a95b3e36 100644 --- a/crates/gateway-local/src/runtime.rs +++ b/crates/gateway-local/src/runtime.rs @@ -352,9 +352,25 @@ fn provision_artifacts_impl( if token.is_cancelled() { return Err(LocalError::Cancelled); } - let (store, _server) = provision_server(config, progress, provision)?; + // Kind preflight (A6): every model's kind is checked before the shared + // server or any model provisions, so a profile with no launchable model + // fails without a single side effect. let mut failures = Vec::new(); + let mut launchable = Vec::new(); for local_model in config.local_models() { + match serve_mode_for(local_model.kind()) { + Ok(_) => launchable.push(local_model), + Err(error) => failures.push(LocalStartFailure { + model: local_model.name().to_owned(), + error, + }), + } + } + if launchable.is_empty() { + return Ok(failures); + } + let (store, _server) = provision_server(config, progress, provision)?; + for local_model in launchable { // Phase boundary: a cancelled command provisions no further models. if token.is_cancelled() { return Err(LocalError::Cancelled); @@ -456,13 +472,38 @@ fn start_impl( if token.is_some_and(CancellationToken::is_cancelled) { return Err(LocalError::Cancelled); } + + // Kind preflight (A6): every model's kind is checked before the shared + // server provisions, so a profile with no launchable model fails without + // a single provisioning side effect. + let mut launchable = Vec::new(); + let mut failures = Vec::new(); + for local_model in config.local_models() { + match serve_mode_for(local_model.kind()) { + Ok(_) => launchable.push(local_model), + Err(error) if policy == StartPolicy::FailFast => return Err(error), + Err(error) => failures.push(LocalStartFailure { + model: local_model.name().to_owned(), + error, + }), + } + } + if launchable.is_empty() { + return Ok(LocalStartOutcome { + runtime: LocalRuntime { + models: Vec::new(), + upstreams: Vec::new(), + cache_dir, + }, + failures, + }); + } let (store, server) = provision_server(config, progress, provision)?; let dominion_queues = dominion_queues(config); - let mut started_models = Vec::with_capacity(config.local_models().len()); - let mut failures = Vec::new(); + let mut started_models = Vec::with_capacity(launchable.len()); - for local_model in config.local_models() { + for local_model in launchable { // Phase boundary: a cancelled command starts no further models, and // the models already started drop with this in-progress outcome, // killing their children. @@ -711,8 +752,29 @@ fn resolve_admission( Ok(LocalAdmission { parallel, queue }) } -fn launch_options(model: &LocalModelConfig, parallel: u32) -> LaunchOptions { - LaunchOptions { +/// The `llama-server` serve mode for a model kind. +/// +/// The mapping is side-effect-free, so a caller can preflight an unsupported +/// kind before any provisioning side effect: a speech model has no local +/// runtime yet, and a kind added to `ModelKind` after this mapping fails +/// loudly instead of launching as a chat server. +fn serve_mode_for(kind: ModelKind) -> Result { + match kind { + ModelKind::Chat => Ok(ServeMode::Chat), + ModelKind::Embedding => Ok(ServeMode::Embeddings), + ModelKind::Classifier => Ok(ServeMode::Reranking), + ModelKind::Speech => Err(LocalError::UnsupportedKind { + kind: ModelKind::Speech, + }), + // `ModelKind` is `#[non_exhaustive]`: a kind added after this mapping + // fails loudly instead of launching as a chat server. + kind => Err(LocalError::UnsupportedKind { kind }), + } +} + +fn launch_options(model: &LocalModelConfig, parallel: u32) -> Result { + let serve_mode = serve_mode_for(model.kind())?; + Ok(LaunchOptions { ctx_size: model.context(), n_predict: model.n_predict(), parallel, @@ -722,16 +784,11 @@ fn launch_options(model: &LocalModelConfig, parallel: u32) -> LaunchOptions { cache_type_v: model.cache_type_v().to_owned(), think: !matches!(model.thinking(), ThinkingMode::Never), chat_template_file: None, - serve_mode: match model.kind() { - ModelKind::Embedding => ServeMode::Embeddings, - ModelKind::Classifier => ServeMode::Reranking, - // Chat (and any kind added after this mapping) launches with no flag. - _ => ServeMode::Chat, - }, + serve_mode, speculative: None, multimodal_projector: None, path_prefix: Vec::new(), - } + }) } fn launch_options_for( @@ -740,7 +797,7 @@ fn launch_options_for( model_path: &Path, admission: &LocalAdmission, ) -> Result { - let mut options = launch_options(model, admission.parallel); + let mut options = launch_options(model, admission.parallel)?; options.chat_template_file = resolve_chat_template_file(store, model, model_path)?; Ok(options) } @@ -1388,6 +1445,365 @@ context = 4096 ); } + #[test] + fn provision_artifacts_with_an_all_speech_profile_has_no_side_effects() { + use crate::testsupport::hex_sha256; + + // The kind preflight (A6) runs before the shared server or any model + // provisions: an all-speech profile collects one refusal per model + // and does nothing else - the server provisioner never runs and the + // cache directory is never even created. + let temp = tempfile::TempDir::new().expect("tempdir"); + let model_file = temp.path().join("tts.gguf"); + std::fs::write(&model_file, b"mock-tts-bytes").expect("write model"); + let cache_dir = temp.path().join("cache"); + let config = Config::from_toml_str(&format!( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:8081" +api_key = "t" + +[local] +cache_dir = '{}' + +[[local_model]] +name = "tts" +kind = "speech" +description = "a local speech model" +source = '{}' +sha256 = "{}" +context = 4096 +"#, + cache_dir.display(), + model_file.display(), + hex_sha256(b"mock-tts-bytes"), + )) + .expect("config"); + + let server_provisions = std::sync::atomic::AtomicUsize::new(0); + let failures = provision_artifacts_impl( + &config, + None, + &CancellationToken::new(), + |_store, _selection, _server| { + server_provisions.fetch_add(1, Ordering::Relaxed); + Ok(ProvisionedServer { + executable: PathBuf::from("mock-llama-server"), + path_prefix: Vec::new(), + }) + }, + ) + .expect("an unsupported kind is a per-model failure, not a fatal one"); + assert_eq!(failures.len(), 1); + assert_eq!(failures[0].model(), "tts"); + assert!( + matches!( + failures[0].error(), + LocalError::UnsupportedKind { + kind: ModelKind::Speech + } + ), + "the refusal names the speech kind: {:?}", + failures[0].error() + ); + assert_eq!( + server_provisions.load(Ordering::Relaxed), + 0, + "the shared server is never provisioned" + ); + assert!( + !cache_dir.exists(), + "the model store is never touched: the cache directory is never created" + ); + } + + #[test] + fn provision_artifacts_with_a_mixed_profile_provisions_only_supported_models() { + use crate::testsupport::hex_sha256; + + // A mixed profile keeps supported-model progress: the chat model's + // blob is verified into the cache while the speech model is refused + // as a per-model failure and never provisioned. + let temp = tempfile::TempDir::new().expect("tempdir"); + let chat_file = temp.path().join("chat.gguf"); + std::fs::write(&chat_file, b"mock-chat-bytes").expect("write chat model"); + let tts_file = temp.path().join("tts.gguf"); + std::fs::write(&tts_file, b"mock-tts-bytes").expect("write tts model"); + let cache_dir = temp.path().join("cache"); + let config = Config::from_toml_str(&format!( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:8081" +api_key = "t" + +[local] +cache_dir = '{}' + +[[local_model]] +name = "chat" +description = "a local chat model" +source = '{}' +sha256 = "{}" +context = 4096 + +[[local_model]] +name = "tts" +kind = "speech" +description = "a local speech model" +source = '{}' +sha256 = "{}" +context = 4096 +"#, + cache_dir.display(), + chat_file.display(), + hex_sha256(b"mock-chat-bytes"), + tts_file.display(), + hex_sha256(b"mock-tts-bytes"), + )) + .expect("config"); + + let server_provisions = std::sync::atomic::AtomicUsize::new(0); + let failures = provision_artifacts_impl( + &config, + None, + &CancellationToken::new(), + |_store, _selection, _server| { + server_provisions.fetch_add(1, Ordering::Relaxed); + Ok(ProvisionedServer { + executable: PathBuf::from("mock-llama-server"), + path_prefix: Vec::new(), + }) + }, + ) + .expect("a per-model refusal is not fatal"); + assert_eq!( + failures + .iter() + .map(LocalStartFailure::model) + .collect::>(), + ["tts"] + ); + assert!( + matches!( + failures[0].error(), + LocalError::UnsupportedKind { + kind: ModelKind::Speech + } + ), + "the refusal names the speech kind: {:?}", + failures[0].error() + ); + assert_eq!( + server_provisions.load(Ordering::Relaxed), + 1, + "the shared server provisions once for the supported model" + ); + let chat_key = artifacts::source_cache_key(&chat_file.to_string_lossy()); + assert!( + cache_dir + .join("markers") + .join(format!("{chat_key}.verified")) + .is_file(), + "the supported model's blob is verified into the cache" + ); + let tts_key = artifacts::source_cache_key(&tts_file.to_string_lossy()); + assert!( + !cache_dir + .join("markers") + .join(format!("{tts_key}.verified")) + .exists(), + "the refused speech model's blob is never provisioned" + ); + } + + #[test] + fn start_with_an_all_speech_profile_has_no_side_effects() { + use crate::testsupport::hex_sha256; + + // The kind preflight (A6) runs before the shared server provisions: + // an all-speech profile collects one refusal per model and does + // nothing else - the server provisioner never runs and the cache + // directory is never even created. + let temp = tempfile::TempDir::new().expect("tempdir"); + let model_file = temp.path().join("tts.gguf"); + std::fs::write(&model_file, b"mock-tts-bytes").expect("write model"); + let cache_dir = temp.path().join("cache"); + let config = Config::from_toml_str(&format!( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:8081" +api_key = "t" + +[local] +cache_dir = '{}' + +[[local_model]] +name = "tts" +kind = "speech" +description = "a local speech model" +source = '{}' +sha256 = "{}" +context = 4096 +"#, + cache_dir.display(), + model_file.display(), + hex_sha256(b"mock-tts-bytes"), + )) + .expect("config"); + + let server_provisions = std::sync::atomic::AtomicUsize::new(0); + let outcome = start_impl( + &config, + None, + &startup_interrupt_flag(), + None, + |_store, _selection, _server| { + server_provisions.fetch_add(1, Ordering::Relaxed); + Ok(ProvisionedServer { + executable: PathBuf::from("mock-llama-server"), + path_prefix: Vec::new(), + }) + }, + |_, _, _, _, _| panic!("a refused model never spawns"), + StartPolicy::KeepReady, + ) + .expect("a per-model refusal is not fatal under the partial policy"); + assert_eq!(outcome.runtime().child_count(), 0); + assert_eq!(outcome.failures().len(), 1); + assert!( + matches!( + outcome.failures()[0].error(), + LocalError::UnsupportedKind { + kind: ModelKind::Speech + } + ), + "the refusal names the speech kind: {:?}", + outcome.failures()[0].error() + ); + assert_eq!( + server_provisions.load(Ordering::Relaxed), + 0, + "the shared server is never provisioned" + ); + assert!( + !cache_dir.exists(), + "the model store is never touched: the cache directory is never created" + ); + } + + #[test] + fn start_with_a_mixed_profile_provisions_the_server_once() { + use crate::testsupport::hex_sha256; + + // A mixed profile keeps supported-model progress: the server + // provisions once, the embedding model's blob is verified into the + // cache and reaches the spawn, and the speech model is refused as a + // per-model failure whose blob is never provisioned. + let temp = tempfile::TempDir::new().expect("tempdir"); + let embed_file = temp.path().join("embed.gguf"); + std::fs::write(&embed_file, b"mock-embed-bytes").expect("write embed model"); + let tts_file = temp.path().join("tts.gguf"); + std::fs::write(&tts_file, b"mock-tts-bytes").expect("write tts model"); + let cache_dir = temp.path().join("cache"); + let config = Config::from_toml_str(&format!( + r#" +config-version = 2 +[server] +bind = "127.0.0.1:8081" +api_key = "t" +[local] +cache_dir = '{}' +[[local_model]] +name = "embed" +kind = "embedding" +description = "a local embedding model" +source = '{}' +sha256 = "{}" +context = 512 +[[local_model]] +name = "tts" +kind = "speech" +description = "a local speech model" +source = '{}' +context = 4096 +"#, + cache_dir.display(), + embed_file.display(), + hex_sha256(b"mock-embed-bytes"), + tts_file.display(), + )) + .expect("config"); + let server_provisions = std::sync::atomic::AtomicUsize::new(0); + let spawns = std::sync::atomic::AtomicUsize::new(0); + let outcome = start_impl( + &config, + None, + &startup_interrupt_flag(), + None, + |_store, _selection, _server| { + server_provisions.fetch_add(1, Ordering::Relaxed); + Ok(ProvisionedServer { + executable: PathBuf::from("mock-llama-server"), + path_prefix: Vec::new(), + }) + }, + |_, _, _, _, _| { + spawns.fetch_add(1, Ordering::Relaxed); + Err(LocalError::EarlyExit { + status: "the mock layout has no llama-server to spawn".to_owned(), + }) + }, + StartPolicy::KeepReady, + ) + .expect("per-model failures are not fatal under the partial policy"); + assert_eq!(outcome.runtime().child_count(), 0); + assert_eq!( + server_provisions.load(Ordering::Relaxed), + 1, + "the shared server provisions once for the supported model" + ); + assert_eq!( + spawns.load(Ordering::Relaxed), + 1, + "only the supported model reaches the spawn" + ); + assert!( + outcome.failures().iter().any(|failure| { + failure.model() == "tts" + && matches!( + failure.error(), + LocalError::UnsupportedKind { + kind: ModelKind::Speech + } + ) + }), + "the speech refusal is a per-model failure naming the kind: {:?}", + outcome.failures() + ); + let embed_key = artifacts::source_cache_key(&embed_file.to_string_lossy()); + let tts_key = artifacts::source_cache_key(&tts_file.to_string_lossy()); + assert!( + cache_dir + .join("markers") + .join(format!("{embed_key}.verified")) + .is_file(), + "the supported model's blob is verified into the cache" + ); + assert!( + !cache_dir + .join("markers") + .join(format!("{tts_key}.verified")) + .exists(), + "the refused speech model's blob is never provisioned" + ); + } + #[tokio::test] async fn parallel_field_feeds_parallel_arg_and_queue_limit() { // A local model with `parallel = 3` launches its child with @@ -1416,7 +1832,12 @@ parallel = 3 let admission = resolve_admission(&queues, model).expect("admission"); assert_eq!(admission.parallel, 3); - assert_eq!(launch_options(model, admission.parallel).parallel, 3); + assert_eq!( + launch_options(model, admission.parallel) + .expect("launch options") + .parallel, + 3 + ); let _first = admission.queue.admit("client").await.unwrap(); let _second = admission.queue.admit("client").await.unwrap(); @@ -1518,8 +1939,14 @@ context = 4096 .expect("config"); let embed = &config.local_models()[0]; let chat = &config.local_models()[1]; - assert_eq!(launch_options(embed, 1).serve_mode, ServeMode::Embeddings); - assert_eq!(launch_options(chat, 1).serve_mode, ServeMode::Chat); + assert_eq!( + launch_options(embed, 1).expect("launch options").serve_mode, + ServeMode::Embeddings + ); + assert_eq!( + launch_options(chat, 1).expect("launch options").serve_mode, + ServeMode::Chat + ); } #[test] @@ -1553,10 +1980,54 @@ context = 4096 let classifier = &config.local_models()[0]; let chat = &config.local_models()[1]; assert_eq!( - launch_options(classifier, 1).serve_mode, + launch_options(classifier, 1) + .expect("launch options") + .serve_mode, ServeMode::Reranking ); - assert_eq!(launch_options(chat, 1).serve_mode, ServeMode::Chat); + assert_eq!( + launch_options(chat, 1).expect("launch options").serve_mode, + ServeMode::Chat + ); + } + + #[test] + fn speech_kind_refuses_to_launch_as_chat() { + // A speech model has no `llama-server` serve mode: `launch_options` + // errors rather than falling through to the chat default, which is + // what the wildcard arm did before the mapping went fallible. + let config = Config::from_toml_str( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:8081" +api_key = "t" + +[[local_model]] +name = "tts" +kind = "speech" +description = "a local speech model" +source = "/models/tts.gguf" +context = 4096 +"#, + ) + .expect("config"); + let speech = &config.local_models()[0]; + let error = launch_options(speech, 1).expect_err("a speech model must not launch as chat"); + assert!( + matches!( + error, + LocalError::UnsupportedKind { + kind: ModelKind::Speech + } + ), + "the refusal names the speech kind: {error:?}" + ); + assert_eq!( + error.to_string(), + "local speech models are not yet supported" + ); } fn companion_config(body: &str) -> Config { @@ -1612,7 +2083,7 @@ sha256 = "{}" let temp = tempfile::TempDir::new().expect("tempdir"); let store = ArtifactStore::new(temp.path()).expect("store"); - let mut options = launch_options(model, 1); + let mut options = launch_options(model, 1).expect("launch options"); provision_companions(&store, model, &mut options, None).expect("provision companions"); let speculative = options.speculative.expect("speculative launch state"); @@ -1658,7 +2129,7 @@ draft_max = 2 let temp = tempfile::TempDir::new().expect("tempdir"); let store = ArtifactStore::new(temp.path()).expect("store"); let model = &mismatching.local_models()[0]; - let mut options = launch_options(model, 1); + let mut options = launch_options(model, 1).expect("launch options"); let error = provision_companions(&store, model, &mut options, None) .expect_err("pin mismatch must fail provisioning"); assert!(matches!(error, LocalError::DigestMismatch { .. })); @@ -1671,7 +2142,7 @@ source = "/definitely/not/a/real/mmproj.gguf" "#, ); let model = &missing.local_models()[0]; - let mut options = launch_options(model, 1); + let mut options = launch_options(model, 1).expect("launch options"); let error = provision_companions(&store, model, &mut options, None) .expect_err("a missing local source must fail provisioning"); assert!(matches!(error, LocalError::InvalidSource { .. })); @@ -1687,7 +2158,7 @@ source = "/definitely/not/a/real/mmproj.gguf" let model = &config.local_models()[0]; let temp = tempfile::TempDir::new().expect("tempdir"); let store = ArtifactStore::new(temp.path()).expect("store"); - let mut options = launch_options(model, 1); + let mut options = launch_options(model, 1).expect("launch options"); let before = options.clone(); provision_companions(&store, model, &mut options, None).expect("no companions"); assert_eq!(options, before); diff --git a/crates/gateway-routing/src/model.rs b/crates/gateway-routing/src/model.rs index 7f88d871..8dccb0e2 100644 --- a/crates/gateway-routing/src/model.rs +++ b/crates/gateway-routing/src/model.rs @@ -43,7 +43,8 @@ impl std::fmt::Debug for Endpoint { pub struct Model { /// The caller-facing model name. pub name: String, - /// The workload this model serves: chat, embedding, or classifier. + /// The workload this model serves: chat, embedding, classifier, or + /// speech. pub kind: ModelKind, /// Prose describing the model for catalog consumers. pub description: String, diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 43b55be8..240b1e1e 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -135,6 +135,13 @@ stt = ["dep:gateway-stt"] test-fixtures = ["local", "gateway-local/test-fixtures"] [dev-dependencies] +# The crate dev-depends on itself so every test target builds the library +# with test-fixtures enabled, without gate commands needing a --features +# flag: the integration suites drive the speech relay's test-scaled bounds. +# default-features = false keeps `--no-default-features` test invocations +# (the boot race jobs) headless; plain `cargo test` still gets the defaults +# from the command line's own implicit feature request. +gateway = { path = ".", features = ["test-fixtures"], default-features = false } base64.workspace = true hound.workspace = true # Encodes the generated test image for the live CUDA projector proof. diff --git a/crates/gateway/README.md b/crates/gateway/README.md index 589ea485..9d3e25e4 100644 --- a/crates/gateway/README.md +++ b/crates/gateway/README.md @@ -22,7 +22,7 @@ The config path comes from the `--config` flag or the `PROMPTFORGE_GATEWAY_CONFI A serving run logs to `gateway.log` in the `logs` directory under the state directory, rotating the previous run aside on startup and retaining five previous runs; every record crosses a redaction pass that masks bearer tokens, authorization and cookie header values, and `api_key` assignments before it reaches disk. When a run fails before it can serve, `promptforge-gateway diagnostics` prints a read-only JSON report of the state directory, the resolved config path, the current and retained log files, and the gateway discovery file - it never serves, rotates a log, parses a config, or prints secrets. -Configure endpoints, models, and credentials in the TOML catalog. The gateway accepts `POST /v1/chat/completions`, serves a model catalog at `GET /v1/models`, and, with the default-on `stt` feature, serves Realtime transcription at `WS /v1/realtime?intent=transcription` plus OpenAI-compatible multipart transcription at `POST /v1/audio/transcriptions`. +Configure endpoints, models, and credentials in the TOML catalog. The gateway accepts `POST /v1/chat/completions`, serves a model catalog at `GET /v1/models`, streams speech synthesis for `kind = "speech"` models at `POST /v1/audio/speech` with the voice union at `GET /v1/audio/voices`, and, with the default-on `stt` feature, serves Realtime transcription at `WS /v1/realtime?intent=transcription` plus OpenAI-compatible multipart transcription at `POST /v1/audio/transcriptions`. Embedding hosts use the library API instead of the binary: `spawn` starts the gateway on a dedicated thread with its own runtime and blocks until the listener is bound, returning a `GatewayHandle` that carries the bound URL and a graceful-shutdown switch (`url()`, `shutdown()`, `join()`). The bind is the readiness signal: the gateway serves health, status, progress, configuration, and every ready route before any model provisioning begins. Slow boot work - downloads, local model spawns, and the process's one speech-engine load - runs afterward as the queued boot profile command, visible on the status and progress endpoints. `Gateway::from_config` is the eager alternative reserved for tests and embedders that need a fully provisioned gateway in hand; every production path (`spawn`, `run`, the tray, the packaged binary) binds first and provisions through the queue. @@ -141,6 +141,34 @@ Clients may negotiate the PromptForge extension `item.input_audio_transcription. `GET /v1/models` advertises active physical speech names for batch calls and advertises `realtime-transcribe` only when the complete interim and final pair is ready. `GET /admin/status` reports generic `speech` facts: `configured`, `ready`, and `gpu`; while the boot load runs, the queue and progress surfaces report it like any other command. +### Speech synthesis models + +Speech synthesis models are ordinary remote catalog entries with `kind = "speech"`, served at `POST /v1/audio/speech` and backed by any OpenAI-shaped provider: + +```toml +[[endpoint]] +id = "together" +protocol = "openai" +base_url = "https://api.together.xyz/v1" +api_key = "${TOGETHER_API_KEY}" + +[[model]] +name = "orpheus" +kind = "speech" +description = "Orpheus 3B conversational speech synthesis" +upstream = "canopylabs/orpheus-3b-0.1-ft" +endpoints = ["together"] +context = 8192 +voices = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"] +``` + +| Field | Default | Meaning | +|---|---|---| +| `kind` | `chat` | `speech` routes the model to the speech endpoint; chat-only fields (`thinking`, the effort knobs, `default_max_tokens`, `tool_dialect`) are rejected at load. | +| `voices` | `[]` | Speech-only voice catalog. Entries must be non-empty and unique; an empty list means no fixed voice set, and the route accepts any voice. | + +The route speaks the OpenAI speech dialect: `model`, `input` (capped at 4096 characters), and `voice` (a plain name or the `{"id": "..."}` object form) are required; `response_format` (`mp3`, `opus`, `aac`, `flac`, `wav`, `pcm`) resolves an omitted field to `mp3` in the wire type, because OpenAI defaults to mp3 while Together defaults to wav; `speed`, `instructions`, and `stream_format` are optional, and every field the gateway does not name passes through verbatim. A voice outside the model's declared list is a 400 naming the valid voices, judged before queue admission; upstream 429 and 503 answers map to client-facing 429 `upstream_rate_limited` and 503 `upstream_unavailable` envelopes. The reply is the provider's audio bytes streamed through unread: the upstream `Content-Type` is forwarded (when missing or invalid, the framing selector first, so an SSE stream is labeled `text/event-stream` and never an audio type, then the requested format's MIME type) and `Content-Length` is never set. `stream_format` is forwarded to the provider, but the streaming scope is narrow in phase 1: Together's SSE mode (`stream=true`, SSE of base64 PCM rather than chunked binary) requires `response_format = "raw"`, which the wire type rejects, so phase-1 Together speech is non-streaming and the forwarding is forward-looking for SSE-capable OpenAI-compatible providers. `GET /v1/audio/voices` answers the active profile's deduplicated, sorted voice union as id-first `{"id", "name"}` entries, and `GET /v1/models` advertises the kind and voices verbatim. Local speech models are refused at launch: no local speech runtime exists yet. + ## Local model companions A chat `[[local_model]]` can declare two companions, each provisioned through the same pinned, digest-verified cache machinery as the main model: diff --git a/crates/gateway/src/error.rs b/crates/gateway/src/error.rs index ca91d875..146715f0 100644 --- a/crates/gateway/src/error.rs +++ b/crates/gateway/src/error.rs @@ -43,6 +43,18 @@ pub(crate) enum GatewayError { #[error("malformed request: {0}")] MalformedRequest(String), + /// The speech request's `voice` is not one of the model's catalog + /// voices. Checked at the route before queue admission, so the 400 + /// never burns a queue slot; the message names the valid voices. + #[non_exhaustive] + #[error("unknown voice {voice}; the model offers: {}", valid.join(", "))] + InvalidVoice { + /// The voice the request named. + voice: String, + /// The voices the model's catalog declares. + valid: Vec, + }, + /// A transport- or protocol-level failure from the upstream seam. The /// variants live in [`ProtocolError`]; the gateway wraps them so a route /// handler deals with one error type. @@ -59,6 +71,19 @@ pub(crate) enum GatewayError { #[error("queue rejected at capacity")] QueueRejected, + /// The upstream provider rate-limited a speech request. Maps to 429 so + /// an OpenAI client surfaces a retryable rate-limit error rather than + /// a server failure. Speech-only: every other route keeps the shared + /// [`ProtocolError`] mapping, so their envelopes stay bit-identical. + #[error("upstream rate limited")] + UpstreamRateLimited, + + /// The upstream provider was unavailable for a speech request. Maps to + /// 503 so a client can retry, rather than the shared mapping's 502. + /// Speech-only for the same reason as [`GatewayError::UpstreamRateLimited`]. + #[error("upstream unavailable")] + UpstreamUnavailable, + /// A bounded profile-switch drain expired and cancelled the request. #[error("request cancelled for profile switch")] RequestCancelled, @@ -318,6 +343,11 @@ impl GatewayError { "invalid_request_error", "malformed_request", ), + GatewayError::InvalidVoice { .. } => ( + StatusCode::BAD_REQUEST, + "invalid_request_error", + "invalid_voice", + ), GatewayError::Protocol(error) => error.classify(), GatewayError::QueueFull => ( StatusCode::SERVICE_UNAVAILABLE, @@ -329,6 +359,16 @@ impl GatewayError { "rate_limit_error", "queue_rejected", ), + GatewayError::UpstreamRateLimited => ( + StatusCode::TOO_MANY_REQUESTS, + "rate_limit_error", + "upstream_rate_limited", + ), + GatewayError::UpstreamUnavailable => ( + StatusCode::SERVICE_UNAVAILABLE, + "server_error", + "upstream_unavailable", + ), GatewayError::RequestCancelled => ( StatusCode::SERVICE_UNAVAILABLE, "server_error", @@ -485,6 +525,10 @@ mod tests { use std::error::Error as _; #[test] + #[expect( + clippy::too_many_lines, + reason = "a flat status table with one row per error variant" + )] fn gateway_error_classify_is_table_driven() { let cases: Vec<(GatewayError, (StatusCode, &str, &str))> = vec![ ( @@ -531,6 +575,33 @@ mod tests { "queue_rejected", ), ), + ( + GatewayError::InvalidVoice { + voice: "coral".to_owned(), + valid: vec!["alloy".to_owned(), "nova".to_owned()], + }, + ( + StatusCode::BAD_REQUEST, + "invalid_request_error", + "invalid_voice", + ), + ), + ( + GatewayError::UpstreamRateLimited, + ( + StatusCode::TOO_MANY_REQUESTS, + "rate_limit_error", + "upstream_rate_limited", + ), + ), + ( + GatewayError::UpstreamUnavailable, + ( + StatusCode::SERVICE_UNAVAILABLE, + "server_error", + "upstream_unavailable", + ), + ), ( GatewayError::switch_failed("build-routing", std::io::Error::other("x")), ( diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 04ad7bb0..7783cd9d 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -34,7 +34,10 @@ //! `GET /admin/chat-templates` family catalog and per-model effective //! resolution view (local builds), a bearer-authed //! `POST /v1/audio/transcriptions` OpenAI-compatible multipart STT endpoint -//! (stt builds), a bearer-authed +//! (stt builds), a bearer-authed `POST /v1/audio/speech` speech-synthesis +//! passthrough for `kind = "speech"` models streaming the upstream's audio +//! bytes unread, a bearer-authed `GET /v1/audio/voices` union catalog of +//! the speech models' configured voices, a bearer-authed //! `GET /admin/system` snapshot of host CPU, RAM, cache-drive, and GPU //! metrics, a bearer-authed `GET /admin/hf/search` and //! `GET /admin/hf/model/{repo}` proxy onto the Hugging Face hub API @@ -137,10 +140,11 @@ pub use gateway_config::{ use std::collections::BTreeSet; use std::sync::Arc; +use std::time::Duration; use axum::Json; -use axum::body::Body; -use axum::extract::State; +use axum::body::{Body, Bytes}; +use axum::extract::{FromRequest, Request, State}; use axum::http::HeaderValue; #[cfg(feature = "stt")] use axum::http::header::ORIGIN; @@ -160,6 +164,7 @@ use crate::local::LocalRuntime; use crate::routing::Routing; use crate::wire::{ ChatRequest, EmbeddingRequest, EmbeddingResponse, ModelInfo, RerankRequest, RerankResponse, + SpeechRequest, SpeechResponseFormat, SpeechStreamFormat, SpeechVoice, }; use gateway_config::ModelKind; #[cfg(feature = "web-search")] @@ -169,6 +174,7 @@ use gateway_stt::SpeechService; #[cfg(feature = "web-search")] use gateway_web_search::{WebSearchRequest, WebSearchResponse, WebSearchState}; use shared_progress::{EventState, OperationId, ProgressEvent, ProgressHub, ProgressTree}; +use shared_protocol::ProtocolError; /// Mutable live configuration held behind a lock so profile switches can swap /// routing and local children without rebuilding the axum router. @@ -497,6 +503,8 @@ pub(crate) fn build_router(state: AppState, bound: Option) .route("/v1/chat/completions", post(chat_completions)) .route("/v1/embeddings", post(embeddings)) .route("/v1/rerank", post(rerank)) + .route("/v1/audio/speech", post(audio_speech)) + .route("/v1/audio/voices", get(audio_voices)) .route("/v1/models", get(list_models)) .route("/health", get(health)) .route("/admin/profiles", get(admin_list_profiles)) @@ -935,6 +943,330 @@ async fn rerank( Ok(Json(response)) } +/// The speech route to a backend: the same auth, routing, kind guard, and +/// dominion queue admission as chat, for `kind = "speech"` models. +/// +/// Two deliberate departures from the other routes. First, auth runs before +/// body extraction: the handler takes the ungated [`Caller`] parts +/// extractor and a raw [`Request`], runs [`check_auth`], and only then +/// extracts `Json` by hand, so an unauthorized caller never +/// makes the gateway parse a body. Second, the reply is a byte passthrough, +/// not a typed relay: audio frames are opaque bytes the gateway cannot +/// re-validate per chunk, so the upstream body is forwarded unread - the +/// one departure from the gateway's typed-relay norm, the same trade +/// [`relay_sse`] documents for its own design. Because the response is a +/// long-lived byte stream, this route must never sit under a +/// `CompressionLayer` or a whole-request `TimeoutLayer`: both buffer or +/// kill long-lived streams. The stream runs under the bounded background +/// relay [`relay_audio`] documents, so every early end - a tripped bound, an +/// upstream failure, a profile-switch cancellation - fails the client's +/// body read rather than truncating it. +async fn audio_speech( + State(state): State, + caller: Caller, + request: Request, +) -> Result { + check_auth(&state, &caller).await?; + let Json(request) = Json::::from_request(request, &state) + .await + .map_err(|rejection| GatewayError::MalformedRequest(rejection.body_text()))?; + request + .validate() + .map_err(|reason| GatewayError::MalformedRequest(reason.to_owned()))?; + let in_flight = state.begin_inference().await; + let model = resolve_routed_model(&state, &request.model).await?; + crate::routing::require_kind(&model, ModelKind::Speech)?; + // A voice the model does not offer is a client error, so it is judged + // before queue admission: a 400 never burns a queue slot. + let voices = model.capabilities.voices(); + if !voices.is_empty() { + let requested = match &request.voice { + SpeechVoice::Name(name) => name.as_str(), + SpeechVoice::Id { id } => id.as_str(), + }; + if !voices.iter().any(|voice| voice == requested) { + return Err(GatewayError::InvalidVoice { + voice: requested.to_owned(), + valid: voices.to_vec(), + }); + } + } + let format = request.response_format; + let stream_format = request.stream_format; + let client_id = crate::queue::ClientId::from_header( + caller + .get(CLIENT_HEADER) + .and_then(|value| value.to_str().ok()), + ); + let permit = tokio::select! { + result = model.endpoint.queue.admit(client_id.as_str()) => result?, + () = in_flight.cancelled() => return Err(GatewayError::RequestCancelled), + }; + // A failure here is before the response starts, so it is consumed as a + // normal JSON error, never a stream that dies mid-flight. The speech + // path maps upstream 429/503 to its own envelope codes; every other + // error keeps the shared protocol mapping. + let streamed = tokio::select! { + result = model.endpoint.upstream.send_speech(request, &model.upstream_name) => { + result.map_err(|error| match error { + ProtocolError::UpstreamStatus { status: 429, .. } => { + GatewayError::UpstreamRateLimited + } + ProtocolError::UpstreamStatus { status: 503, .. } => { + GatewayError::UpstreamUnavailable + } + other => GatewayError::Protocol(other), + })? + }, + () = in_flight.cancelled() => return Err(GatewayError::RequestCancelled), + }; + Ok(relay_audio( + streamed, + format, + stream_format, + permit, + in_flight, + )) +} + +/// Total lifetime of one speech relay, headers to terminal end: a bound on +/// streams that would otherwise outlive every other budget by dripping. +#[cfg(not(any(test, feature = "test-fixtures")))] +const SPEECH_RELAY_TOTAL_LIFETIME: Duration = Duration::from_secs(60 * 60); +/// Test-scaled so the relay boundary tests run in milliseconds. +#[cfg(any(test, feature = "test-fixtures"))] +const SPEECH_RELAY_TOTAL_LIFETIME: Duration = Duration::from_secs(2); + +/// Ceiling on the response bytes one speech relay forwards. +#[cfg(not(any(test, feature = "test-fixtures")))] +const SPEECH_RELAY_BYTE_CEILING: u64 = 1 << 30; +/// Test-scaled so the relay boundary tests run in milliseconds. +#[cfg(any(test, feature = "test-fixtures"))] +const SPEECH_RELAY_BYTE_CEILING: u64 = 16 * 1024 * 1024; + +/// Per-read idle budget on the opened upstream body: a silent upstream ends +/// the stream within this window. Time-to-headers is never governed here; +/// it is the upstream layer's first-response budget. +#[cfg(not(any(test, feature = "test-fixtures")))] +const SPEECH_RELAY_UPSTREAM_IDLE: Duration = Duration::from_secs(30); +/// Test-scaled so the relay boundary tests run in milliseconds. +#[cfg(any(test, feature = "test-fixtures"))] +const SPEECH_RELAY_UPSTREAM_IDLE: Duration = Duration::from_millis(200); + +/// Budget for one blocked channel send: a downstream that stops reading +/// backpressures the bounded channel, and the relay ends the stream rather +/// than holding the permit forever. +#[cfg(not(any(test, feature = "test-fixtures")))] +const SPEECH_RELAY_DOWNSTREAM_BLOCKED: Duration = Duration::from_secs(60); +/// Test-scaled so the relay boundary tests run in milliseconds. +#[cfg(any(test, feature = "test-fixtures"))] +const SPEECH_RELAY_DOWNSTREAM_BLOCKED: Duration = Duration::from_millis(400); + +/// Data chunks buffered between the relay task and the HTTP body. The +/// channel is built one slot larger: the extra slot is reserved up front +/// for the terminal error item, so delivering it never waits on the +/// downstream. +const SPEECH_RELAY_CHANNEL_CAPACITY: usize = 4; + +/// Re-emit an upstream audio byte stream as the response body, holding the +/// dominion queue permit for the stream's lifetime. +/// +/// The relay is untyped on purpose: audio frames are opaque bytes, so the +/// chunks pass through unread rather than being validated and re-serialized +/// the way [`relay_sse`] re-emits chat chunks. The response forwards the +/// upstream `Content-Type` when present and otherwise falls back to the +/// requested format's MIME mapping (or `text/event-stream` when the framing +/// selector is `sse`); `Content-Length` is never set, so hyper emits +/// `Transfer-Encoding: chunked`. +/// +/// The forwarding runs in a spawned task that owns the upstream body, the +/// permit, and the cancellation guard, feeding a small bounded channel the +/// HTTP body consumes, so the permit's lifetime never depends on downstream +/// polling. Four named bounds cap the stream: [`SPEECH_RELAY_TOTAL_LIFETIME`], +/// [`SPEECH_RELAY_BYTE_CEILING`], [`SPEECH_RELAY_UPSTREAM_IDLE`], and +/// [`SPEECH_RELAY_DOWNSTREAM_BLOCKED`]. +/// +/// No error envelope can follow 200 plus audio bytes, so every terminal +/// path - a bound tripped, an upstream body error, a profile-switch +/// cancellation, the downstream gone - emits exactly one `Err` item into +/// the channel, then drops the permit and the guard: the client's body read +/// fails rather than seeing a clean EOF, the same fail-rather-than-truncate +/// trade [`relay_sse`] makes with its RequestCancelled envelope. Over the +/// wire a body-stream error aborts the response, so the item's message is +/// server-side diagnostics; the client observes a failed read. Only a +/// stream that ran to a clean upstream end inside every bound ends the body +/// without an error item. +fn relay_audio( + streamed: crate::upstream::StreamedAudio, + format: SpeechResponseFormat, + stream_format: Option, + permit: crate::queue::Permit, + in_flight: drain::InFlightGuard, +) -> Response { + let (tx, rx) = tokio::sync::mpsc::channel(SPEECH_RELAY_CHANNEL_CAPACITY + 1); + tokio::spawn(relay_speech_stream(streamed.body, tx, permit, in_flight)); + let relayed = futures_util::stream::unfold(rx, |mut rx| async move { + rx.recv().await.map(|item| (item, rx)) + }); + let mut response = Response::new(Body::from_stream(relayed)); + let content_type = if streamed.content_type.is_empty() { + speech_fallback_mime(format, stream_format) + } else { + HeaderValue::from_str(&streamed.content_type) + .unwrap_or_else(|_| speech_fallback_mime(format, stream_format)) + }; + response.headers_mut().insert(CONTENT_TYPE, content_type); + response +} + +/// The relay task behind [`relay_audio`]: reads the upstream body under the +/// idle and total-lifetime budgets, forwards each chunk under the +/// blocked-delivery budget, and on every terminal path emits exactly one +/// `Err` item through the reserved channel slot before returning, which +/// drops the upstream body, the permit, and the cancellation guard +/// together. A clean upstream end is the one exit with no error item. +async fn relay_speech_stream( + mut body: futures_util::stream::BoxStream<'static, Result>, + tx: tokio::sync::mpsc::Sender>, + permit: crate::queue::Permit, + in_flight: drain::InFlightGuard, +) { + use futures_util::StreamExt as _; + + // Reserve the terminal slot before the first send competes for the + // channel: the terminal error item is delivered even when every data + // slot is full. + let error_slot = tx + .clone() + .try_reserve_owned() + .unwrap_or_else(|_| unreachable!("a fresh channel always has capacity")); + let deadline = tokio::time::Instant::now() + SPEECH_RELAY_TOTAL_LIFETIME; + let mut total_bytes: u64 = 0; + let terminal: Option = 'relay: loop { + let item = tokio::select! { + item = tokio::time::timeout(SPEECH_RELAY_UPSTREAM_IDLE, body.next()) => item, + () = in_flight.cancelled() => { + break 'relay Some(relay_terminal("request cancelled for profile switch")); + } + () = tokio::time::sleep_until(deadline) => { + break 'relay Some(relay_terminal("speech relay total lifetime exceeded")); + } + () = tx.closed() => { + break 'relay Some(relay_terminal("speech relay downstream gone")); + } + }; + let chunk = match item { + Ok(Some(Ok(chunk))) => chunk, + // A clean upstream end inside every bound: the one exit with no + // error item. + Ok(None) => break 'relay None, + // The upstream's own mid-stream failure is the terminal item. + Ok(Some(Err(error))) => break 'relay Some(error), + Err(_idle) => { + break 'relay Some(relay_terminal("speech relay upstream idle")); + } + }; + total_bytes += u64::try_from(chunk.len()).unwrap_or(u64::MAX); + if total_bytes > SPEECH_RELAY_BYTE_CEILING { + break 'relay Some(relay_terminal("speech relay byte ceiling exceeded")); + } + let delivered = tokio::select! { + result = tokio::time::timeout(SPEECH_RELAY_DOWNSTREAM_BLOCKED, tx.send(Ok(chunk))) => { + result + } + () = in_flight.cancelled() => { + break 'relay Some(relay_terminal("request cancelled for profile switch")); + } + () = tokio::time::sleep_until(deadline) => { + break 'relay Some(relay_terminal("speech relay total lifetime exceeded")); + } + () = tx.closed() => { + break 'relay Some(relay_terminal("speech relay downstream gone")); + } + }; + match delivered { + Ok(Ok(())) => {} + // The downstream is gone mid-send or past the blocked budget. + Ok(Err(_closed)) => { + break 'relay Some(relay_terminal("speech relay downstream gone")); + } + Err(_elapsed) => { + break 'relay Some(relay_terminal("speech relay downstream blocked")); + } + } + }; + if let Some(error) = terminal { + // The reserved slot makes this send immediate; when the downstream + // is already gone the item is simply discarded. + let _ = error_slot.send(Err(error)); + } + // Explicit about the ownership the task exists for: the upstream body, + // the dominion permit, and the cancellation guard are released together + // on every exit, so no path can end the stream while holding the slot. + drop((body, permit, in_flight)); +} + +/// The relay's terminal condition as a transport-classified protocol error: +/// the request may have reached the provider, and the message is +/// server-side diagnostics (a body-stream error aborts the response, so the +/// client observes a failed read, never this text). +fn relay_terminal(message: &'static str) -> ProtocolError { + ProtocolError::transport(std::io::Error::other(message)) +} + +/// The `Content-Type` a speech response falls back to when the upstream +/// omits it or sends an invalid one: the framing selector first, so an SSE +/// stream is labeled `text/event-stream` and never an audio type, then the +/// requested format's MIME type (the OpenAI spellings). +fn speech_fallback_mime( + format: SpeechResponseFormat, + stream_format: Option, +) -> HeaderValue { + if matches!(stream_format, Some(SpeechStreamFormat::Sse)) { + return HeaderValue::from_static("text/event-stream"); + } + HeaderValue::from_static(match format { + SpeechResponseFormat::Mp3 => "audio/mpeg", + SpeechResponseFormat::Opus => "audio/ogg", + SpeechResponseFormat::Aac => "audio/aac", + SpeechResponseFormat::Flac => "audio/flac", + SpeechResponseFormat::Wav => "audio/wav", + SpeechResponseFormat::Pcm => "audio/pcm", + }) +} + +/// Bearer-authed union of the active profile's speech voices for host +/// bind: every `kind = "speech"` model's configured `voices`, deduplicated +/// and sorted, as id-first `{"id", "name"}` entries under +/// `{"voices": [...]}`. +/// +/// OpenAI has no voice-list route; the OpenAI-compatible ecosystem +/// (Kokoro-FastAPI, vLLM-Omni, Fish Audio) converged on this one, and +/// clients such as Open WebUI read the `id` key, so the entry shape is a +/// compatibility surface pinned by the integration suite. `name` mirrors +/// `id`: the catalog configures voices as bare strings with no separate +/// display name. +async fn audio_voices( + State(state): State, + caller: Caller, +) -> Result, GatewayError> { + check_auth(&state, &caller).await?; + let _publication = state.switch.lock().await; + let live = state.live.read().await; + let voices = live + .routing + .models() + .iter() + .filter(|model| model.kind == ModelKind::Speech) + .flat_map(|model| model.capabilities.voices().iter()) + .collect::>() + .into_iter() + .map(|voice| serde_json::json!({ "id": voice, "name": voice })) + .collect::>(); + drop(live); + Ok(Json(serde_json::json!({ "voices": voices }))) +} + /// Bearer-authed catalog of configured models for host bind. async fn list_models( State(state): State, @@ -1098,29 +1430,17 @@ async fn admin_status( .any(|model| model.kind() == kind) }; let routed = |kind: ModelKind| live.routing.models().iter().any(|model| model.kind == kind); - let endpoints = vec![ - endpoint_status( - "/v1/chat/completions", - "Chat completions", - configured(ModelKind::Chat), - routed(ModelKind::Chat), - command_active, - ), - endpoint_status( - "/v1/embeddings", - "Embeddings", - configured(ModelKind::Embedding), - routed(ModelKind::Embedding), - command_active, - ), - endpoint_status( - "/v1/rerank", - "Rerank", - configured(ModelKind::Classifier), - routed(ModelKind::Classifier), - command_active, - ), - ]; + let endpoints = [ + ("/v1/chat/completions", "Chat completions", ModelKind::Chat), + ("/v1/embeddings", "Embeddings", ModelKind::Embedding), + ("/v1/rerank", "Rerank", ModelKind::Classifier), + ("/v1/audio/speech", "Speech synthesis", ModelKind::Speech), + ] + .into_iter() + .map(|(path, name, kind)| { + endpoint_status(path, name, configured(kind), routed(kind), command_active) + }) + .collect::>(); #[cfg(feature = "stt")] let (endpoints, speech) = with_speech_endpoint(endpoints, state.speech.status(), command_active); @@ -1860,6 +2180,74 @@ mod transcription_auth_tests { } } +#[cfg(test)] +mod speech_auth_tests { + //! The speech route's auth ordering through the real router. The route + //! is unconditional, so these tests sit beside, not inside, the + //! stt-gated `transcription_auth_tests` module. + + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use gateway_config::Config; + use tower::ServiceExt; + + use crate::build_router; + use crate::test_support::app_state; + + fn state() -> crate::AppState { + let config = Config::from_toml_str( + "config-version = 2\n\ + [server]\nbind = \"127.0.0.1:0\"\napi_key = \"test-token\"\n\ + [workshop]\n", + ) + .expect("config parses"); + app_state(config, None) + } + + #[tokio::test] + async fn speech_checks_bearer_auth_before_json_extraction() { + let response = build_router(state(), None) + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/audio/speech") + .header("content-type", "application/json") + .body(Body::from("{not json")) + .expect("request builds"), + ) + .await + .expect("router answers"); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "auth refuses the request before its malformed body is extracted" + ); + } + + #[tokio::test] + async fn authenticated_malformed_json_uses_the_openai_error_envelope() { + let response = build_router(state(), None) + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/audio/speech") + .header("authorization", "Bearer test-token") + .header("content-type", "application/json") + .body(Body::from("{not json")) + .expect("request builds"), + ) + .await + .expect("router answers"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("body reads"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("body is JSON"); + assert_eq!(json["error"]["code"], "malformed_request"); + assert_eq!(json["error"]["type"], "invalid_request_error"); + } +} + #[cfg(test)] mod tray_status_tests { use gateway_config::Config; @@ -4707,7 +5095,7 @@ mod status_surface_tests { chat["provisioning"], false, "no command is running, so nothing provisions" ); - for path in ["/v1/embeddings", "/v1/rerank"] { + for path in ["/v1/embeddings", "/v1/rerank", "/v1/audio/speech"] { let entry = endpoints .iter() .find(|entry| entry["path"] == path) diff --git a/crates/gateway/src/model_info.rs b/crates/gateway/src/model_info.rs index 21426abe..d4ba38dd 100644 --- a/crates/gateway/src/model_info.rs +++ b/crates/gateway/src/model_info.rs @@ -45,7 +45,7 @@ pub(crate) struct CatalogModelsResponse { #[derive(Debug, Serialize)] #[serde(untagged)] pub(crate) enum CatalogModelInfo { - /// Existing chat, embedding, or classifier metadata. + /// Existing chat, embedding, classifier, or speech metadata. Inference(ModelInfo), /// Generic transcription metadata. #[cfg(feature = "stt")] diff --git a/crates/gateway/src/profile_switch.rs b/crates/gateway/src/profile_switch.rs index 483c16d9..9ceaeaab 100644 --- a/crates/gateway/src/profile_switch.rs +++ b/crates/gateway/src/profile_switch.rs @@ -25,6 +25,18 @@ use gateway_web_search::WebSearchState; const PREPARED_CREATE_ATTEMPTS: u64 = 16; /// Shared deadline for target staging and prior-runtime reconstruction. pub(super) const STAGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Deadline for the cut-over drain of in-flight requests before the switch +/// cancels the stragglers: long enough that a healthy request finishes on +/// its own, short enough that a switch cannot park forever behind one. +#[cfg(not(any(test, feature = "test-fixtures")))] +const INFERENCE_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +/// Test-scaled so a switch cancelling an open stream lands in milliseconds; +/// comfortably above the in-flight switch tests' ~100 ms release point and +/// below the speech relay's 2 s total-lifetime bound, so a cancellation +/// beats the relay's own deadline to the terminal error item. +#[cfg(any(test, feature = "test-fixtures"))] +const INFERENCE_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); static PERSISTENCE_NAMES: LazyLock u128>> = LazyLock::new(|| ProcessPreparationNames::new(std::process::id(), random_persistence_nonce)); @@ -1266,7 +1278,7 @@ impl OldRuntimes { async fn drain_inference(state: &AppState) { if !state .in_flight - .drain_or_cancel(std::time::Duration::from_secs(30)) + .drain_or_cancel(INFERENCE_DRAIN_TIMEOUT) .await { tracing::warn!( diff --git a/crates/gateway/tests/it/boot.rs b/crates/gateway/tests/it/boot.rs index 72c9c24a..9c500965 100644 --- a/crates/gateway/tests/it/boot.rs +++ b/crates/gateway/tests/it/boot.rs @@ -705,6 +705,19 @@ fn workshop_launch_lock_and_direct_launch_do_not_deadlock_or_double_boot() { /// The ordinary production binary has no compiled rendezvous hook: even /// environment names used by the feature-enabled fixture are inert. +/// +/// Retired from every current runner: the gateway's self dev-dependency +/// (`gateway = { path = ".", features = ["test-fixtures"], ... }`, added so +/// the speech relay suites get test-scaled bounds without a `--features` +/// flag) forces `test-fixtures` into every test-target build, so this +/// `not(test-fixtures)` test compiles out under plain `cargo test -p +/// gateway` just as it does under CI's `--all-features` and `--features +/// test-fixtures` invocations. The property it pins still matters - a +/// default-feature binary must ignore the rendezvous environment - so the +/// test stays. Anything that builds the gateway test targets without +/// `test-fixtures` re-enables it: a `cargo test -p gateway +/// --no-default-features`-shaped invocation once the self dev-dependency no +/// longer forces the feature in, or the dev-dependency's removal. #[cfg(not(feature = "test-fixtures"))] #[test] fn the_default_binary_ignores_test_rendezvous_environment() { diff --git a/crates/gateway/tests/it/chat.rs b/crates/gateway/tests/it/chat.rs index 5d2c9860..1844ed38 100644 --- a/crates/gateway/tests/it/chat.rs +++ b/crates/gateway/tests/it/chat.rs @@ -133,13 +133,22 @@ description = "a classifier model" context = 8192 upstream = "backend-model" endpoints = ["fake"] + +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] +voices = ["alloy"] "# ); let config = Config::from_toml_str(&toml).unwrap(); let gateway = Gateway::from_config(&config, ProfilesContext::default()).unwrap(); let gateway = TestServer::start(gateway).await; - for model in ["embed-model", "reranker"] { + for model in ["embed-model", "reranker", "tts-model"] { let response = send_within( reqwest::Client::new() .post(format!("http://{}/v1/chat/completions", gateway.addr)) diff --git a/crates/gateway/tests/it/embeddings.rs b/crates/gateway/tests/it/embeddings.rs index 4b0252e3..711fe130 100644 --- a/crates/gateway/tests/it/embeddings.rs +++ b/crates/gateway/tests/it/embeddings.rs @@ -289,13 +289,22 @@ description = "a classifier model" context = 8192 upstream = "backend-model" endpoints = ["fake"] + +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] +voices = ["alloy"] "# ); let config = Config::from_toml_str(&toml).unwrap(); let gateway = Gateway::from_config(&config, ProfilesContext::default()).unwrap(); let gateway = TestServer::start(gateway).await; - for model in ["chat-model", "reranker"] { + for model in ["chat-model", "reranker", "tts-model"] { let response = send_within( reqwest::Client::new() .post(format!("http://{}/v1/embeddings", gateway.addr)) diff --git a/crates/gateway/tests/it/main.rs b/crates/gateway/tests/it/main.rs index 9259aa0e..cb0a07e9 100644 --- a/crates/gateway/tests/it/main.rs +++ b/crates/gateway/tests/it/main.rs @@ -8,9 +8,10 @@ //! //! The suite is split into cohesive area modules (IT-007): shared scaffolding //! lives in [`support`]; tests are grouped by surface into [`chat`], -//! [`embeddings`], [`rerank`], [`web_search`], [`queue`], [`profiles`], and -//! [`local`]. The `cuda` module holds the opt-in live CUDA proof, and the -//! Windows-only `icon` module pins the exe's embedded program icon. +//! [`embeddings`], [`rerank`], [`speech`], [`web_search`], [`queue`], +//! [`profiles`], and [`local`]. The `cuda` module holds the opt-in live CUDA +//! proof, and the Windows-only `icon` module pins the exe's embedded program +//! icon. #![expect( clippy::unwrap_used, clippy::expect_used, @@ -37,6 +38,7 @@ mod queue; mod realtime_stt; mod rerank; mod sidecar; +mod speech; mod surface; #[cfg(feature = "web-search")] mod web_search; diff --git a/crates/gateway/tests/it/rerank.rs b/crates/gateway/tests/it/rerank.rs index 5526c1f4..a21732d2 100644 --- a/crates/gateway/tests/it/rerank.rs +++ b/crates/gateway/tests/it/rerank.rs @@ -198,13 +198,22 @@ description = "an embedding model" context = 8192 upstream = "backend-model" endpoints = ["fake"] + +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] +voices = ["alloy"] "# ); let config = Config::from_toml_str(&toml).unwrap(); let gateway = Gateway::from_config(&config, ProfilesContext::default()).unwrap(); let gateway = TestServer::start(gateway).await; - for model in ["chat-model", "embed-model"] { + for model in ["chat-model", "embed-model", "tts-model"] { let response = send_within( reqwest::Client::new() .post(format!("http://{}/v1/rerank", gateway.addr)) diff --git a/crates/gateway/tests/it/speech.rs b/crates/gateway/tests/it/speech.rs new file mode 100644 index 00000000..71d4643a --- /dev/null +++ b/crates/gateway/tests/it/speech.rs @@ -0,0 +1,1945 @@ +//! Speech route: remote passthrough of opaque audio bytes, voice validation, +//! dominion queue admission, the kind guard, the upstream-error envelope +//! mapping, and the bounded background relay's terminal paths (byte ceiling, +//! total lifetime, upstream idle, blocked downstream, cancellation), each +//! proving the permit goes back by the admission of a later request. + +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::State; +use axum::http::header::{AUTHORIZATION, CONTENT_TYPE}; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode}; +use axum::response::Response; +use axum::routing::post; +use futures_util::StreamExt as _; +use gateway::{Config, Gateway, ProfileName, ProfilesContext}; +use serde_json::Value; +use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; +use tokio::sync::oneshot; + +use crate::support::{ + PHASE_TIMEOUT, RecordedRequest, Recorder, ReleaseTx, TestServer, join_within, json_within, + next_arrival, parse_sse, send_within, spawn_backend, text_within, +}; + +/// Canned audio bytes with non-UTF8 content, so a text-handling mistake on +/// the passthrough path shows up as a byte difference. +const CANNED_AUDIO: &[u8] = b"\xff\xfb\x90\x00ID3 fake mp3 frames \x00\x01\x02 stream"; + +fn speech_body() -> Value { + serde_json::json!({ + "model": "tts-model", + "input": "hello from the gateway", + "voice": "alloy" + }) +} + +fn spawn_speech( + client: &reqwest::Client, + url: &str, +) -> tokio::task::JoinHandle> { + let client = client.clone(); + let url = url.to_string(); + tokio::spawn(async move { + client + .post(url) + .bearer_auth("test-token") + .json(&speech_body()) + .send() + .await + }) +} + +/// Reads a full binary body bounded by [`PHASE_TIMEOUT`] (IT-003). +async fn bytes_within(response: reqwest::Response) -> Vec { + tokio::time::timeout(PHASE_TIMEOUT, response.bytes()) + .await + .expect("HTTP body read exceeded the phase timeout") + .expect("HTTP body read failed") + .to_vec() +} + +/// A fake speech backend that records each request, then replies 200 with +/// the canned audio bytes and the given `Content-Type` (or none at all, so +/// the route's format-to-MIME fallback is exercised). +async fn recording_speech_backend(content_type: Option<&'static str>) -> (SocketAddr, Recorder) { + async fn speech( + State((recorder, content_type)): State<(Recorder, Option<&'static str>)>, + method: Method, + uri: axum::http::Uri, + headers: HeaderMap, + body: Bytes, + ) -> Response { + let authorization = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body = serde_json::from_slice(&body).unwrap_or(Value::Null); + recorder.lock().unwrap().push(RecordedRequest { + method: method.to_string(), + path: uri.path().to_string(), + authorization, + body, + }); + let mut response = Response::new(Body::from(CANNED_AUDIO)); + if let Some(content_type) = content_type { + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static(content_type)); + } + response + } + + let recorder: Recorder = Arc::new(Mutex::new(Vec::new())); + let router = Router::new() + .route("/audio/speech", post(speech)) + .with_state((Arc::clone(&recorder), content_type)); + (spawn_backend(router).await, recorder) +} + +/// A fake speech backend that hands the test a release handle on each +/// arrival: the first audio chunk flows immediately and the second waits +/// for the handle, so a test can hold the stream open mid-flight. No +/// sleeps: arrival and release are rendezvous. +async fn gated_audio_backend() -> (SocketAddr, UnboundedReceiver) { + async fn speech(State(arrivals): State>) -> Response { + let (release, released) = oneshot::channel(); + let _ = arrivals.send(release); + let first = futures_util::stream::once(async { + Ok::<_, std::convert::Infallible>(Bytes::from_static(b"audio-chunk-1;")) + }); + let rest = futures_util::stream::once(async move { + let _ = released.await; + Ok(Bytes::from_static(b"audio-chunk-2;")) + }); + let mut response = Response::new(Body::from_stream(first.chain(rest))); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + + let (arrivals, receiver) = mpsc::unbounded_channel::(); + let router = Router::new() + .route("/audio/speech", post(speech)) + .with_state(arrivals); + (spawn_backend(router).await, receiver) +} + +/// A fake speech backend that answers every request with the given status +/// and body, for the upstream-error envelope tests. +async fn status_speech_backend(status: StatusCode, body: &'static str) -> SocketAddr { + async fn speech( + State((status, body)): State<(StatusCode, &'static str)>, + ) -> (StatusCode, &'static str) { + (status, body) + } + spawn_backend( + Router::new() + .route("/audio/speech", post(speech)) + .with_state((status, body)), + ) + .await +} + +/// Start a gateway serving one remote speech model. `voices` renders the +/// catalog list (`Some(&[])` renders an explicit empty list, `None` omits +/// the field). With `pool`, the endpoint binds to a dominion capped at that +/// many in-flight requests with the given waiting depth and policy; +/// without it the endpoint is an unlimited pass-through. +async fn speech_gateway( + backend: SocketAddr, + voices: Option<&[&str]>, + pool: Option<(usize, usize, &str)>, +) -> TestServer { + let voices = voices.map_or_else(String::new, |list| { + let list = list + .iter() + .map(|voice| format!("\"{voice}\"")) + .collect::>() + .join(", "); + format!("voices = [{list}]") + }); + let (dominion, binding) = match pool { + Some((concurrency, depth, policy)) => ( + format!( + r#" +[[dominion]] +id = "pool" +kind = "remote" +max_concurrency = {concurrency} +max_queue = {depth} +policy = "{policy}" +"# + ), + "\ndominion = \"pool\"", + ), + None => (String::new(), ""), + }; + let toml = format!( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:0" +api_key = "test-token" +trust_loopback = false +{dominion} +[[endpoint]] +id = "fake" +protocol = "openai" +base_url = "http://{backend}" +api_key = ""{binding} + +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model for integration" +context = 8192 +upstream = "backend-tts" +endpoints = ["fake"] +{voices} +"# + ); + let config = Config::from_toml_str(&toml).unwrap(); + let gateway = Gateway::from_config(&config, ProfilesContext::default()).unwrap(); + TestServer::start(gateway).await +} + +/// IT-005/006 for the speech route: the backend records the request, so we +/// assert exactly what the gateway forwarded - method, path, the rewritten +/// upstream model, the intact input and voice, and the structural mp3 pin +/// reaching the provider when the client omits `response_format` - and that +/// the client's bearer is not leaked upstream. The response body is the +/// upstream's bytes, unchanged, under the upstream's own `Content-Type`. +#[tokio::test] +async fn remote_passthrough_streams_audio_bytes_unchanged() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy", "nova"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + assert_eq!( + response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("audio/mpeg"), + "the upstream content type is forwarded" + ); + assert!( + response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .is_none(), + "a streamed audio body never carries Content-Length" + ); + let body = bytes_within(response).await; + assert_eq!(body, CANNED_AUDIO, "audio bytes pass through unchanged"); + + let seen = recorder.lock().unwrap().clone(); + assert_eq!(seen.len(), 1, "backend saw exactly one request"); + let request = &seen[0]; + assert_eq!(request.method, "POST"); + assert_eq!(request.path, "/audio/speech"); + assert_eq!( + request.body.get("model").and_then(Value::as_str), + Some("backend-tts"), + "public model name rewritten to the upstream alias" + ); + assert_eq!( + request.body.get("input").and_then(Value::as_str), + Some("hello from the gateway"), + "input forwarded intact, emotion tag and all: angle-bracket markup is never sanitized" + ); + assert_eq!( + request.body.get("voice").and_then(Value::as_str), + Some("alloy"), + "voice forwarded intact" + ); + assert_eq!( + request.body.get("response_format").and_then(Value::as_str), + Some("mp3"), + "the structural mp3 pin reaches the provider when the field is omitted" + ); + assert_ne!( + request.authorization.as_deref(), + Some("Bearer test-token"), + "caller bearer must not leak to the upstream" + ); + gateway.shutdown().await; +} + +/// When the upstream omits `Content-Type`, the route falls back to the +/// requested format's MIME type (the OpenAI spellings). +#[tokio::test] +async fn content_type_falls_back_to_the_format_mime_mapping() { + let (backend, _recorder) = recording_speech_backend(None).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + let client = reqwest::Client::new(); + for (format, mime) in [ + ("mp3", "audio/mpeg"), + ("opus", "audio/ogg"), + ("aac", "audio/aac"), + ("flac", "audio/flac"), + ("wav", "audio/wav"), + ("pcm", "audio/pcm"), + ] { + let response = send_within( + client + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "tts-model", + "input": "format check", + "voice": "alloy", + "response_format": format, + })), + ) + .await; + assert_eq!(response.status().as_u16(), 200, "format {format}"); + assert_eq!( + response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some(mime), + "format {format} falls back to its MIME type" + ); + } + gateway.shutdown().await; +} + +/// A model configured for a non-speech kind is rejected on the speech route +/// with 400 and `kind_mismatch` before any queue admission or upstream call. +#[tokio::test] +async fn non_speech_kinds_are_rejected_on_the_speech_route() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let toml = format!( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:0" +api_key = "test-token" + +[[endpoint]] +id = "fake" +protocol = "openai" +base_url = "http://{backend}" +api_key = "" + +[[model]] +name = "chat-model" +description = "a chat model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] + +[[model]] +name = "embed-model" +kind = "embedding" +description = "an embedding model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] + +[[model]] +name = "reranker" +kind = "classifier" +description = "a classifier model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] +"# + ); + let config = Config::from_toml_str(&toml).unwrap(); + let gateway = Gateway::from_config(&config, ProfilesContext::default()).unwrap(); + let gateway = TestServer::start(gateway).await; + + for model in ["chat-model", "embed-model", "reranker"] { + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": model, + "input": "say something", + "voice": "alloy" + })), + ) + .await; + assert_eq!(response.status().as_u16(), 400, "model {model}"); + let body = json_within(response).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("kind_mismatch"), + "model {model}" + ); + } + assert!( + recorder.lock().unwrap().is_empty(), + "a kind-mismatched request must never reach the backend" + ); + gateway.shutdown().await; +} + +/// A voice outside the model's catalog list is rejected with 400 +/// `invalid_voice` naming the valid voices, before any upstream call. +#[tokio::test] +async fn unknown_voice_is_rejected_naming_the_valid_voices() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy", "nova"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "tts-model", + "input": "say something", + "voice": "coral" + })), + ) + .await; + assert_eq!(response.status().as_u16(), 400); + let body = json_within(response).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("invalid_voice") + ); + assert_eq!( + body.pointer("/error/type").and_then(Value::as_str), + Some("invalid_request_error") + ); + let message = body + .pointer("/error/message") + .and_then(Value::as_str) + .unwrap(); + assert!( + message.contains("alloy") && message.contains("nova"), + "the error names the valid voices: {message}" + ); + assert!( + recorder.lock().unwrap().is_empty(), + "a voice rejection never reaches the backend" + ); + gateway.shutdown().await; +} + +/// The voice check runs before dominion queue admission: with the only +/// concurrency slot held and the pool on the fail-fast `reject` policy, a +/// bad voice still earns 400 rather than the pool's 429, while a valid +/// voice earns the 429 - proving the pool really was full. +#[tokio::test] +async fn voice_validation_precedes_queue_admission() { + let (backend, mut arrivals) = gated_audio_backend().await; + let gateway = speech_gateway(backend, Some(&["alloy", "nova"]), Some((1, 100, "reject"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let first = spawn_speech(&client, &url); + let release_first = next_arrival(&mut arrivals).await; + + let bad_voice = send_within(client.post(&url).bearer_auth("test-token").json( + &serde_json::json!({ + "model": "tts-model", + "input": "say something", + "voice": "coral" + }), + )) + .await; + assert_eq!( + bad_voice.status().as_u16(), + 400, + "voice validation fires before admission, so a full pool cannot turn it into a 429" + ); + let body = json_within(bad_voice).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("invalid_voice") + ); + + let good_voice = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!( + good_voice.status().as_u16(), + 429, + "the pool really was full: a valid request is rejected at admission" + ); + let body = json_within(good_voice).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("queue_rejected") + ); + + release_first.send(()).unwrap(); + let first = join_within(first).await.unwrap(); + assert_eq!(first.status().as_u16(), 200); + assert_eq!(bytes_within(first).await, b"audio-chunk-1;audio-chunk-2;"); + gateway.shutdown().await; +} + +/// A model with an empty catalog `voices` list exposes no fixed voice set: +/// any voice name passes the route's check and is forwarded verbatim. +#[tokio::test] +async fn an_empty_voices_list_skips_the_voice_check() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&[]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "tts-model", + "input": "say something", + "voice": "anything-goes" + })), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let seen = recorder.lock().unwrap().clone(); + assert_eq!(seen.len(), 1, "backend saw exactly one request"); + assert_eq!( + seen[0].body.get("voice").and_then(Value::as_str), + Some("anything-goes"), + "the unchecked voice is forwarded verbatim" + ); + gateway.shutdown().await; +} + +/// The OpenAI object voice form (`{"id": ...}`) is validated by its `id` +/// against the catalog list and forwarded verbatim. +#[tokio::test] +async fn voice_object_form_is_validated_by_id_and_forwarded() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy", "nova"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "tts-model", + "input": "say something", + "voice": { "id": "nova" } + })), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let seen = recorder.lock().unwrap().clone(); + assert_eq!(seen.len(), 1, "backend saw exactly one request"); + assert_eq!( + seen[0].body.get("voice"), + Some(&serde_json::json!({ "id": "nova" })), + "the object form is forwarded unchanged" + ); + gateway.shutdown().await; +} + +/// The speech handler admits through the model's dominion queue exactly +/// like chat: with one in-flight slot and one waiting slot, the third +/// request is 503 `queue_full`. +#[tokio::test] +async fn queue_full_returns_503_when_waiting_slots_exhausted() { + let (backend, mut arrivals) = gated_audio_backend().await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 1, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let first = spawn_speech(&client, &url); + let release_first = next_arrival(&mut arrivals).await; + + // Exactly one of these acquires the single waiting slot; the other is 503. + // The race is bounded by PHASE_TIMEOUT (IT-003): if admission broke so + // neither request completes, the test fails instead of hanging. + let mut second = spawn_speech(&client, &url); + let mut third = spawn_speech(&client, &url); + let (rejected, survivor) = tokio::time::timeout(PHASE_TIMEOUT, async { + tokio::select! { + r = &mut second => (r, third), + r = &mut third => (r, second), + } + }) + .await + .expect("neither queued request completed within the phase timeout"); + let rejected = rejected.unwrap().unwrap(); + assert_eq!(rejected.status().as_u16(), 503); + let body = json_within(rejected).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("queue_full") + ); + + release_first.send(()).unwrap(); + let first = join_within(first).await.unwrap(); + assert_eq!(first.status().as_u16(), 200); + let _ = bytes_within(first).await; + + let release_survivor = next_arrival(&mut arrivals).await; + release_survivor.send(()).unwrap(); + let survivor = join_within(survivor).await.unwrap(); + assert_eq!(survivor.status().as_u16(), 200); + let _ = bytes_within(survivor).await; + gateway.shutdown().await; +} + +/// Bounded negative wait for a request that must not be admitted: long +/// enough that a released slot would deterministically let the request +/// reach the backend over loopback, and comfortably under the relay's +/// scaled 200 ms upstream-idle budget so the held stream's relay cannot +/// trip idle and free the slot mid-wait (the same ceiling [`DRIP_GAP`] +/// stays under). +const ADMISSION_GRACE: Duration = Duration::from_millis(100); + +/// Under concurrency=1, a speech request holds the dominion queue permit +/// for the audio stream's whole lifetime: a second request is not admitted +/// until the first stream has ended. +#[tokio::test] +async fn stream_permit_is_held_until_the_audio_stream_ends() { + let (backend, mut arrivals) = gated_audio_backend().await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let first = spawn_speech(&client, &url); + let release_first = next_arrival(&mut arrivals).await; + + // The second request cannot be admitted while the first stream holds the + // only concurrency slot: a bounded negative wait proves no arrival, where + // a bare try_recv would pass vacuously on a current-thread runtime that + // never polled the spawned request. + let second = spawn_speech(&client, &url); + let arrived = tokio::time::timeout(ADMISSION_GRACE, arrivals.recv()).await; + assert!( + arrived.is_err(), + "second request must not reach the backend while the stream is open" + ); + + let first_response = join_within(first).await.unwrap(); + assert_eq!(first_response.status().as_u16(), 200); + release_first.send(()).unwrap(); + // Drain the body so the relay finishes and releases the permit. + let body = bytes_within(first_response).await; + assert_eq!(body, b"audio-chunk-1;audio-chunk-2;", "stream completed"); + + // After the stream ends, the second is admitted and reaches the backend. + let release_second = next_arrival(&mut arrivals).await; + release_second.send(()).unwrap(); + let second = join_within(second).await.unwrap(); + assert_eq!(second.status().as_u16(), 200); + let _ = bytes_within(second).await; + gateway.shutdown().await; +} + +/// A client disconnect mid-stream cancels the upstream stream: dropping the +/// response body drops the relay, which drops the gateway's upstream +/// connection, which the backend observes as its own response body being +/// dropped. Drop is the entire mechanism - there is no explicit cancel +/// path. The released permit admits a later request under concurrency=1. +#[tokio::test] +async fn client_disconnect_aborts_the_upstream_stream_and_releases_the_permit() { + /// Signals once the backend's response body is dropped mid-stream. + struct NotifyOnDrop(UnboundedSender<()>); + impl Drop for NotifyOnDrop { + fn drop(&mut self) { + let _ = self.0.send(()); + } + } + + let (dropped, mut observed) = mpsc::unbounded_channel::<()>(); + let backend = spawn_backend(Router::new().route( + "/audio/speech", + post(move || { + let dropped = dropped.clone(); + async move { + let first = futures_util::stream::once(async { + Ok::<_, std::convert::Infallible>(Bytes::from_static(b"audio-chunk-1;")) + }); + let rest = futures_util::stream::once(async move { + let _notify = NotifyOnDrop(dropped); + futures_util::future::pending::<()>().await; + unreachable!("the stream never yields a second chunk") + }); + let mut response = Response::new(Body::from_stream(first.chain(rest))); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + }), + )) + .await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let mut response = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + // Read the first chunk so the stream is genuinely mid-flight, then hang up. + let first = tokio::time::timeout(PHASE_TIMEOUT, response.chunk()) + .await + .expect("first chunk read exceeded the phase timeout") + .expect("first chunk read failed"); + assert!(first.is_some(), "first chunk arrived"); + drop(response); + + tokio::time::timeout(PHASE_TIMEOUT, observed.recv()) + .await + .expect("backend did not observe the disconnect within the phase timeout") + .expect("disconnect notification channel closed"); + + // The permit went back: a second request is admitted under concurrency=1. + let mut second = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(second.status().as_u16(), 200); + let chunk = tokio::time::timeout(PHASE_TIMEOUT, second.chunk()) + .await + .expect("second chunk read exceeded the phase timeout") + .expect("second chunk read failed"); + assert!( + chunk.is_some(), + "the second request was admitted and answered after the disconnect released the permit" + ); + drop(second); + gateway.shutdown().await; +} + +/// A mid-stream upstream failure after HTTP 200 cannot become an error +/// envelope - audio bytes already flowed - so the relay propagates the +/// failure and the client's body read fails on the truncation; the bytes +/// that did arrive are pure audio with no JSON spliced in. +#[tokio::test] +async fn mid_stream_upstream_error_fails_the_body_read_without_an_envelope() { + const AUDIO_PREFIX: &[u8] = b"audio-so-far;"; + + // The backend sends one chunk, then waits for the test to trigger the + // failure, so the error is guaranteed to land after the client holds a + // 200 and real audio bytes: a rendezvous, not a race. + let (fail, wait_fail) = oneshot::channel::<()>(); + let wait_fail = Arc::new(Mutex::new(Some(wait_fail))); + let backend = spawn_backend(Router::new().route( + "/audio/speech", + post(move || { + let wait_fail = Arc::clone(&wait_fail); + async move { + let wait = wait_fail + .lock() + .unwrap() + .take() + .expect("the test sends one request"); + let first = futures_util::stream::once(async { + Ok::<_, std::io::Error>(Bytes::from_static(AUDIO_PREFIX)) + }); + let rest = futures_util::stream::once(async move { + let _ = wait.await; + Err(std::io::Error::other("upstream died mid-stream")) + }); + let mut response = Response::new(Body::from_stream(first.chain(rest))); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + }), + )) + .await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let mut response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!( + response.status().as_u16(), + 200, + "the failure is mid-stream, after the 200" + ); + let first = tokio::time::timeout(PHASE_TIMEOUT, response.chunk()) + .await + .expect("first chunk read exceeded the phase timeout") + .expect("first chunk read failed") + .expect("the first audio chunk arrived"); + let mut received = first.to_vec(); + fail.send(()).expect("the backend is waiting on the signal"); + let failed = loop { + match tokio::time::timeout(PHASE_TIMEOUT, response.chunk()) + .await + .expect("body read exceeded the phase timeout") + { + Ok(Some(chunk)) => received.extend_from_slice(&chunk), + Ok(None) => break false, + Err(_) => break true, + } + }; + assert!( + failed, + "the client's body read fails on a mid-stream upstream error" + ); + assert_eq!( + received, AUDIO_PREFIX, + "only audio bytes arrived; no JSON envelope was spliced into the stream" + ); + gateway.shutdown().await; +} + +/// `GET /v1/models` surfaces a speech model's kind and catalog voices. +#[tokio::test] +async fn models_catalog_shows_the_speech_kind_and_voices() { + let (backend, _recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy", "nova"]), None).await; + + let response = send_within( + reqwest::Client::new() + .get(format!("http://{}/v1/models", gateway.addr)) + .bearer_auth("test-token"), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let body = json_within(response).await; + let data = body.get("data").and_then(Value::as_array).unwrap(); + let model = data + .iter() + .find(|entry| entry.get("id").and_then(Value::as_str) == Some("tts-model")) + .expect("the speech model is listed"); + assert_eq!(model.get("kind").and_then(Value::as_str), Some("speech")); + assert_eq!( + model.get("voices").and_then(Value::as_array), + Some(&vec![Value::from("alloy"), Value::from("nova")]), + "the catalog voices are listed: {model}" + ); + gateway.shutdown().await; +} + +/// Start a gateway whose catalog is the given `[[model]]` TOML fragments, +/// all resolving to one fake backend. The voices route never calls an +/// upstream; the backend exists only to satisfy config validation. +async fn catalog_gateway(backend: SocketAddr, models: &str) -> TestServer { + let toml = format!( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:0" +api_key = "test-token" +trust_loopback = false + +[[endpoint]] +id = "fake" +protocol = "openai" +base_url = "http://{backend}" +api_key = "" + +{models} +"# + ); + let config = Config::from_toml_str(&toml).unwrap(); + let gateway = Gateway::from_config(&config, ProfilesContext::default()).unwrap(); + TestServer::start(gateway).await +} + +/// `GET /v1/audio/voices` with the test bearer, returning the raw body. +async fn voices_body(gateway: &TestServer) -> (u16, String) { + let response = send_within( + reqwest::Client::new() + .get(format!("http://{}/v1/audio/voices", gateway.addr)) + .bearer_auth("test-token"), + ) + .await; + let status = response.status().as_u16(); + let body = bytes_within(response).await; + ( + status, + String::from_utf8(body).expect("the voices body is UTF-8"), + ) +} + +/// `GET /v1/audio/voices` answers the union of the speech models' catalog +/// voices as id-first `{"id", "name"}` entries under `{"voices": [...]}`. +/// OpenAI has no voice-list route, but the OpenAI-compatible ecosystem +/// converged on this shape and clients read the `id` key, so the entry +/// shape is a compatibility surface pinned here on the raw body. +#[tokio::test] +async fn voices_route_returns_the_id_first_union_shape() { + let (backend, _recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = catalog_gateway( + backend, + r#" +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model" +context = 8192 +upstream = "backend-tts" +endpoints = ["fake"] +voices = ["alloy"] +"#, + ) + .await; + + let (status, body) = voices_body(&gateway).await; + assert_eq!(status, 200); + assert_eq!(body, r#"{"voices":[{"id":"alloy","name":"alloy"}]}"#); + gateway.shutdown().await; +} + +/// The union is deduplicated and sorted across every speech model in the +/// active profile. +#[tokio::test] +async fn voices_route_deduplicates_and_sorts_across_speech_models() { + let (backend, _recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = catalog_gateway( + backend, + r#" +[[model]] +name = "tts-one" +kind = "speech" +description = "a speech model" +context = 8192 +upstream = "backend-tts" +endpoints = ["fake"] +voices = ["nova", "alloy"] + +[[model]] +name = "tts-two" +kind = "speech" +description = "another speech model" +context = 8192 +upstream = "backend-tts" +endpoints = ["fake"] +voices = ["shimmer", "nova"] +"#, + ) + .await; + + let (status, body) = voices_body(&gateway).await; + assert_eq!(status, 200); + assert_eq!( + body, + r#"{"voices":[{"id":"alloy","name":"alloy"},{"id":"nova","name":"nova"},{"id":"shimmer","name":"shimmer"}]}"# + ); + gateway.shutdown().await; +} + +/// With no speech model in the active profile the union is empty. +#[tokio::test] +async fn voices_route_is_empty_without_speech_models() { + let (backend, _recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = catalog_gateway( + backend, + r#" +[[model]] +name = "chat-model" +description = "a chat model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] +"#, + ) + .await; + + let (status, body) = voices_body(&gateway).await; + assert_eq!(status, 200); + assert_eq!(body, r#"{"voices":[]}"#); + gateway.shutdown().await; +} + +/// Non-speech models contribute nothing to the union. +#[tokio::test] +async fn voices_route_ignores_non_speech_models() { + let (backend, _recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = catalog_gateway( + backend, + r#" +[[model]] +name = "chat-model" +description = "a chat model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] + +[[model]] +name = "embed-model" +kind = "embedding" +description = "an embedding model" +context = 8192 +upstream = "backend-model" +endpoints = ["fake"] + +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model" +context = 8192 +upstream = "backend-tts" +endpoints = ["fake"] +voices = ["nova"] +"#, + ) + .await; + + let (status, body) = voices_body(&gateway).await; + assert_eq!(status, 200); + assert_eq!(body, r#"{"voices":[{"id":"nova","name":"nova"}]}"#); + gateway.shutdown().await; +} + +/// Auth runs before the union is computed: a request with no +/// Authorization header is refused 401 with the `unauthorized` envelope, +/// and no voice entry leaves the handler. +#[tokio::test] +async fn unauthenticated_voices_request_is_refused_before_the_union_is_computed() { + let (backend, _recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = catalog_gateway( + backend, + r#" +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model" +context = 8192 +upstream = "backend-tts" +endpoints = ["fake"] +voices = ["alloy"] +"#, + ) + .await; + + let response = + send_within(reqwest::Client::new().get(format!("http://{}/v1/audio/voices", gateway.addr))) + .await; + assert_eq!(response.status().as_u16(), 401); + let body = json_within(response).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("unauthorized") + ); + assert!( + body.get("voices").is_none(), + "the union is never computed for an unauthenticated request: {body}" + ); + gateway.shutdown().await; +} + +/// An upstream 429 maps to the speech-only 429 envelope +/// (`rate_limit_error` / `upstream_rate_limited`), so an OpenAI client sees +/// a retryable rate-limit error rather than a server failure. +#[tokio::test] +async fn upstream_429_maps_to_rate_limited() { + let backend = status_speech_backend(StatusCode::TOO_MANY_REQUESTS, "provider throttled").await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 429); + let body = json_within(response).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("upstream_rate_limited") + ); + assert_eq!( + body.pointer("/error/type").and_then(Value::as_str), + Some("rate_limit_error") + ); + gateway.shutdown().await; +} + +/// An upstream 503 maps to the speech-only 503 envelope +/// (`server_error` / `upstream_unavailable`) rather than the shared +/// mapping's 502. +#[tokio::test] +async fn upstream_503_maps_to_unavailable() { + let backend = status_speech_backend(StatusCode::SERVICE_UNAVAILABLE, "provider down").await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 503); + let body = json_within(response).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("upstream_unavailable") + ); + assert_eq!( + body.pointer("/error/type").and_then(Value::as_str), + Some("server_error") + ); + gateway.shutdown().await; +} + +/// An upstream error body carries provider internals - stack text, internal +/// hosts, request ids - and none of it may reach the client: the envelope +/// message is the gateway's own fixed string on both the speech-only +/// variants and the shared protocol arm. +#[tokio::test] +async fn upstream_error_bodies_never_reach_the_client() { + const INTERNALS: &str = "java.lang.IllegalStateException: voice clone failed\n\ + at com.acme.tts.Synth.speak(Synth.java:412)\n\ + host http://tts-internal.acme.corp:9090\n\ + x-request-id req-01HZX8AEFGH"; + for (status, expected) in [ + (StatusCode::TOO_MANY_REQUESTS, 429), + (StatusCode::INTERNAL_SERVER_ERROR, 502), + ] { + let backend = status_speech_backend(status, INTERNALS).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), expected, "status {status}"); + let body = tokio::time::timeout(PHASE_TIMEOUT, response.text()) + .await + .expect("body read exceeded the phase timeout") + .expect("body read failed"); + for marker in ["java.lang", "tts-internal.acme.corp", "req-01HZX8AEFGH"] { + assert!( + !body.contains(marker), + "provider internals must not leak ({marker}) into: {body}" + ); + } + gateway.shutdown().await; + } +} + +/// Auth runs before the body is parsed: an unauthenticated request with a +/// malformed JSON body is refused 401, and nothing reaches the backend. +#[tokio::test] +async fn unauthenticated_speech_is_refused_before_the_body_is_parsed() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body("{not json"), + ) + .await; + assert_eq!(response.status().as_u16(), 401); + let body = json_within(response).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("unauthorized") + ); + assert!( + recorder.lock().unwrap().is_empty(), + "an unauthenticated request never reaches the backend" + ); + gateway.shutdown().await; +} + +/// An unknown model on the speech route is a 404 `model_not_found` +/// envelope, exactly as on the chat route, and never reaches the backend. +#[tokio::test] +async fn unknown_model_on_the_speech_route_returns_model_not_found() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "ghost", + "input": "say something", + "voice": "alloy" + })), + ) + .await; + assert_eq!(response.status().as_u16(), 404); + let body = json_within(response).await; + assert_eq!( + body.pointer("/error/code").and_then(Value::as_str), + Some("model_not_found") + ); + assert!( + recorder.lock().unwrap().is_empty(), + "an unknown model never reaches the backend" + ); + gateway.shutdown().await; +} + +/// `stream_format` rides the verbatim passthrough: the provider sees the +/// framing selector exactly as the client sent it. +#[tokio::test] +async fn stream_format_sse_is_forwarded_to_the_upstream_body() { + let (backend, recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "tts-model", + "input": "say something", + "voice": "alloy", + "stream_format": "sse", + })), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let _ = bytes_within(response).await; + let seen = recorder.lock().unwrap().clone(); + assert_eq!(seen.len(), 1, "backend saw exactly one request"); + assert_eq!( + seen[0].body.get("stream_format").and_then(Value::as_str), + Some("sse"), + "the framing selector is forwarded into the outbound upstream body" + ); + gateway.shutdown().await; +} + +/// With `stream_format = "sse"` the fallback media type follows the framing +/// selector: an upstream that omits `Content-Type` is labeled +/// `text/event-stream`, never an audio type. +#[tokio::test] +async fn sse_framing_falls_back_to_event_stream_when_content_type_is_missing() { + let (backend, _recorder) = recording_speech_backend(None).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "tts-model", + "input": "say something", + "voice": "alloy", + "stream_format": "sse", + })), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + assert_eq!( + response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("text/event-stream"), + "the fallback takes the framing selector, never an audio type" + ); + gateway.shutdown().await; +} + +/// A present upstream `Content-Type` is forwarded verbatim even with +/// `stream_format = "sse"`: the framing selector drives only the fallback. +#[tokio::test] +async fn sse_framing_still_forwards_a_present_upstream_content_type() { + let (backend, _recorder) = recording_speech_backend(Some("audio/mpeg")).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&serde_json::json!({ + "model": "tts-model", + "input": "say something", + "voice": "alloy", + "stream_format": "sse", + })), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + assert_eq!( + response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("audio/mpeg"), + "a present upstream content type wins over the framing fallback" + ); + gateway.shutdown().await; +} + +// Bounded-relay boundary tests. The gateway builds this suite with its +// `test-fixtures` feature (the crate dev-depends on itself with it), which +// scales the relay's bounds down: byte ceiling 16 MiB, upstream read idle +// 200 ms, blocked downstream delivery 400 ms, total stream lifetime 2 s, and +// the profile-switch drain deadline 1 s. These mirrors name the same numbers +// so the boundary tests run in milliseconds; the relay's own constants are +// the source of truth. + +/// The relay's test-scaled response byte ceiling. +const RELAY_BYTE_CEILING: u64 = 16 * 1024 * 1024; +/// A drip gap comfortably under the scaled 200 ms upstream-idle budget. +const DRIP_GAP: Duration = Duration::from_millis(60); +/// A header delay past the scaled idle budget but far under the +/// first-response budget: time-to-headers is never the relay's business. +const HEADER_DELAY: Duration = Duration::from_millis(300); + +/// Waits for a backend arrival ping, bounded by [`PHASE_TIMEOUT`]. +async fn next_ping(arrivals: &mut UnboundedReceiver<()>) { + tokio::time::timeout(PHASE_TIMEOUT, arrivals.recv()) + .await + .expect("timed out waiting for backend arrival") + .expect("arrivals channel closed"); +} + +/// Reads a response body to its end or its failure, returning the bytes +/// that arrived and whether the read failed. A relay terminal path surfaces +/// over HTTP only as a failed read (never a clean EOF): hyper aborts the +/// response on a body-stream error, so the terminal item's message stays +/// server-side and the tests discriminate the bounds by stream shape and +/// timing instead. The whole read is bounded by [`PHASE_TIMEOUT`], so a +/// stream that never ends and never fails is a test failure, never a hang. +async fn read_to_end_or_error(response: reqwest::Response) -> (Vec, bool) { + let mut response = response; + let mut received = Vec::new(); + let deadline = tokio::time::Instant::now() + PHASE_TIMEOUT; + loop { + let item = tokio::time::timeout_at(deadline, response.chunk()) + .await + .expect("HTTP body read exceeded the phase timeout"); + match item { + Ok(Some(chunk)) => received.extend_from_slice(&chunk), + Ok(None) => return (received, false), + Err(_) => return (received, true), + } + } +} + +/// Proves the first stream's permit went back: under a one-slot dominion a +/// second request is admitted only once the relay holding the slot ended, +/// so its arrival at the backend is the release proof. The admitted request +/// is answered and then dropped mid-stream. +async fn assert_permit_released_by_admission( + client: &reqwest::Client, + url: &str, + arrivals: &mut UnboundedReceiver<()>, +) { + let second = spawn_speech(client, url); + next_ping(arrivals).await; + let second = join_within(second) + .await + .expect("the second request sends once the permit is free"); + assert_eq!( + second.status().as_u16(), + 200, + "a later request is admitted once the ended relay released the permit" + ); + drop(second); +} + +/// A fake speech backend streaming exactly `total` bytes in 64 KiB chunks, +/// pinging the arrivals channel per request so a test can prove a later +/// request was admitted after the first stream's relay ended. +async fn fixed_size_audio_backend(total: u64) -> (SocketAddr, UnboundedReceiver<()>) { + async fn speech(State((total, arrivals)): State<(u64, UnboundedSender<()>)>) -> Response { + let _ = arrivals.send(()); + let stream = futures_util::stream::unfold(0_u64, move |sent| async move { + let remaining = total - sent; + if remaining == 0 { + return None; + } + let len = usize::try_from(remaining.min(64 * 1024)).unwrap(); + let chunk = Bytes::from(vec![0xAB; len]); + Some(( + Ok::<_, std::convert::Infallible>(chunk), + sent + u64::try_from(len).unwrap(), + )) + }); + let mut response = Response::new(Body::from_stream(stream)); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + + let (arrivals, receiver) = mpsc::unbounded_channel::<()>(); + let router = Router::new() + .route("/audio/speech", post(speech)) + .with_state((total, arrivals)); + (spawn_backend(router).await, receiver) +} + +/// A fake speech backend dripping one small chunk every `gap`: `Some(n)` +/// chunks then a clean end, or `None` chunks forever. Arrivals are +/// signalled per request. +async fn dripping_audio_backend( + gap: Duration, + chunks: Option, +) -> (SocketAddr, UnboundedReceiver<()>) { + async fn speech( + State((gap, chunks, arrivals)): State<(Duration, Option, UnboundedSender<()>)>, + ) -> Response { + let _ = arrivals.send(()); + let stream = futures_util::stream::unfold(chunks, move |remaining| async move { + if remaining == Some(0) { + return None; + } + tokio::time::sleep(gap).await; + let remaining = remaining.map(|left| left - 1); + Some(( + Ok::<_, std::convert::Infallible>(Bytes::from_static(b"audio-drip;")), + remaining, + )) + }); + let mut response = Response::new(Body::from_stream(stream)); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + + let (arrivals, receiver) = mpsc::unbounded_channel::<()>(); + let router = Router::new() + .route("/audio/speech", post(speech)) + .with_state((gap, chunks, arrivals)); + (spawn_backend(router).await, receiver) +} + +/// A fake speech backend that sends one audio chunk and then pends forever, +/// so the relay's upstream-idle budget is the only thing that can end the +/// stream. Arrivals are signalled per request. +async fn stalling_audio_backend() -> (SocketAddr, UnboundedReceiver<()>) { + async fn speech(State(arrivals): State>) -> Response { + let _ = arrivals.send(()); + let first = futures_util::stream::once(async { + Ok::<_, std::convert::Infallible>(Bytes::from_static(b"audio-chunk-1;")) + }); + let mut response = Response::new(Body::from_stream( + first.chain(futures_util::stream::pending()), + )); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + + let (arrivals, receiver) = mpsc::unbounded_channel::<()>(); + let router = Router::new() + .route("/audio/speech", post(speech)) + .with_state(arrivals); + (spawn_backend(router).await, receiver) +} + +/// A fake speech backend pouring unbounded 64 KiB chunks as fast as the +/// connection takes them, so a client that stops reading backpressures the +/// relay's bounded channel. Arrivals are signalled per request. +async fn saturating_audio_backend() -> (SocketAddr, UnboundedReceiver<()>) { + async fn speech(State(arrivals): State>) -> Response { + let _ = arrivals.send(()); + let stream = futures_util::stream::unfold((), |()| async { + Some(( + Ok::<_, std::convert::Infallible>(Bytes::from(vec![0xCD; 64 * 1024])), + (), + )) + }); + let mut response = Response::new(Body::from_stream(stream)); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + + let (arrivals, receiver) = mpsc::unbounded_channel::<()>(); + let router = Router::new() + .route("/audio/speech", post(speech)) + .with_state(arrivals); + (spawn_backend(router).await, receiver) +} + +/// A fake speech backend that waits `delay` before sending any headers, then +/// answers with the canned audio: time-to-headers is the first-response +/// budget's business, never the relay's per-read idle budget. +async fn slow_headers_audio_backend(delay: Duration) -> SocketAddr { + async fn speech(State(delay): State) -> Response { + tokio::time::sleep(delay).await; + let mut response = Response::new(Body::from(CANNED_AUDIO)); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + spawn_backend( + Router::new() + .route("/audio/speech", post(speech)) + .with_state(delay), + ) + .await +} + +/// The relay's response-byte ceiling: a stream totaling exactly the ceiling +/// is accepted whole and ends cleanly. +#[tokio::test] +async fn relay_accepts_a_stream_at_the_exact_byte_ceiling() { + let (backend, mut arrivals) = fixed_size_audio_backend(RELAY_BYTE_CEILING).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let response = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let (received, failed) = read_to_end_or_error(response).await; + assert!(!failed, "an exactly-at-ceiling stream ends cleanly"); + assert_eq!( + u64::try_from(received.len()).unwrap(), + RELAY_BYTE_CEILING, + "every byte arrived" + ); + + assert_permit_released_by_admission(&client, &url, &mut arrivals).await; + gateway.shutdown().await; +} + +/// One byte past the ceiling fails the client's body read: the relay emits +/// its terminal error item instead of the crossing chunk, so no more than +/// the ceiling ever reaches the client and the read fails rather than +/// ending cleanly. Only the byte ceiling can produce that shape: the stream +/// is read eagerly (no idle, no blockage) and finishes far under the total +/// deadline. The lower bound is fuzzy by the chunks hyper had in flight +/// when the terminal item aborted the response. The permit goes back. +#[tokio::test] +async fn relay_fails_the_stream_one_byte_over_the_byte_ceiling() { + let (backend, mut arrivals) = fixed_size_audio_backend(RELAY_BYTE_CEILING + 1).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let response = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let (received, failed) = read_to_end_or_error(response).await; + assert!(failed, "the read fails one byte over the ceiling"); + let received = u64::try_from(received.len()).unwrap(); + assert!( + received <= RELAY_BYTE_CEILING, + "the crossing chunk is never forwarded: {received} <= {RELAY_BYTE_CEILING}" + ); + assert!( + received + 1024 * 1024 >= RELAY_BYTE_CEILING, + "the stream ran to the ceiling; only in-flight chunks were lost to the abort: {received}" + ); + + assert_permit_released_by_admission(&client, &url, &mut arrivals).await; + gateway.shutdown().await; +} + +/// The total-lifetime deadline ends a stream whose drip would otherwise run +/// forever. Only the deadline can fire here: the drip stays under the +/// per-read idle budget, the client reads eagerly, and the bytes are +/// nowhere near the ceiling. The permit goes back. +#[tokio::test] +async fn relay_ends_a_drip_at_the_total_lifetime_deadline() { + let (backend, mut arrivals) = dripping_audio_backend(DRIP_GAP, None).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let response = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let (received, failed) = read_to_end_or_error(response).await; + assert!(!received.is_empty(), "the drip flowed before the deadline"); + assert!( + failed, + "the total deadline fails the read, never a clean EOF" + ); + + assert_permit_released_by_admission(&client, &url, &mut arrivals).await; + gateway.shutdown().await; +} + +/// A drip whose chunks arrive under the upstream-idle budget is healthy: +/// the idle budget is per-read, never a cap on the stream's length. +#[tokio::test] +async fn relay_tolerates_a_drip_under_the_idle_budget() { + let (backend, _arrivals) = dripping_audio_backend(DRIP_GAP, Some(5)).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let (received, failed) = read_to_end_or_error(response).await; + assert!(!failed, "a sub-idle drip ends cleanly"); + assert_eq!(received, b"audio-drip;".repeat(5), "every drip arrived"); + gateway.shutdown().await; +} + +/// An upstream that goes silent after headers trips the per-read idle +/// budget: the read fails well before the total-lifetime deadline (the only +/// other bound that could end a silent stream), and the permit goes back. +#[tokio::test] +async fn relay_fails_an_upstream_that_goes_idle_after_headers() { + let (backend, mut arrivals) = stalling_audio_backend().await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let mut response = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let first = tokio::time::timeout(PHASE_TIMEOUT, response.chunk()) + .await + .expect("first chunk read exceeded the phase timeout") + .expect("first chunk read failed"); + assert!(first.is_some(), "the first chunk arrived before the stall"); + let stalled = tokio::time::Instant::now(); + let (_rest, failed) = read_to_end_or_error(response).await; + assert!(failed, "the idle budget fails the read, never a clean EOF"); + assert!( + stalled.elapsed() < Duration::from_secs(1), + "the idle budget fires well ahead of the 2 s total deadline" + ); + + assert_permit_released_by_admission(&client, &url, &mut arrivals).await; + gateway.shutdown().await; +} + +/// A client that stops reading backpressures the relay's bounded channel; +/// the blocked-delivery budget ends the stream and frees the permit while +/// the client still holds the unread response, and the client's eventual +/// read fails instead of seeing a clean EOF. +#[tokio::test] +async fn relay_fails_a_client_that_stops_reading() { + let (backend, mut arrivals) = saturating_audio_backend().await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let mut first = send_within( + client + .post(&url) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(first.status().as_u16(), 200); + let chunk = tokio::time::timeout(PHASE_TIMEOUT, first.chunk()) + .await + .expect("first chunk read exceeded the phase timeout") + .expect("first chunk read failed"); + assert!(chunk.is_some(), "the stream started"); + + // Stop reading. The relay fills its channel and the blocked-delivery + // budget ends the stream; the permit release is observable as the second + // request's admission under the one-slot dominion, well ahead of the 2 s + // total deadline (the only other bound that could end this stream). + let stalled = tokio::time::Instant::now(); + assert_permit_released_by_admission(&client, &url, &mut arrivals).await; + assert!( + stalled.elapsed() < Duration::from_secs(1), + "the blocked-delivery budget fires well ahead of the 2 s total deadline" + ); + + // The stalled client's resumed read drains the buffered chunks and then + // fails on the terminal error item - never a clean EOF. + let (_buffered, failed) = read_to_end_or_error(first).await; + assert!(failed, "the stalled client's read fails, never a clean EOF"); + gateway.shutdown().await; +} + +/// Headers arriving after the relay's (scaled) per-read idle budget but +/// within the first-response budget are accepted: the idle budget guards +/// only an opened body, never the wait for headers. +#[tokio::test] +async fn headers_delayed_past_the_idle_budget_are_accepted() { + let backend = slow_headers_audio_backend(HEADER_DELAY).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), None).await; + + let response = send_within( + reqwest::Client::new() + .post(format!("http://{}/v1/audio/speech", gateway.addr)) + .bearer_auth("test-token") + .json(&speech_body()), + ) + .await; + assert_eq!(response.status().as_u16(), 200); + assert_eq!(bytes_within(response).await, CANNED_AUDIO); + gateway.shutdown().await; +} + +/// An upstream body error after the permit is held propagates as exactly +/// one terminal error item, and the permit goes back: a later request is +/// admitted and answered under the one-slot dominion. +#[tokio::test] +async fn relay_releases_the_permit_after_an_upstream_body_error() { + // The first request streams one chunk and then, once the test fires the + // trigger (a rendezvous, so the failure lands after real bytes flowed), + // fails; every later request streams the canned audio to a clean end. + type FailWait = Arc>>>; + async fn speech( + State((calls, wait_fail, arrivals)): State<( + Arc>, + FailWait, + UnboundedSender<()>, + )>, + ) -> Response { + let _ = arrivals.send(()); + let call = { + let mut calls = calls.lock().unwrap(); + *calls += 1; + *calls + }; + let stream = if call == 1 { + let wait = wait_fail + .lock() + .unwrap() + .take() + .expect("only the first request waits on the trigger"); + let first = futures_util::stream::once(async { + Ok::<_, std::io::Error>(Bytes::from_static(b"audio-chunk-1;")) + }); + let failure = futures_util::stream::once(async move { + let _ = wait.await; + Err(std::io::Error::other("upstream died mid-stream")) + }); + Body::from_stream(first.chain(failure)) + } else { + Body::from(CANNED_AUDIO) + }; + let mut response = Response::new(stream); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("audio/mpeg")); + response + } + + let (fail, wait_fail) = oneshot::channel::<()>(); + let (arrivals, mut receiver) = mpsc::unbounded_channel::<()>(); + let router = Router::new() + .route("/audio/speech", post(speech)) + .with_state(( + Arc::new(Mutex::new(0_usize)), + Arc::new(Mutex::new(Some(wait_fail))), + arrivals, + )); + let backend = spawn_backend(router).await; + let gateway = speech_gateway(backend, Some(&["alloy"]), Some((1, 10, "queue"))).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let first = spawn_speech(&client, &url); + next_ping(&mut receiver).await; + let mut first = join_within(first).await.unwrap(); + assert_eq!(first.status().as_u16(), 200); + let chunk = tokio::time::timeout(PHASE_TIMEOUT, first.chunk()) + .await + .expect("first chunk read exceeded the phase timeout") + .expect("first chunk read failed"); + assert!( + chunk.is_some(), + "real audio bytes flowed before the failure" + ); + fail.send(()).expect("the backend is waiting on the signal"); + let (_rest, failed) = read_to_end_or_error(first).await; + assert!( + failed, + "the upstream body error is the terminal item: the read fails, never a clean EOF" + ); + + assert_permit_released_by_admission(&client, &url, &mut receiver).await; + gateway.shutdown().await; +} + +/// Start a gateway with two profiles (`alpha` active, `beta` idle) that both +/// route the same speech model through a one-slot dominion, so a switch +/// cancels the open stream and the new profile's admission of a later +/// request proves the permit went back. +async fn speech_profile_gateway(backend: SocketAddr) -> (tempfile::TempDir, TestServer) { + let catalog = |backend: SocketAddr| { + format!( + r#" +config-version = 2 + +[server] +bind = "127.0.0.1:0" +api_key = "test-token" +trust_loopback = false + +[[dominion]] +id = "pool" +kind = "remote" +max_concurrency = 1 +max_queue = 10 +policy = "queue" + +[[endpoint]] +id = "fake" +protocol = "openai" +base_url = "http://{backend}" +api_key = "" +dominion = "pool" + +[[model]] +name = "tts-model" +kind = "speech" +description = "a speech model for integration" +context = 8192 +upstream = "backend-tts" +endpoints = ["fake"] +voices = ["alloy"] + +[[profile]] +name = "alpha" +models = ["tts-model"] + +[[profile]] +name = "beta" +models = ["tts-model"] +"# + ) + }; + let temp = tempfile::TempDir::new().expect("temp dir"); + let path = temp.path().join("gateway.toml"); + std::fs::write(&path, catalog(backend)).expect("write config"); + std::fs::write( + gateway_config::profile_state_path(&path), + "active_profile = \"alpha\"\n", + ) + .expect("write state"); + let alpha = ProfileName::parse("alpha").expect("name"); + let config = Config::from_toml_str(&catalog(backend)) + .expect("catalog parses") + .select_profile(&alpha) + .expect("alpha selects"); + let context = ProfilesContext::new(Some(path), Some(alpha)); + let server = + TestServer::start(Gateway::from_config(&config, context).expect("gateway builds")).await; + (temp, server) +} + +/// Drives a profile switch to completion over its SSE stream and returns +/// the events. +async fn switch_to(http: &reqwest::Client, addr: SocketAddr, name: &str) -> Vec { + let response = send_within( + http.post(format!("http://{addr}/admin/switch-profile")) + .bearer_auth("test-token") + .json(&serde_json::json!({ "name": name })), + ) + .await; + assert_eq!(response.status(), reqwest::StatusCode::OK); + parse_sse(&text_within(response).await) +} + +/// A profile switch cancels an open speech stream: the relay emits one +/// terminal error item, so the client's next read fails rather than seeing +/// a clean EOF (the `relay_sse` RequestCancelled envelope is the precedent +/// for failing rather than truncating). The switch's drain only completes +/// once the request's guard is dropped, and the guard and the dominion +/// permit live and die together in the relay task, so a completed switch +/// proves the permit went back; the new profile then admits and answers a +/// later request. +#[tokio::test] +async fn profile_switch_cancels_the_stream_and_releases_the_permit() { + let (backend, mut arrivals) = gated_audio_backend().await; + let (_temp, gateway) = speech_profile_gateway(backend).await; + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/audio/speech", gateway.addr); + + let first = spawn_speech(&client, &url); + // Held, never fired: the stream stays mid-flight until the switch + // cancels it. Consuming the arrival also keeps the second request's + // handle next in the channel. + let _release_first = next_arrival(&mut arrivals).await; + let mut first = join_within(first).await.unwrap(); + assert_eq!(first.status().as_u16(), 200); + let chunk = tokio::time::timeout(PHASE_TIMEOUT, first.chunk()) + .await + .expect("first chunk read exceeded the phase timeout") + .expect("first chunk read failed"); + assert!(chunk.is_some(), "the stream is mid-flight"); + + let addr = gateway.addr; + let switch = + tokio::spawn(async move { switch_to(&reqwest::Client::new(), addr, "beta").await }); + + // The switch's (test-scaled) drain deadline passes, the cancellation + // fires, and the client's next read fails on the synthesized error item. + // The relay's own total deadline is 2 s against the drain's 1 s, so the + // cancellation is the only bound that can end this stream. + let (_rest, failed) = read_to_end_or_error(first).await; + assert!( + failed, + "a profile switch fails the open body read, never a clean EOF" + ); + + let events = join_within(switch).await; + assert_eq!( + events.last(), + Some(&serde_json::json!({"status": "ready", "profile": "beta"})), + "the switch completed, so the cancelled request's guard is gone" + ); + + // The permit went back with the guard: the new profile admits and + // answers a speech request on the one-slot dominion. + let second = spawn_speech(&client, &url); + let release_second = next_arrival(&mut arrivals).await; + release_second.send(()).unwrap(); + let second = join_within(second).await.unwrap(); + assert_eq!(second.status().as_u16(), 200); + assert_eq!(bytes_within(second).await, b"audio-chunk-1;audio-chunk-2;"); + gateway.shutdown().await; +} diff --git a/crates/shared-protocol/Cargo.toml b/crates/shared-protocol/Cargo.toml index 1987be11..39a3dbd3 100644 --- a/crates/shared-protocol/Cargo.toml +++ b/crates/shared-protocol/Cargo.toml @@ -15,16 +15,17 @@ documentation = "https://cppalliance.github.io/promptforge/" [dependencies] async-trait.workspace = true +bytes.workspace = true futures-util.workspace = true gateway-config.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +tokio.workspace = true tracing.workspace = true [dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt"] } tracing-subscriber.workspace = true [lints] diff --git a/crates/shared-protocol/README.md b/crates/shared-protocol/README.md index 71e48df7..1d57a537 100644 --- a/crates/shared-protocol/README.md +++ b/crates/shared-protocol/README.md @@ -2,8 +2,9 @@ The OpenAI wire protocol and upstream abstraction for the PromptForge inference gateway: request/response wire types with trust-boundary -validation, the `Upstream` trait and its `OpenAiUpstream` passthrough, -bounded HTTP client helpers, and the protocol-level error types. +validation, the `Upstream` trait and its `OpenAiUpstream` passthrough +(chat, embeddings, rerank, and streaming speech), bounded HTTP client +helpers, and the protocol-level error types. This crate is the shared protocol contract between the gateway, its local inference subsystem, and external clients. It contains no local inference, diff --git a/crates/shared-protocol/src/http_util.rs b/crates/shared-protocol/src/http_util.rs index 845021d2..c8075eba 100644 --- a/crates/shared-protocol/src/http_util.rs +++ b/crates/shared-protocol/src/http_util.rs @@ -43,6 +43,28 @@ pub fn streaming_client() -> reqwest::Client { .unwrap_or_else(|_| reqwest::Client::new()) } +/// TCP keepalive interval for audio streams, so a silently dead peer or an +/// idle middlebox drop surfaces instead of hanging the stream forever. +const AUDIO_TCP_KEEPALIVE: Duration = Duration::from_secs(60); + +/// Build a reqwest client for long-lived binary audio streams. +/// +/// Like [`streaming_client`] there is no whole-request timeout, which would +/// kill any stream that outlives it, and TCP keepalive keeps middleboxes +/// from dropping the connection between reads. There is no `read_timeout` +/// either: reqwest arms it during the wait for response headers, and the +/// speech path gives time-to-headers its own, larger budget, so both +/// deadlines live in +/// [`Upstream::send_speech`](crate::upstream::Upstream::send_speech). +#[must_use] +pub fn audio_streaming_client() -> reqwest::Client { + reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .tcp_keepalive(AUDIO_TCP_KEEPALIVE) + .build() + .unwrap_or_else(|_| reqwest::Client::new()) +} + /// Read at most `cap` bytes from `response`, stopping early once the cap is hit. /// /// The body is streamed chunk by chunk so an oversized or stalled response never diff --git a/crates/shared-protocol/src/upstream.rs b/crates/shared-protocol/src/upstream.rs index 0213e8fd..86d93f82 100644 --- a/crates/shared-protocol/src/upstream.rs +++ b/crates/shared-protocol/src/upstream.rs @@ -5,7 +5,10 @@ //! unchanged. Adding an Anthropic or pack upstream later is a new implementation //! behind this same trait, with no change to routing or the request handler. +use std::time::Duration; + use async_trait::async_trait; +use bytes::Bytes; use futures_util::StreamExt; use futures_util::stream::BoxStream; use gateway_config::Secret; @@ -13,7 +16,7 @@ use gateway_config::Secret; use crate::error::{ProtocolError, ShutdownError}; use crate::wire::{ ChatChunk, ChatRequest, ChatResponse, EmbeddingRequest, EmbeddingResponse, RerankRequest, - RerankResponse, + RerankResponse, SpeechRequest, }; /// An opened streaming chat completion: the upstream response headers worth @@ -39,6 +42,28 @@ impl std::fmt::Debug for StreamedChunks { } } +/// An opened streaming speech synthesis: the upstream `Content-Type` plus the +/// raw audio byte stream. +pub struct StreamedAudio { + /// The upstream `Content-Type`, forwarded verbatim; empty when the + /// upstream omitted the header, which is the route's signal to apply its + /// format-to-MIME fallback. + pub content_type: String, + /// The untransformed audio byte stream: audio frames are opaque bytes the + /// gateway forwards unread. A mid-stream read failure surfaces as an + /// `Err` item rather than a silently truncated stream. Dropping the + /// stream drops the upstream response and aborts the connection. + pub body: BoxStream<'static, Result>, +} + +impl std::fmt::Debug for StreamedAudio { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StreamedAudio") + .field("content_type", &self.content_type) + .finish_non_exhaustive() + } +} + /// A backend the gateway can forward a chat completion to. #[async_trait] pub trait Upstream: Send + Sync { @@ -128,6 +153,28 @@ pub trait Upstream: Send + Sync { Err(ProtocolError::ModelUnavailable(req.model)) } + /// Forward a speech synthesis `req` to the backend, substituting + /// `upstream_model` for the caller's model name, and return the audio + /// stream. + /// + /// The default is [`ProtocolError::ModelUnavailable`]: upstreams without a + /// speech implementation (a local chat server, for example) decline the + /// workload rather than fabricate a response. + /// + /// # Errors + /// Returns [`ProtocolError::UpstreamConnect`] when the connection itself + /// fails, [`ProtocolError::UpstreamTransport`] on a mid-flight transport + /// failure, [`ProtocolError::UpstreamStatus`] on a non-success backend + /// status, and [`ProtocolError::ModelUnavailable`] when the upstream + /// cannot serve speech at all. + async fn send_speech( + &self, + req: SpeechRequest, + _upstream_model: &str, + ) -> Result { + Err(ProtocolError::ModelUnavailable(req.model)) + } + /// Explicitly release any owned resources (for example a child process) and /// disable further recovery, surfacing any teardown failure. /// @@ -157,6 +204,12 @@ pub struct OpenAiUpstream { /// whole-request timeout covers the body read and would kill any /// long-lived SSE stream, so streams never use `http`. http_stream: reqwest::Client, + /// Client for the speech path: like `http_stream` it carries no + /// whole-request timeout, and it adds TCP keepalive so a silently dead + /// peer surfaces. It carries no `read_timeout`: reqwest arms that during + /// the wait for response headers, so the speech path applies its own + /// split deadlines in `send_speech` instead. + http_audio: reqwest::Client, } impl OpenAiUpstream { @@ -168,6 +221,7 @@ impl OpenAiUpstream { api_key, http: crate::http_util::bounded_client(), http_stream: crate::http_util::streaming_client(), + http_audio: crate::http_util::audio_streaming_client(), } } @@ -183,7 +237,8 @@ impl OpenAiUpstream { base_url: base_url.trim_end_matches('/').to_string(), api_key, http: http.clone(), - http_stream: http, + http_stream: http.clone(), + http_audio: http, } } @@ -339,6 +394,27 @@ pub fn sse_chunks(response: reqwest::Response, requested: String) -> StreamedChu } } +/// Deadline for the upstream's first response bytes (headers) on the speech +/// path: a maximum-length batch generation can legitimately take longer to +/// first byte than the per-read body idle budget, so time-to-headers carries +/// its own, larger budget. reqwest's `read_timeout` cannot express the split +/// because it also governs the header wait, so the deadline is applied here +/// rather than on the client. +#[cfg(not(test))] +const FIRST_RESPONSE_TIMEOUT: Duration = Duration::from_secs(120); +/// Test-scaled so the deadline-separation tests run in milliseconds: the +/// stalled-headers arm waits on this budget, so it cannot stay at 120 s. +#[cfg(test)] +const FIRST_RESPONSE_TIMEOUT: Duration = Duration::from_millis(1000); + +/// Per-read idle deadline on an opened audio body: a stalled upstream is +/// detected within this window without capping the stream's total length. +#[cfg(not(test))] +const AUDIO_READ_TIMEOUT: Duration = Duration::from_secs(30); +/// Test-scaled so the deadline-separation tests run in milliseconds. +#[cfg(test)] +const AUDIO_READ_TIMEOUT: Duration = Duration::from_millis(200); + #[async_trait] impl Upstream for OpenAiUpstream { async fn send( @@ -395,6 +471,51 @@ impl Upstream for OpenAiUpstream { .await?; Ok(sse_chunks(response, requested)) } + + async fn send_speech( + &self, + mut req: SpeechRequest, + upstream_model: &str, + ) -> Result { + req.model = upstream_model.to_string(); + // Time-to-headers and per-read body idle are separate budgets: the + // send await is bounded by FIRST_RESPONSE_TIMEOUT, each body read by + // AUDIO_READ_TIMEOUT. + let response = tokio::time::timeout( + FIRST_RESPONSE_TIMEOUT, + self.post(&self.http_audio, "audio/speech", &req), + ) + .await + .map_err(ProtocolError::transport)??; + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + .unwrap_or_default(); + let body = futures_util::stream::unfold( + (response.bytes_stream().boxed(), false), + |(mut bytes, terminated)| async move { + if terminated { + return None; + } + match tokio::time::timeout(AUDIO_READ_TIMEOUT, bytes.next()).await { + Ok(Some(Ok(chunk))) => Some((Ok(chunk), (bytes, false))), + Ok(None) => None, + // A mid-stream transport failure or a read idle timeout + // surfaces as one Err item and ends the stream, so a + // caller never mistakes a truncated stream for a + // complete one. + Ok(Some(Err(error))) => { + Some((Err(ProtocolError::upstream_transport(error)), (bytes, true))) + } + Err(elapsed) => Some((Err(ProtocolError::transport(elapsed)), (bytes, true))), + } + }, + ) + .boxed(); + Ok(StreamedAudio { content_type, body }) + } } #[cfg(test)] @@ -1037,4 +1158,374 @@ mod tests { ); let _ = handle.join(); } + + fn speech_request(model: &str) -> SpeechRequest { + SpeechRequest { + model: model.to_owned(), + input: "hello world".to_owned(), + voice: crate::wire::SpeechVoice::Name("tara".to_owned()), + response_format: crate::wire::SpeechResponseFormat::Mp3, + speed: None, + instructions: None, + stream_format: None, + rest: Map::new(), + } + } + + /// A one-shot mock audio backend: serves a single canned binary body with + /// the given `Content-Type` (or none) and returns its base URL plus the + /// captured raw request for assertions. + fn serve_audio(content_type: Option<&str>, body: &[u8]) -> (String, JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock audio backend"); + let addr = listener.local_addr().expect("addr"); + let content_type = content_type.map(str::to_owned); + let body = body.to_vec(); + let handle = thread::spawn(move || -> String { + let (mut stream, _) = listener.accept().expect("accept"); + // Same bounded-capture pattern as serve_once: the read timeout ends + // the wait once the client is awaiting a response. + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let mut request = Vec::new(); + let mut buf = [0_u8; 4096]; + loop { + match stream.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => request.extend_from_slice(&buf[..n]), + } + } + let content_type = content_type + .map(|value| format!("Content-Type: {value}\r\n")) + .unwrap_or_default(); + let head = format!( + "HTTP/1.1 200 OK\r\n{content_type}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + let _ = stream.flush(); + String::from_utf8_lossy(&request).into_owned() + }); + (format!("http://{addr}"), handle) + } + + #[tokio::test] + async fn speech_rewrites_caller_model_and_posts_to_audio_speech() { + // UP-008: same contract as chat - the upstream model is what the + // backend sees; the caller's model name never leaks into the body. + let (base, handle) = serve_audio(Some("audio/mpeg"), b"fake-mp3"); + let upstream = OpenAiUpstream::new(&base, Secret::new(String::new())); + let streamed = upstream + .send_speech(speech_request("caller-model"), "backend-tts") + .await + .expect("send ok"); + assert_eq!(streamed.content_type, "audio/mpeg"); + let sent = handle.join().expect("join"); + assert!(sent.contains("POST /audio/speech"), "{sent}"); + assert!(sent.contains("backend-tts"), "forwarded body: {sent}"); + assert!( + sent.contains("\"voice\":\"tara\""), + "forwarded body: {sent}" + ); + assert!( + !sent.contains("caller-model"), + "caller model leaked: {sent}" + ); + } + + #[tokio::test] + async fn speech_forwards_the_bearer_credential() { + // The endpoint credential rides the audio request exactly as it does + // the chat and embeddings requests. + let (base, handle) = serve_audio(Some("audio/mpeg"), b"fake-mp3"); + let upstream = OpenAiUpstream::new(&base, Secret::new("test-key".to_owned())); + let _streamed = upstream + .send_speech(speech_request("m"), "u") + .await + .expect("send ok"); + let sent = handle.join().expect("join").to_ascii_lowercase(); + assert!( + sent.contains("authorization: bearer test-key"), + "bearer forwarded: {sent}" + ); + } + + #[tokio::test] + async fn speech_non_success_status_is_upstream_status() { + // A backend 429/503 surfaces as the protocol-level UpstreamStatus + // shape with the capped body - never a client-facing envelope (the + // route owns the envelope mapping). + for (status_line, status) in [ + ("429 Too Many Requests", 429), + ("503 Service Unavailable", 503), + ] { + let (base, handle) = serve_once(status_line, "backend says no"); + let upstream = OpenAiUpstream::new(&base, Secret::new(String::new())); + let err = upstream + .send_speech(speech_request("m"), "u") + .await + .expect_err("should fail"); + match err { + ProtocolError::UpstreamStatus { status: got, body } => { + assert_eq!(got, status); + assert_eq!(body, "backend says no"); + } + other => panic!("expected UpstreamStatus {status}, got {other:?}"), + } + let _ = handle.join(); + } + } + + #[tokio::test] + async fn speech_streams_bytes_untransformed() { + // Byte passthrough: audio frames are opaque, so the body stream is the + // upstream's bytes verbatim - including invalid UTF-8, which no + // decoding layer ever touches. + let body: Vec = (0..=255).cycle().take(4097).collect(); + let (base, handle) = serve_audio(Some("audio/mpeg"), &body); + let upstream = OpenAiUpstream::new(&base, Secret::new(String::new())); + let mut streamed = upstream + .send_speech(speech_request("m"), "u") + .await + .expect("send ok"); + assert_eq!(streamed.content_type, "audio/mpeg"); + let mut collected = Vec::new(); + while let Some(item) = streamed.body.next().await { + collected.extend_from_slice(&item.expect("chunk ok")); + } + assert_eq!(collected, body, "audio bytes pass through untransformed"); + let _ = handle.join(); + } + + #[tokio::test] + async fn speech_content_type_is_empty_when_the_upstream_omits_it() { + // An upstream without a Content-Type yields an empty string, the + // route's signal to apply its format-to-MIME fallback. + let (base, handle) = serve_audio(None, b"raw-audio"); + let upstream = OpenAiUpstream::new(&base, Secret::new(String::new())); + let mut streamed = upstream + .send_speech(speech_request("m"), "u") + .await + .expect("send ok"); + assert_eq!(streamed.content_type, ""); + while let Some(item) = streamed.body.next().await { + item.expect("chunk ok"); + } + let _ = handle.join(); + } + + #[tokio::test] + async fn speech_times_out_on_a_stalled_server() { + // A backend that accepts and then stalls must fail as a transport + // error on the read deadline, never hang the caller. A timeout is + // NEVER connect: the request may have reached the provider. + let (base, handle) = serve_stalled(); + let client = reqwest::Client::builder() + .read_timeout(std::time::Duration::from_millis(300)) + .build() + .expect("client"); + let upstream = OpenAiUpstream::with_client(&base, Secret::new(String::new()), client); + let err = upstream + .send_speech(speech_request("m"), "u") + .await + .expect_err("stalled server must time out"); + assert!( + matches!(err, ProtocolError::UpstreamTransport(_)), + "expected UpstreamTransport, got {err:?}" + ); + let _ = handle.join(); + } + + /// Between the two test-scaled speech deadlines: past the per-read body + /// idle budget (200 ms) so the accept arm proves that budget does not + /// govern time-to-headers, within the first-response budget (1 s) so + /// the request is still accepted. + const SLOW_HEADER_PAUSE: Duration = Duration::from_millis(500); + + /// Past the test-scaled first-response budget (1 s) by a wide margin. + /// The mock still answers eventually, so only the deadline can fail the + /// request: removing the deadline turns the reject arm back into an + /// accepted response and the test red. + const STALLED_HEADER_PAUSE: Duration = Duration::from_millis(3000); + + /// A mock audio backend that waits `pause` after the request arrives + /// before sending any headers, then serves the canned body: the + /// slow-headers arm of the deadline-separation pair. + fn serve_slow_headers(pause: Duration, body: &[u8]) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock audio backend"); + let addr = listener.local_addr().expect("addr"); + let body = body.to_vec(); + let handle = thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + // Same bounded-capture pattern as serve_once: the read timeout + // ends the wait once the client is awaiting a response. + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let mut buf = [0_u8; 4096]; + loop { + match stream.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + thread::sleep(pause); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + let _ = stream.flush(); + }); + (format!("http://{addr}"), handle) + } + + /// A mock audio backend that sends headers and a first body chunk, then + /// holds the stream open without sending more: the stalled-body arm of + /// the deadline-separation pair. The thread exits as soon as the client + /// hangs up, bounded by a read timeout. + fn serve_chunk_then_stall(chunk: &[u8]) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock audio backend"); + let addr = listener.local_addr().expect("addr"); + let chunk = chunk.to_vec(); + let handle = thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let mut buf = [0_u8; 4096]; + loop { + match stream.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + // No Content-Length: the body stays open until close, so the + // client must detect the stall itself. + let head = "HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\n\r\n"; + if stream + .write_all(head.as_bytes()) + .and_then(|()| stream.write_all(&chunk)) + .and_then(|()| stream.flush()) + .is_err() + { + return; + } + // Hold the body open until the client hangs up (EOF or reset); + // the read timeout bounds the wait if it never does. + let _ = stream.set_read_timeout(Some(Duration::from_secs(5))); + loop { + match stream.read(&mut buf) { + Ok(0) | Err(_) => return, + Ok(_) => {} + } + } + }); + (format!("http://{addr}"), handle) + } + + #[tokio::test] + async fn speech_slow_headers_within_the_first_response_budget_are_accepted() { + // Deadline separation: headers arriving after the per-read body idle + // budget but within the first-response budget are accepted. Before + // the split, the audio client's `read_timeout` also governed the + // header wait and killed this request at the body-idle deadline. + let (base, handle) = serve_slow_headers(SLOW_HEADER_PAUSE, b"fake-mp3"); + let upstream = OpenAiUpstream::new(&base, Secret::new(String::new())); + let mut streamed = upstream + .send_speech(speech_request("m"), "u") + .await + .expect("slow headers within the first-response budget are accepted"); + assert_eq!(streamed.content_type, "audio/mpeg"); + let mut collected = Vec::new(); + while let Some(item) = streamed.body.next().await { + collected.extend_from_slice(&item.expect("chunk ok")); + } + assert_eq!(collected, b"fake-mp3"); + handle.join().expect("join"); + } + + #[tokio::test] + async fn speech_headers_stalling_past_the_first_response_budget_fail() { + // The reject arm of the first-response split: headers arriving past + // the FIRST_RESPONSE_TIMEOUT budget fail the request as a transport + // error, never hang the caller. A timeout is NEVER connect: the + // request may have reached the provider. The mock answers after + // STALLED_HEADER_PAUSE, so only the deadline can produce the error. + let (base, handle) = serve_slow_headers(STALLED_HEADER_PAUSE, b"fake-mp3"); + let upstream = OpenAiUpstream::new(&base, Secret::new(String::new())); + let err = upstream + .send_speech(speech_request("m"), "u") + .await + .expect_err("headers past the first-response budget must fail"); + assert!( + matches!(err, ProtocolError::UpstreamTransport(_)), + "expected UpstreamTransport, got {err:?}" + ); + assert_eq!(err.envelope()["error"]["code"], "upstream_transport"); + let _ = handle.join(); + } + + #[tokio::test] + async fn speech_opened_body_stalling_past_the_read_idle_budget_fails() { + // The other half of the split: once headers arrive, the per-read + // idle budget guards the body. A body that stalls past it fails the + // stream with a transport error, never a clean EOF. + let (base, handle) = serve_chunk_then_stall(b"first-chunk"); + let upstream = OpenAiUpstream::new(&base, Secret::new(String::new())); + let mut streamed = upstream + .send_speech(speech_request("m"), "u") + .await + .expect("headers arrive promptly"); + let first = streamed.body.next().await.expect("first chunk"); + assert!(first.is_ok(), "first chunk arrives: {first:?}"); + let second = streamed + .body + .next() + .await + .expect("the stall surfaces as an item"); + assert!( + matches!(second, Err(ProtocolError::UpstreamTransport(_))), + "a stalled body fails as transport, got {second:?}" + ); + assert!( + streamed.body.next().await.is_none(), + "the stream ends after the error item" + ); + // Joined off the runtime thread so the client connection task can + // run the close (same pattern as the drop-cancellation test). + drop(streamed); + tokio::task::spawn_blocking(move || handle.join().expect("join")) + .await + .expect("watch task"); + } + + #[tokio::test] + async fn default_send_speech_is_model_unavailable_and_object_safe() { + // Upstreams without a speech implementation decline the workload with + // ModelUnavailable naming the caller's model. The call goes through + // `Arc` to prove the signature stays object-safe. + struct ChatOnly; + + #[async_trait] + impl Upstream for ChatOnly { + async fn send( + &self, + _req: ChatRequest, + _upstream_model: &str, + ) -> Result { + unreachable!("not under test") + } + } + + let upstream: std::sync::Arc = std::sync::Arc::new(ChatOnly); + match upstream + .send_speech(speech_request("local-tts"), "ignored-alias") + .await + { + Err(ProtocolError::ModelUnavailable(model)) => assert_eq!(model, "local-tts"), + Err(other) => panic!("expected ModelUnavailable, got {other:?}"), + Ok(_) => panic!("default must decline"), + } + } } diff --git a/crates/shared-protocol/src/wire.rs b/crates/shared-protocol/src/wire.rs index 33a76c01..bc483c2f 100644 --- a/crates/shared-protocol/src/wire.rs +++ b/crates/shared-protocol/src/wire.rs @@ -335,6 +335,143 @@ impl EmbeddingResponse { } } +/// The largest `input` a speech request may carry, in characters (OpenAI's +/// cap, and the cap the route enforces). +const MAX_SPEECH_INPUT_CHARS: usize = 4096; + +/// The slowest accepted speech `speed` (OpenAI's lower bound). +const MIN_SPEECH_SPEED: f32 = 0.25; + +/// The fastest accepted speech `speed` (OpenAI's upper bound). +const MAX_SPEECH_SPEED: f32 = 4.0; + +/// The voice to synthesize with: a plain name or the OpenAI object form +/// (`{"id": "..."}`). Membership in a model's catalog is checked at the +/// route, never here, because voice sets are per-checkpoint. +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +#[serde(untagged)] +pub enum SpeechVoice { + /// A plain voice name. + Name(String), + /// The OpenAI object form, carrying the voice under `id`. + Id { + /// The voice identifier. + id: String, + }, +} + +/// The audio encoding a speech response is requested in (the OpenAI set). +/// +/// The set is closed on purpose: provider-only spellings (Together's `raw` +/// and `mulaw`) stay unrepresentable until the enum is deliberately widened. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum SpeechResponseFormat { + /// MPEG audio. The default: OpenAI defaults to mp3 while Together + /// defaults to wav, so the pin lives in the type and an omitted field + /// resolves to mp3 at deserialization. + #[default] + Mp3, + /// Opus in an Ogg container. + Opus, + /// AAC in an ADTS container. + Aac, + /// FLAC. + Flac, + /// Uncompressed WAV. + Wav, + /// Raw 24 kHz 16-bit signed little-endian PCM. + Pcm, +} + +/// How a streaming speech response is framed (the OpenAI set). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum SpeechStreamFormat { + /// Chunked binary audio (the behavior when the field is absent). + Audio, + /// Server-sent events carrying base64-encoded audio. + Sse, +} + +/// An incoming speech synthesis request. +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct SpeechRequest { + /// The model name, resolved against the routing table. + pub model: String, + /// The text to synthesize, at most 4096 characters. + pub input: String, + /// The voice to synthesize with. + pub voice: SpeechVoice, + /// The requested audio encoding. An omitted field resolves to `mp3` at + /// deserialization, so the pin is structural and every forwarded body + /// carries it. + #[serde(default)] + pub response_format: SpeechResponseFormat, + /// The playback speed (0.25 to 4.0); absent means the backend's default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub speed: Option, + /// Style-control instructions (the gpt-4o-mini-tts dialect's field); + /// absent means the backend's default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub instructions: Option, + /// The streaming framing selector; absent means chunked binary audio. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream_format: Option, + /// Every field the gateway does not name, preserved verbatim. + #[serde(flatten)] + pub rest: Map, +} + +impl SpeechRequest { + /// Reserved top-level keys that must never appear in the passthrough `rest`. + const RESERVED: [&'static str; 7] = [ + "model", + "input", + "voice", + "response_format", + "speed", + "instructions", + "stream_format", + ]; + + /// Validate the request shape at the trust boundary, without coercion. + /// + /// Rejects an empty model, an empty or over-cap `input`, an out-of-range + /// `speed`, and any reserved key smuggled into the flattened `rest` map + /// (WIRE-001/003). Everything else passes through verbatim. + /// + /// # Errors + /// Returns a static reason string when the model is empty, the input is + /// empty or over the character cap, the speed is out of range, or `rest` + /// collides with a named field. + pub fn validate(&self) -> Result<(), &'static str> { + if self.model.trim().is_empty() { + return Err("model must not be empty"); + } + if self.input.is_empty() { + return Err("input must not be empty"); + } + if self.input.chars().count() > MAX_SPEECH_INPUT_CHARS { + return Err("input must not exceed 4096 characters"); + } + if let Some(speed) = self.speed + && !(MIN_SPEECH_SPEED..=MAX_SPEECH_SPEED).contains(&speed) + { + return Err("speed must be between 0.25 and 4.0"); + } + if Self::RESERVED + .iter() + .any(|key| self.rest.contains_key(*key)) + { + return Err( + "rest must not contain a reserved key (model, input, voice, response_format, speed, instructions, stream_format)", + ); + } + Ok(()) + } +} + /// An incoming rerank request (the llama-server/vLLM/Jina shape: a query and /// a document set in, ranked relevance scores out). #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] @@ -452,7 +589,8 @@ pub struct ModelInfo { pub id: String, /// Always `"model"`. pub object: &'static str, - /// The workload this model serves (`"chat"`, `"embedding"`, `"classifier"`). + /// The workload this model serves (`"chat"`, `"embedding"`, + /// `"classifier"`, `"speech"`). pub kind: ModelKind, /// Prose describing the model for catalog consumers and semantic bind. pub description: String, @@ -873,6 +1011,187 @@ mod tests { assert!(resp.validate().is_err()); } + fn speech_request(model: &str, input: &str) -> SpeechRequest { + SpeechRequest { + model: model.to_owned(), + input: input.to_owned(), + voice: SpeechVoice::Name("tara".to_owned()), + response_format: SpeechResponseFormat::Mp3, + speed: None, + instructions: None, + stream_format: None, + rest: Map::new(), + } + } + + #[test] + fn speech_request_validation_table() { + assert!(speech_request(" ", "hi").validate().is_err()); + assert!(speech_request("m", "").validate().is_err()); + // The wire cap is 4096 characters: 4096 passes, 4097 fails. + assert!(speech_request("m", &"x".repeat(4096)).validate().is_ok()); + assert!(speech_request("m", &"x".repeat(4097)).validate().is_err()); + // The unit is characters, not bytes: U+00E9 is two UTF-8 bytes, so + // 4096 of them are 8192 bytes and still pass, 4097 fail. + assert!( + speech_request("m", &"\u{e9}".repeat(4096)) + .validate() + .is_ok() + ); + assert!( + speech_request("m", &"\u{e9}".repeat(4097)) + .validate() + .is_err() + ); + // Speed outside 0.25..=4.0 is rejected; both bounds are accepted. + let too_slow = SpeechRequest { + speed: Some(0.24), + ..speech_request("m", "hi") + }; + assert!(too_slow.validate().is_err()); + let too_fast = SpeechRequest { + speed: Some(4.01), + ..speech_request("m", "hi") + }; + assert!(too_fast.validate().is_err()); + let at_bounds = SpeechRequest { + speed: Some(0.25), + ..speech_request("m", "hi") + }; + assert!(at_bounds.validate().is_ok()); + let at_top = SpeechRequest { + speed: Some(4.0), + ..speech_request("m", "hi") + }; + assert!(at_top.validate().is_ok()); + assert!(speech_request("m", "hi").validate().is_ok()); + } + + #[test] + fn speech_request_rejects_unknown_response_format() { + // The format set is closed: an unknown spelling fails deserialization, + // so no route can forward one. Together's `raw` is a known provider + // spelling deliberately excluded from the enum; covering it here pins + // the exclusion rather than leaving it incidental. + for format in ["ogg", "raw"] { + let json = serde_json::json!({ + "model": "m", + "input": "hi", + "voice": "tara", + "response_format": format, + }); + assert!( + serde_json::from_value::(json).is_err(), + "response_format {format:?} must be rejected" + ); + } + } + + #[test] + fn speech_request_rejects_unknown_stream_format() { + // The framing set is closed: an unknown spelling fails deserialization, + // so no route can forward one. + let json = serde_json::json!({ + "model": "m", + "input": "hi", + "voice": "tara", + "stream_format": "chunked", + }); + assert!(serde_json::from_value::(json).is_err()); + } + + #[test] + fn speech_request_defaults_response_format_to_mp3() { + // The mp3 pin is structural: an omitted field resolves at + // deserialization and always serializes back onto the wire. + let json = serde_json::json!({ + "model": "m", + "input": "hi", + "voice": "tara", + }); + let req: SpeechRequest = serde_json::from_value(json).expect("parse request"); + assert_eq!(req.response_format, SpeechResponseFormat::Mp3); + assert_eq!( + serde_json::to_value(&req) + .expect("serialize") + .get("response_format") + .and_then(Value::as_str), + Some("mp3") + ); + } + + #[test] + fn speech_request_round_trips_both_voice_forms() { + let string_form: SpeechRequest = serde_json::from_value(serde_json::json!({ + "model": "m", + "input": "hi", + "voice": "tara", + })) + .expect("parse request"); + assert_eq!(string_form.voice, SpeechVoice::Name("tara".to_owned())); + let object_form: SpeechRequest = serde_json::from_value(serde_json::json!({ + "model": "m", + "input": "hi", + "voice": { "id": "tara" }, + })) + .expect("parse request"); + assert_eq!( + object_form.voice, + SpeechVoice::Id { + id: "tara".to_owned() + } + ); + for req in [string_form, object_form] { + let reparsed: SpeechRequest = + serde_json::from_value(serde_json::to_value(&req).expect("serialize")) + .expect("reparse"); + assert_eq!(req, reparsed); + } + } + + #[test] + fn speech_request_preserves_unnamed_fields_verbatim() { + let json = serde_json::json!({ + "model": "m", + "input": "hi", + "voice": "tara", + "response_format": "wav", + "speed": 1.5, + "instructions": "speak cheerfully", + "stream_format": "sse", + "sample_rate": 24000, + }); + let req: SpeechRequest = serde_json::from_value(json).expect("parse request"); + assert_eq!(req.response_format, SpeechResponseFormat::Wav); + assert_eq!(req.speed, Some(1.5)); + assert_eq!(req.instructions.as_deref(), Some("speak cheerfully")); + assert_eq!(req.stream_format, Some(SpeechStreamFormat::Sse)); + // Unnamed fields land in `rest`, not on named fields. + assert!(req.rest.contains_key("sample_rate")); + for key in [ + "model", + "input", + "voice", + "response_format", + "speed", + "instructions", + "stream_format", + ] { + assert!(!req.rest.contains_key(key)); + } + let reparsed: SpeechRequest = + serde_json::from_value(serde_json::to_value(&req).expect("serialize")) + .expect("reparse"); + assert_eq!(req, reparsed); + } + + #[test] + fn speech_request_rejects_reserved_keys_in_rest() { + let mut req = speech_request("m", "hi"); + req.rest.insert("input".to_owned(), serde_json::json!("y")); + assert!(req.validate().is_err()); + } + fn rerank_request(model: &str) -> RerankRequest { RerankRequest { model: model.to_owned(), diff --git a/design/design-gateway-tts-phase-1.md b/design/design-gateway-tts-phase-1.md new file mode 100644 index 00000000..0f44759d --- /dev/null +++ b/design/design-gateway-tts-phase-1.md @@ -0,0 +1,60 @@ +# Gateway Speech Synthesis, Phase 1, As Built: A Routed Speech Kind with Remote Passthrough + +## Executive summary + +The PromptForge gateway synthesizes speech. A catalog model declared with `kind = "speech"` is routed like any chat, embedding, or classifier model, and a client with a bearer token calls `POST /v1/audio/speech` with an OpenAI-shaped request (`model`, `input`, `voice`, optional `response_format`, `speed`, `instructions`, `stream_format`) and receives the provider's audio as a chunked binary stream. A companion route, `GET /v1/audio/voices`, answers the deduplicated union of every speech model's configured voices so clients can discover what they may ask for before synthesizing. Phase 1 is remote passthrough: the gateway holds the provider credential, substitutes the upstream model name, forwards the request verbatim, and streams the audio bytes back unread. Together AI serving Orpheus 3B is the live-probe backend; any OpenAI-shaped speech provider works with no further code. The local Orpheus engine is deliberately not part of this phase and is gated on an engine spike. + +The build landed as nine commits on branch `add-tts-phase-1`, executing the plan in `vibe/2026-09-07-2-gateway-tts-phase-1.md`. The pre-build specification lived in `design/report-gateway-tts-endpoint.md`; that report was removed from this repository when design records moved out (`cc7faa6a`), and this as-built supersedes it. The eight preceding commits are `d092527b`, `e10cc8dd`, `36065e56`, `37e4b166`, `2ed256f7`, `5cba784e`, `cdd6e8cd`, and `d3cf7b60`; this document lands as the ninth. Live-provider verification is deferred to the Phase 3 rerun. `design/note-gateway-tts-phase-1-verification.md` records the probe contract now and will record that run's provenance (commit and tree hash) when the rerun happens. + +## Key design choices + +**1. Speech is a fourth routed model kind, not an STT-style engine slot.** `ModelKind::Speech` (serde and `Display` spelling `"speech"`) joined the kind enum in `crates/gateway-config/src/config.rs`, so a speech model is an ordinary catalog entry: routable by name, visible in `GET /v1/models`, admitted through the dominion queue, and guarded per route by `require_kind`. The report's Pattern B alternative, an engine-slot `[[tts_model]]` catalog mirroring STT, was rejected because STT's defining constraint (audio never leaves the machine) does not apply to synthesis, and the slot pattern has no endpoint, credential, or upstream concept and no routing-table visibility. The Decision Record carries this as the plan's second decision, and the finished work matches it: speech models flow through the same routing, queue, and catalog machinery as every other kind. + +**2. The feature was rebuilt fresh; the earlier branch was discarded, not ported.** Branch `gate-way-tts-phase-1` already carried a complete phase-1 implementation (~2,750 lines) built around a `gateway-tts` service crate when the plan was written. The operator chose to discard it and rebuild on what became `add-tts-phase-1`; the branch remains in the repository untouched as a reference, and none of its code was ported. The Decision Record's first entry documents the fork choice and the later base move from master to `add-tts-phase-1` (which carries PR #18's STT rework); the discard decision survived the rebase unchanged. + +**3. The handlers live inline in the gateway crate; no service crate exists.** `audio_speech` and `audio_voices` sit in `crates/gateway/src/lib.rs` beside the embeddings and rerank handlers, registered unconditionally in `build_router`. The branch's `gateway-tts` crate split was rejected with the rebuild: the embeddings handler is the template, no module ceiling governs lib.rs (only workshop-server carries one), and a crate split adds machinery without a boundary to enforce. The revisit condition stands as recorded: if lib.rs gains a module ceiling or phase 2 gives speech a lifecycle to isolate, the split can be reconsidered. + +**4. Authentication runs before body extraction.** The report asked for the transcription handler's auth-first ordering, but PR #18 had moved the STT routes into gateway-stt behind the `authorize_stt_route` middleware, so there was no in-crate transcription ordering left to match. The built mechanism is the one the Decision Record names: `audio_speech` takes the ungated `Caller` parts-extractor (`crates/gateway/src/auth.rs`) and a raw `Request`, runs `check_auth`, and only then extracts `Json` by hand, mapping the rejection to `malformed_request`. An unauthorized caller never makes the gateway parse a body, and the `speech_auth_tests` module pins the 401-before-400 order through the real router. The extraction re-added an ungated `FromRequest` import, acceptable because axum is an unconditional dependency and the `cargo check -p gateway --no-default-features` headless gate stays green. + +**5. Voice validation precedes queue admission.** The report sequenced the voice check after queue admission; the built handler validates the requested voice against the model's catalog `voices` list before touching the dominion queue, because a 400 must not burn a queue slot. The Decision Record records this as an explicit deviation, and the integration suite pins it: a voice rejection under a full pool never reaches the backend. An empty or absent `voices` list stays valid and skips the check, so providers with no fixed voice catalog keep working. + +**6. The `voices` list is speech-only and content-validated at load.** `validate_kind_scope` (`crates/gateway-config/src/config/validate.rs`) rejects a non-empty `voices` list on any non-speech kind with an error naming the field, symmetric with the chat-only-field discipline, so a stale or misplaced list fails loudly at load instead of applying silently to the wrong kind. A list that passes is content-validated in `validate_capabilities` beside the `effort_levels` checks: no empty entries, no duplicates. The Decision Record carries this as its own entry, and the finished work matches it: `rejects_voices_on_non_speech_models`, `rejects_empty_voice_entries`, and `rejects_duplicate_voices` pin the three rejections, while `accepts_speech_model_with_empty_or_absent_voices` pins the empty list that stays valid and skips the route voice check. + +**7. The wire type pins the contract structurally.** `SpeechRequest` in `crates/shared-protocol/src/wire.rs` names seven fields and flattens every unnamed field into a `rest` map that rides to the provider verbatim, with a `RESERVED` list keeping the seven known keys out of the passthrough. Three sub-decisions, each in the Decision Record, landed as types rather than conventions. `voice` is an untagged string-or-`{"id"}` enum (`SpeechVoice`), so OpenAI's object form stays representable while membership validation stays at the route against per-model catalog data, because voice sets are per-checkpoint and never a shared constant. `response_format` is a closed enum whose `#[serde(default)]` resolves an omitted field to mp3 at deserialization: OpenAI defaults to mp3 while Together defaults to wav, so an unpinned default would silently change the wire for a Together-backed model, and pinning in the type means no route can forget it. Together-only `raw` and `mulaw` spellings fail deserialization and stay unrepresentable until the enum is deliberately widened. `validate()` returns `&'static str`, mirroring `ChatRequest::validate` exactly; it rejects an empty model, empty or over-cap input (4096 characters, matching OpenAI's limit), out-of-range speed (0.25-4.0), and reserved keys smuggled into `rest`. The report's one hard rule held throughout: nothing on the path sanitizes angle-bracket content. The offline suite pins `` passthrough, and the live probe asserts it through the gateway at the Phase 3 rerun. + +**8. The reply is a byte passthrough with header mapping, owned by a bounded background relay.** Speech is the one departure from the gateway's typed-relay norm: audio frames are opaque bytes that cannot be re-validated per chunk, so `relay_audio` re-emits the upstream stream unread. The response forwards the upstream `Content-Type` when present and otherwise falls back through `speech_fallback_mime`, which takes the framing selector first (`text/event-stream` when `stream_format` is `sse`) and only then the requested format's MIME mapping (`mp3` to `audio/mpeg`, `wav` to `audio/wav`, `pcm` to `audio/pcm`, `opus` to `audio/ogg`, `flac` to `audio/flac`, `aac` to `audio/aac`). Together SSE is unrequestable: that dialect needs `response_format = "raw"`, which the wire enum rejects, so phase-1 Together speech is non-streaming and `stream_format` forwarding is forward-looking for SSE-capable OpenAI-compatible providers. `Content-Length` is never set, so hyper emits `Transfer-Encoding: chunked`. + +The permit's lifetime does not ride the HTTP body's Drop chain. A spawned Tokio task (`relay_speech_stream`) owns the upstream body, the dominion permit, and the `InFlightGuard` cancellation guard, and feeds a bounded channel to the HTTP body: `SPEECH_RELAY_CHANNEL_CAPACITY` data slots plus one reserved terminal-error slot, reserved up front so delivering the error never waits on a stalled downstream. Four named static constants bound the stream, cfg(test)-scaled so the boundary tests run in milliseconds: `SPEECH_RELAY_TOTAL_LIFETIME` (60 min), `SPEECH_RELAY_BYTE_CEILING` (1 GiB), `SPEECH_RELAY_UPSTREAM_IDLE` (30 s, after headers; time-to-headers is the upstream layer's first-response budget), and `SPEECH_RELAY_DOWNSTREAM_BLOCKED` (60 s). Every terminal path (a bound tripped, an upstream body error, a profile-switch cancellation, the downstream gone) emits exactly one `Err` item through the reserved slot, then drops the body, the permit, and the guard together. A clean upstream end inside every bound is the one exit with no error item. Profile-switch cancellation synthesizes a body error (`relay_terminal("request cancelled for profile switch")`); it never surfaces as a clean EOF, the same fail-rather-than-truncate trade `relay_sse` makes with its `RequestCancelled` envelope. Over the wire a body-stream error aborts the response, so the item's message is server-side diagnostics and the client observes a failed read. The handler's doc comment carries the standing warning that the route must never sit under a `CompressionLayer` or whole-request `TimeoutLayer`. + +**9. Audio streams get their own HTTP client, and both deadlines live in `send_speech`.** The shared `streaming_client()` is connect-timeout-only because it also serves chat SSE, where a per-read timeout could kill long thinking pauses. Speech instead got `audio_streaming_client()` in `crates/shared-protocol/src/http_util.rs`: connect timeout 10 s and 60 s TCP keepalive, and deliberately no `read_timeout`. A 4,096-character batch generation can legitimately exceed 30 s to first headers, and reqwest 0.12 arms a client-level `read_timeout` during the header wait (`PendingRequest::poll`), which would cap time-to-headers at the body-idle budget. Both budgets therefore live in `OpenAiUpstream::send_speech`: `FIRST_RESPONSE_TIMEOUT` (~120 s, cfg(test)-scaled) wraps the `post` that waits for headers, and `AUDIO_READ_TIMEOUT` (30 s, cfg(test)-scaled) wraps each read of the opened body. `OpenAiUpstream` carries the client as a third field (`http_audio`) beside the chat and SSE clients, and the chat SSE client is untouched. + +**10. Upstream 429 and 503 map to distinct envelopes on the speech path only.** `ProtocolError::classify` renders an upstream 429 as `upstream_client_error` and a 503 as a 502 `upstream_error`, and it and its table test stayed frozen, so chat, embedding, and rerank envelopes are bit-identical. The seam lives in the gateway's own error type: `GatewayError::UpstreamRateLimited` (429 `rate_limit_error`/`upstream_rate_limited`) and `GatewayError::UpstreamUnavailable` (503 `server_error`/`upstream_unavailable`) sit beside `QueueRejected` in `crates/gateway/src/error.rs`, the gateway's exhaustive `classify()` makes the new arms compiler-forced, and the `audio_speech` handler matches `ProtocolError::UpstreamStatus` into them while forwarding every other error through the unchanged `Protocol` arm. New `ProtocolError` variants were rejected because shared-protocol's exhaustive `classify()` cannot compile them without editing the frozen function, and they would offer the codes to every route. A no-leak integration test pins the negative half: an upstream error body carrying provider internals never reaches the client. + +**11. The voices union route reads the live routing table and leads with the id.** `GET /v1/audio/voices` authenticates through `check_auth`, holds the publication lock while reading live routing, and collects every speech model's configured voices into a `BTreeSet`, so the answer is deduplicated and sorted and never calls an upstream. OpenAI has no voice-list route, but the OpenAI-compatible ecosystem (Kokoro-FastAPI, vLLM-Omni, Fish Audio) converged on one whose entries lead with the voice identifier, so each entry is an id-first `{"id", "name"}` object with `name` mirroring `id`, the catalog configuring voices as bare strings with no separate display name. The entry shape is a compatibility surface and is pinned on the raw response body; tolerant clients also accept plain strings, which the guide documents. The live probe asserts the union shape through the gateway at the Phase 3 rerun. + +**12. Building launch options is fallible, unknown kinds are refused, and the kind check runs before any provisioning side effect.** `launch_options` in `crates/gateway-local/src/runtime.rs` previously mapped serve mode through a `_ => ServeMode::Chat` catch-all, which would have compiled clean and launched a `kind = "speech"` local model as a chat server. The kind mapping is a side-effect-free `serve_mode_for(kind) -> Result`; `launch_options` calls it and returns `Result`. The speech arm and the retained wildcard both fail with `LocalError::UnsupportedKind`, which carries the offending kind and renders "local {kind} models are not yet supported". The wildcard stays because `ModelKind` is `#[non_exhaustive]`: a kind added later fails loudly instead of inheriting the chat default. `start_impl` and `provision_artifacts_impl` run `serve_mode_for` before any server, model, companion, or cache side effect (A6), so an all-speech profile touches neither the server provisioner nor the model store; a mixed profile provisions only supported models and records a per-model failure for each unsupported kind. No `ServeMode::Speech` arm exists, because no local speech runtime exists yet; the error is the entire launch behavior for the kind, and the guide's local-models chapter says so. The `tool_dialect` wildcard stays with its deliberate chat default, harmless for a kind that never launches. + +**13. Live verification is a repeatable gateway-only Node probe, not a one-shot manual check.** `tools/gateway-tts-live.mjs` is a zero-dependency Node script (built-in `fetch`, paired `tools/gateway-tts-live.test.mjs` covering startup failure, readiness timeout, request failure, assertion failure, process cleanup, key-absent skip, and secret-free output). It always runs `cargo build -p gateway` first so a stale binary can never be tested against a recorded current hash, then boots the fresh binary on an ephemeral loopback port with a throwaway Together-backed profile and asserts the speech and voices surfaces through the gateway's own responses. The vendor key comes from the process environment only (no dotenv parsing) and is ferried only to the gateway subprocess; the throwaway config carries the `api_key = "${TOGETHER_API_KEY}"` interpolation and no secret material. The script never calls a vendor directly (A19) and never prints or persists the key. With no key in the environment it prints a skip and exits 0 without building or touching the network. It never runs in CI. It deliberately never provokes the 429/503 envelopes on a paid provider, exercises no voice rejection or kind mismatch, and reads every response to completion; those behaviors stay with the Rust integration suite. The live run's provenance is recorded in `design/note-gateway-tts-phase-1-verification.md` at the Phase 3 rerun. + +**14. Config examples follow the shipped schema's credential spelling.** The report's example entries used `secret = "env:TOGETHER_API_KEY"`, which predates the config schema. The built examples, in `gateway.local.example.toml`, the gateway README, the guide, and the live probe's throwaway config, all use `api_key = "${TOGETHER_API_KEY}"` with the schema's `${VAR}` interpolation. The Decision Record names this a conformance fix, not a design change. + +**15. The config UI speaks speech; the voice picker does not.** `speech` joined the Kind dropdown options in the config UI (`models-view.ts`), and `voices` got a chips editor in the settings registry that renders only when the model kind is speech, the same control class as `effort_levels`; the round-trip into the PUT body is test-pinned. A request-time voice picker stayed a non-goal, and Discover's hardcoded `kind: "chat"` filter is a named gap left for the discover flow's own pass, exactly as the Decision Record scoped it. + +**16. Documentation landed with the feature, generated artifacts included.** The guide gained `guide/src/gateway/06-speech-synthesis.md`, inserted at number 06 with the five following chapters moved one number up as pure renames and `SUMMARY.md` regenerated by the `build-user-guide` assembler, never hand-edited. The gateway README gained a "Speech synthesis models" section documenting the route dialect; the kind lists in the remote and local model chapters include `speech`; the example configuration carries a commented speech `[[model]]` block; and the compiled single-file guide was regenerated in the same change so it mirrors the source verbatim. + +## Deferred to phase 2 + +The plan explicitly defers these, and they remain open: + +- The local speech engine: a spike choosing between a managed CrispASR child and llama-server plus in-process SNAC decode, scored on time-to-first-audio, real-time factor under concurrent load, and an ASR-roundtrip quality gate. Phase 2's engine follows the gateway-stt crate trio as its structural template, and `ServeMode::Speech` lands with the winner. +- Encoder and WAV-header policy for locally generated audio. +- Speech-model sampling-default pins (the report's Risks recipe targets Orpheus generation): no remote speech dialect carries sampling fields, so the pin lands with the local engine that actually generates. +- Together SSE reframing: Together's streaming mode needs `response_format = "raw"`, which the wire enum rejects, so Together SSE is unrequestable in phase 1 and `stream_format` forwarding is forward-looking for SSE-capable OpenAI-compatible providers. +- Workshop playback of synthesized audio. +- The config-UI voice picker. +- Technical-text conditioning before synthesis. +- ElevenLabs and Baseten adapters, both structurally divergent from the OpenAI shape. + +--- + +*2026-09-10 - Cursor Grok 4.6 (Cursor agent)* diff --git a/design/note-gateway-tts-phase-1-verification.md b/design/note-gateway-tts-phase-1-verification.md new file mode 100644 index 00000000..3f27b4ba --- /dev/null +++ b/design/note-gateway-tts-phase-1-verification.md @@ -0,0 +1,41 @@ +# Gateway TTS phase 1: live-provider verification note + +## Status + +Live verification is deferred to the Phase 3 rerun; no run has been recorded yet. The live probe is `tools/gateway-tts-live.mjs`, a dev-only Node script (zero dependencies, built-in `fetch`, never in CI) that builds the gateway fresh, boots it with a throwaway Together-backed profile, and asserts the speech and voices surfaces through the gateway's own responses. This note records the probe's contract now and its findings after the rerun. Behavior and wire shapes only: no credential material appears in this note, and none is written to disk by a run. + +## Provenance + +Recorded at the Phase 3 live rerun: + +- Commit: +- Tree hash: + +## Run boundary + +- Command: `node tools/gateway-tts-live.mjs`. Exit 0 with `LIVE OK` when every assertion passes; with no `TOGETHER_API_KEY` in the process environment the script prints a skip and exits 0 without building or touching the network. +- The script always runs `cargo build -p gateway` first, so the probed binary always matches the recorded commit; a stale binary can never be tested against a current hash. It then boots the gateway on an ephemeral loopback port with a throwaway config in a temp directory: one `[[endpoint]]` for `https://api.together.xyz/v1` with `api_key = "${TOGETHER_API_KEY}"`, one `kind = "speech"` model named `orpheus` upstreaming to `canopylabs/orpheus-3b-0.1-ft` with the eight Orpheus voices configured, and a throwaway `tts-live` profile selected via `--profile`. +- Credential invariant A19 (`vibe/archdoc.md`: keep vendor and remote-service credentials inside the gateway): the script takes the vendor key from the process environment only (no dotenv parsing, no `.env` reading) and ferries it only to the gateway subprocess environment, where the config's `${TOGETHER_API_KEY}` interpolation resolves it. The script never calls a vendor directly and never prints or persists the key. +- Calls, all through the gateway: `POST /v1/audio/speech` with default format, with `wav`, and with an emotion-tag input; `GET /v1/audio/voices`; three field-tolerance probes. +- Probe input: a single English sentence (~90 characters). + +## Assertions (each fails the run) + +- Default-format speech call returns `Content-Type: audio/mpeg` with a byte-nonempty mp3 body (ID3 tag or frame-sync magic). Together's own documented default is wav, so an mp3 answer through the gateway proves the structural `response_format = "mp3"` pin reached the provider. +- The response is streamed: no `Content-Length`, matching the chunked relay contract. +- `response_format = "wav"` returns `Content-Type: audio/wav` with a byte-nonempty RIFF/WAVE body. +- An input carrying `` returns 200 with audio: angle-bracket emotion tags pass through the gateway untouched. +- `GET /v1/audio/voices` returns 200 with `{"voices": [...]}` holding the eight configured voices as sorted `{"id", "name"}` objects with `name` mirroring `id`. +- The named optional `instructions` field is forwarded and tolerated with a 2xx, and fields outside the gateway's named wire set (`sample_rate`, a bogus `promptforge_probe`) ride the verbatim passthrough and come back 2xx. + +## Observed dialect (filled in at the Phase 3 live rerun) + +- Default format, framing, and byte counts: +- Emotion-tag handling: +- Rejected or ignored fields: +- 429/503 envelopes: not provoked unless the rerun finds a cost-free trigger; provoking a rate limit on a paid provider for observation is not worth the cost. The gateway's distinct `UpstreamRateLimited`/`UpstreamUnavailable` mappings stay covered by the Rust integration suite regardless. + +## Not exercised live + +- Voice rejection (a voice outside the catalog's `voices` list earns a 400 naming the valid set) and kind mismatch are covered by the gateway integration suite, not this probe. +- Mid-stream disconnect cancellation is covered by the integration suite; the probe reads every response to completion. diff --git a/gateway.local.example.toml b/gateway.local.example.toml index 10e188fd..298ae56d 100644 --- a/gateway.local.example.toml +++ b/gateway.local.example.toml @@ -91,6 +91,22 @@ vram_gb = 24 # default_effort = "medium" # adaptive_thinking = false +# Optional remote speech synthesis model, backed by its own endpoint. +# [[endpoint]] +# id = "together" +# protocol = "openai" +# base_url = "https://api.together.xyz/v1" +# api_key = "${TOGETHER_API_KEY}" +# +# [[model]] +# name = "orpheus" +# kind = "speech" +# description = "Orpheus 3B conversational speech synthesis" +# upstream = "canopylabs/orpheus-3b-0.1-ft" +# endpoints = ["together"] +# context = 8192 +# voices = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"] + # Local chat model served by a managed llama-server child. [[local_model]] name = "qwen-local" diff --git a/guide/promptforge-gateway-guide.md b/guide/promptforge-gateway-guide.md index 17a8a28a..a2c185b4 100644 --- a/guide/promptforge-gateway-guide.md +++ b/guide/promptforge-gateway-guide.md @@ -237,7 +237,7 @@ Every remote model must list at least one endpoint, and every endpoint it names ## Kinds and thinking modes -Every model carries a `kind`: `chat`, `embedding`, or `classifier`. The kind scopes which fields are meaningful. Chat-only fields such as `thinking` and `default_max_tokens` are rejected for non-chat kinds at load time. +Every model carries a `kind`: `chat`, `embedding`, `classifier`, or `speech`. The kind scopes which fields are meaningful. Chat-only fields such as `thinking` and `default_max_tokens` are rejected for non-chat kinds at load time. Record each chat model's thinking behavior as `never`, `always`, or `switchable`. Switchable means the client may toggle thinking per request. @@ -268,7 +268,7 @@ adaptive_thinking = true The capability fields are `max_output`, `default_temperature`, `images`, `parallel_tool_calls`, `effort_levels`, `default_effort`, and `adaptive_thinking`. They obey cross-field rules at load time. A `default_effort` without `effort_levels` fails. A `default_effort` not listed in `effort_levels` fails. Effort fields fail when thinking is `never`. A `max_output` larger than `context` fails; an exact fit passes. -Enumerated fields accept a fixed spelling vocabulary. Use the spellings verbatim: protocol `openai`; thinking `never`, `always`, or `switchable`; tool_dialect `openai` or `gemma3_tool_code`; model kind `chat`, `embedding`, or `classifier`. +Enumerated fields accept a fixed spelling vocabulary. Use the spellings verbatim: protocol `openai`; thinking `never`, `always`, or `switchable`; tool_dialect `openai` or `gemma3_tool_code`; model kind `chat`, `embedding`, `classifier`, or `speech`. ## What the caller sees @@ -327,7 +327,7 @@ Local inference runs on a pinned llama-server build, b10082. The gateway prefers The gateway runs one managed llama-server child per configured `[[local_model]]`. Children get supervised respawn and deterministic teardown. Staged CUDA bundle directories are prepended to the child process's PATH only; the gateway's own environment is never mutated. Local models appear to clients as ordinary routed models under their configured names. -A local model's `kind` selects the child's serving mode: embedding models serve embeddings, and classifier models serve reranking. The `parallel` key sets both the child's concurrency and its admission limit. The thinking setting changes the child's sampling preset: thinking models sample at temperature 1.0 and top-p 0.95, while non-thinking models run with reasoning switched off and sample at 0.7 and 0.8. +A local model's `kind` selects the child's serving mode: embedding models serve embeddings, and classifier models serve reranking. A `speech` kind has no local serving mode and is refused at launch: local speech models are not yet supported. The `parallel` key sets both the child's concurrency and its admission limit. The thinking setting changes the child's sampling preset: thinking models sample at temperature 1.0 and top-p 0.95, while non-thinking models run with reasoning switched off and sample at 0.7 and 0.8. ## Chat templates @@ -462,6 +462,66 @@ Speech loads exactly once per process, from the profile active at boot. Switchin --- +# Speech Synthesis + +This chapter teaches you the gateway's speech synthesis surface: how to declare a speech model, how to call the synthesis route, and how to enumerate voices. Synthesis builds on remote models, because the gateway routes speech to remote providers only; a `[[local_model]]` with `kind = "speech"` is refused at launch. + +## Declare a speech model + +A speech synthesis model is an ordinary `[[model]]` entry with `kind = "speech"`, backed by an ordinary `[[endpoint]]`: + +```` +[[endpoint]] +id = "together" +protocol = "openai" +base_url = "https://api.together.xyz/v1" +api_key = "${TOGETHER_API_KEY}" + +[[model]] +name = "orpheus" +kind = "speech" +description = "Orpheus 3B conversational speech synthesis" +upstream = "canopylabs/orpheus-3b-0.1-ft" +endpoints = ["together"] +context = 8192 +voices = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"] +```` + +The entry carries the usual remote-model fields, and the kind scopes which of them are meaningful. Chat-only fields such as `thinking`, the effort knobs, `default_max_tokens`, and `tool_dialect` are rejected on a speech model at load time. The speech-only `voices` list declares the voices the model offers: setting it on any other kind fails at load, entries must be non-empty and unique, and an empty or omitted list means the model exposes no fixed voice list, so the route accepts any voice name. The catalog advertises the kind and the voice list verbatim on GET /v1/models, so clients can shape requests before sending them. + +## Synthesize speech + +The gateway serves OpenAI-shaped speech synthesis at POST /v1/audio/speech: + +```` +curl -H "Authorization: Bearer $GATEWAY_KEY" http://127.0.0.1:8081/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model": "orpheus", "input": "The quick brown fox.", "voice": "tara"}' \ + -o speech.mp3 +```` + +The request carries `model` and `input` (both required; the input is non-empty and capped at 4096 characters), `voice` (required; a plain name or the OpenAI object form `{"id": "tara"}`), and four optional fields: `response_format` from the closed set `mp3`, `opus`, `aac`, `flac`, `wav`, `pcm`; `speed` between 0.25 and 4.0; `instructions`, the gpt-4o-mini-tts dialect's style-control string; and `stream_format`, `sse` or `audio`. An omitted `response_format` resolves to `mp3` before the request leaves the gateway: OpenAI defaults to mp3 while Together defaults to wav, so the pin lives in the wire type and every forwarded body carries it. Fields the gateway does not name pass through to the provider verbatim, so provider extras such as Together's `sample_rate` ride the same request, and angle-bracket emotion tags such as `` in the input reach the provider untouched. + +Authentication runs before the body is parsed, so a bad key earns 401 even for a malformed body. Shape failures earn 400: an empty or over-cap `input` or an out-of-range `speed` as `malformed_request`, a non-speech model as `kind_mismatch`, and a voice outside the model's declared list as `invalid_voice` naming the valid voices, judged before queue admission so the rejection never burns a queue slot. A full queue earns 503 with code `queue_full`. An upstream 429 comes back as 429 with code `upstream_rate_limited` and an upstream 503 as 503 with code `upstream_unavailable`, so an OpenAI client sees a retryable rate-limit or server error rather than a generic failure. + +The response is the provider's audio bytes streamed through unread: the gateway forwards the upstream `Content-Type` (when missing or invalid, the framing selector first, so an SSE stream is labeled `text/event-stream` and never an audio type, then the requested format's MIME type), never sets `Content-Length`, and emits `Transfer-Encoding: chunked`. No JSON error can follow 200 plus audio bytes, so a mid-stream upstream failure surfaces as a truncated body and the client's read fails. + +## List voices + +GET /v1/audio/voices answers the union of the active profile's speech voices, deduplicated and sorted: + +```` +{"voices": [{"id": "dan", "name": "dan"}, {"id": "jess", "name": "jess"}, {"id": "leah", "name": "leah"}]} +```` + +Each entry is an object with `id` first and `name` mirroring it, because the catalog configures voices as bare strings with no separate display name. OpenAI has no voice-list route; the OpenAI-compatible ecosystem converged on this one, and clients such as Open WebUI read the `id` key, so the entry shape is a compatibility surface. Tolerant clients also accept the plain-string form some servers answer with. A profile with no speech models returns an empty list, and non-speech models contribute nothing. + +## The stream_format caveat + +`stream_format = "sse"` is OpenAI's selector for event-stream framing, and the gateway forwards it verbatim like any other field. Provider dialects differ: Together spells its streaming mode `stream=true`, and only with `response_format = "raw"`, which the wire enum rejects, so Together SSE cannot be requested in phase 1 and Together speech is non-streaming. The forwarding is forward-looking, aimed at SSE-capable OpenAI-compatible providers, and the gateway does no reframing or decoding: whatever framing the provider answers with passes through untouched, so a client that receives an event stream owns decoding it itself. + +--- + # Profiles and Switching This chapter teaches you profiles: named checklists that decide which models the gateway serves, and how to switch between them at runtime. Profiles are how one config file serves a work machine, a travel laptop, and a demo box without editing a single model entry. diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index e3ed1f84..dd4d32df 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -24,11 +24,12 @@ - [Remote Models and Endpoints](gateway/03-remote-models.md) - [Local Models](gateway/04-local-models.md) - [Speech-to-Text](gateway/05-speech.md) -- [Profiles and Switching](gateway/06-profiles.md) -- [Dominions and Queues](gateway/07-dominions.md) -- [Editing Configuration Safely](gateway/08-editing-configuration.md) -- [The Configuration UI](gateway/09-config-ui.md) -- [Serving and Observing](gateway/10-serving-and-observing.md) +- [Speech Synthesis](gateway/06-speech-synthesis.md) +- [Profiles and Switching](gateway/07-profiles.md) +- [Dominions and Queues](gateway/08-dominions.md) +- [Editing Configuration Safely](gateway/09-editing-configuration.md) +- [The Configuration UI](gateway/10-config-ui.md) +- [Serving and Observing](gateway/11-serving-and-observing.md) # The Prompt Language diff --git a/guide/src/gateway/03-remote-models.md b/guide/src/gateway/03-remote-models.md index 1acd90d7..916dce07 100644 --- a/guide/src/gateway/03-remote-models.md +++ b/guide/src/gateway/03-remote-models.md @@ -37,7 +37,7 @@ Every remote model must list at least one endpoint, and every endpoint it names ## Kinds and thinking modes -Every model carries a `kind`: `chat`, `embedding`, or `classifier`. The kind scopes which fields are meaningful. Chat-only fields such as `thinking` and `default_max_tokens` are rejected for non-chat kinds at load time. +Every model carries a `kind`: `chat`, `embedding`, `classifier`, or `speech`. The kind scopes which fields are meaningful. Chat-only fields such as `thinking` and `default_max_tokens` are rejected for non-chat kinds at load time. Record each chat model's thinking behavior as `never`, `always`, or `switchable`. Switchable means the client may toggle thinking per request. @@ -68,7 +68,7 @@ adaptive_thinking = true The capability fields are `max_output`, `default_temperature`, `images`, `parallel_tool_calls`, `effort_levels`, `default_effort`, and `adaptive_thinking`. They obey cross-field rules at load time. A `default_effort` without `effort_levels` fails. A `default_effort` not listed in `effort_levels` fails. Effort fields fail when thinking is `never`. A `max_output` larger than `context` fails; an exact fit passes. -Enumerated fields accept a fixed spelling vocabulary. Use the spellings verbatim: protocol `openai`; thinking `never`, `always`, or `switchable`; tool_dialect `openai` or `gemma3_tool_code`; model kind `chat`, `embedding`, or `classifier`. +Enumerated fields accept a fixed spelling vocabulary. Use the spellings verbatim: protocol `openai`; thinking `never`, `always`, or `switchable`; tool_dialect `openai` or `gemma3_tool_code`; model kind `chat`, `embedding`, `classifier`, or `speech`. ## What the caller sees diff --git a/guide/src/gateway/04-local-models.md b/guide/src/gateway/04-local-models.md index afcd8419..5877506a 100644 --- a/guide/src/gateway/04-local-models.md +++ b/guide/src/gateway/04-local-models.md @@ -41,7 +41,7 @@ Local inference runs on a pinned llama-server build, b10082. The gateway prefers The gateway runs one managed llama-server child per configured `[[local_model]]`. Children get supervised respawn and deterministic teardown. Staged CUDA bundle directories are prepended to the child process's PATH only; the gateway's own environment is never mutated. Local models appear to clients as ordinary routed models under their configured names. -A local model's `kind` selects the child's serving mode: embedding models serve embeddings, and classifier models serve reranking. The `parallel` key sets both the child's concurrency and its admission limit. The thinking setting changes the child's sampling preset: thinking models sample at temperature 1.0 and top-p 0.95, while non-thinking models run with reasoning switched off and sample at 0.7 and 0.8. +A local model's `kind` selects the child's serving mode: embedding models serve embeddings, and classifier models serve reranking. A `speech` kind has no local serving mode and is refused at launch: local speech models are not yet supported. The `parallel` key sets both the child's concurrency and its admission limit. The thinking setting changes the child's sampling preset: thinking models sample at temperature 1.0 and top-p 0.95, while non-thinking models run with reasoning switched off and sample at 0.7 and 0.8. ## Chat templates diff --git a/guide/src/gateway/06-speech-synthesis.md b/guide/src/gateway/06-speech-synthesis.md new file mode 100644 index 00000000..814ad7ef --- /dev/null +++ b/guide/src/gateway/06-speech-synthesis.md @@ -0,0 +1,57 @@ +# Speech Synthesis + +This chapter teaches you the gateway's speech synthesis surface: how to declare a speech model, how to call the synthesis route, and how to enumerate voices. Synthesis builds on remote models, because the gateway routes speech to remote providers only; a `[[local_model]]` with `kind = "speech"` is refused at launch. + +## Declare a speech model + +A speech synthesis model is an ordinary `[[model]]` entry with `kind = "speech"`, backed by an ordinary `[[endpoint]]`: + +```` +[[endpoint]] +id = "together" +protocol = "openai" +base_url = "https://api.together.xyz/v1" +api_key = "${TOGETHER_API_KEY}" + +[[model]] +name = "orpheus" +kind = "speech" +description = "Orpheus 3B conversational speech synthesis" +upstream = "canopylabs/orpheus-3b-0.1-ft" +endpoints = ["together"] +context = 8192 +voices = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"] +```` + +The entry carries the usual remote-model fields, and the kind scopes which of them are meaningful. Chat-only fields such as `thinking`, the effort knobs, `default_max_tokens`, and `tool_dialect` are rejected on a speech model at load time. The speech-only `voices` list declares the voices the model offers: setting it on any other kind fails at load, entries must be non-empty and unique, and an empty or omitted list means the model exposes no fixed voice list, so the route accepts any voice name. The catalog advertises the kind and the voice list verbatim on GET /v1/models, so clients can shape requests before sending them. + +## Synthesize speech + +The gateway serves OpenAI-shaped speech synthesis at POST /v1/audio/speech: + +```` +curl -H "Authorization: Bearer $GATEWAY_KEY" http://127.0.0.1:8081/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model": "orpheus", "input": "The quick brown fox.", "voice": "tara"}' \ + -o speech.mp3 +```` + +The request carries `model` and `input` (both required; the input is non-empty and capped at 4096 characters), `voice` (required; a plain name or the OpenAI object form `{"id": "tara"}`), and four optional fields: `response_format` from the closed set `mp3`, `opus`, `aac`, `flac`, `wav`, `pcm`; `speed` between 0.25 and 4.0; `instructions`, the gpt-4o-mini-tts dialect's style-control string; and `stream_format`, `sse` or `audio`. An omitted `response_format` resolves to `mp3` before the request leaves the gateway: OpenAI defaults to mp3 while Together defaults to wav, so the pin lives in the wire type and every forwarded body carries it. Fields the gateway does not name pass through to the provider verbatim, so provider extras such as Together's `sample_rate` ride the same request, and angle-bracket emotion tags such as `` in the input reach the provider untouched. + +Authentication runs before the body is parsed, so a bad key earns 401 even for a malformed body. Shape failures earn 400: an empty or over-cap `input` or an out-of-range `speed` as `malformed_request`, a non-speech model as `kind_mismatch`, and a voice outside the model's declared list as `invalid_voice` naming the valid voices, judged before queue admission so the rejection never burns a queue slot. A full queue earns 503 with code `queue_full`. An upstream 429 comes back as 429 with code `upstream_rate_limited` and an upstream 503 as 503 with code `upstream_unavailable`, so an OpenAI client sees a retryable rate-limit or server error rather than a generic failure. + +The response is the provider's audio bytes streamed through unread: the gateway forwards the upstream `Content-Type` (when missing or invalid, the framing selector first, so an SSE stream is labeled `text/event-stream` and never an audio type, then the requested format's MIME type), never sets `Content-Length`, and emits `Transfer-Encoding: chunked`. No JSON error can follow 200 plus audio bytes, so a mid-stream upstream failure surfaces as a truncated body and the client's read fails. + +## List voices + +GET /v1/audio/voices answers the union of the active profile's speech voices, deduplicated and sorted: + +```` +{"voices": [{"id": "dan", "name": "dan"}, {"id": "jess", "name": "jess"}, {"id": "leah", "name": "leah"}]} +```` + +Each entry is an object with `id` first and `name` mirroring it, because the catalog configures voices as bare strings with no separate display name. OpenAI has no voice-list route; the OpenAI-compatible ecosystem converged on this one, and clients such as Open WebUI read the `id` key, so the entry shape is a compatibility surface. Tolerant clients also accept the plain-string form some servers answer with. A profile with no speech models returns an empty list, and non-speech models contribute nothing. + +## The stream_format caveat + +`stream_format = "sse"` is OpenAI's selector for event-stream framing, and the gateway forwards it verbatim like any other field. Provider dialects differ: Together spells its streaming mode `stream=true`, and only with `response_format = "raw"`, which the wire enum rejects, so Together SSE cannot be requested in phase 1 and Together speech is non-streaming. The forwarding is forward-looking, aimed at SSE-capable OpenAI-compatible providers, and the gateway does no reframing or decoding: whatever framing the provider answers with passes through untouched, so a client that receives an event stream owns decoding it itself. diff --git a/guide/src/gateway/06-profiles.md b/guide/src/gateway/07-profiles.md similarity index 100% rename from guide/src/gateway/06-profiles.md rename to guide/src/gateway/07-profiles.md diff --git a/guide/src/gateway/07-dominions.md b/guide/src/gateway/08-dominions.md similarity index 100% rename from guide/src/gateway/07-dominions.md rename to guide/src/gateway/08-dominions.md diff --git a/guide/src/gateway/08-editing-configuration.md b/guide/src/gateway/09-editing-configuration.md similarity index 100% rename from guide/src/gateway/08-editing-configuration.md rename to guide/src/gateway/09-editing-configuration.md diff --git a/guide/src/gateway/09-config-ui.md b/guide/src/gateway/10-config-ui.md similarity index 100% rename from guide/src/gateway/09-config-ui.md rename to guide/src/gateway/10-config-ui.md diff --git a/guide/src/gateway/10-serving-and-observing.md b/guide/src/gateway/11-serving-and-observing.md similarity index 100% rename from guide/src/gateway/10-serving-and-observing.md rename to guide/src/gateway/11-serving-and-observing.md diff --git a/guide/src/gateway/index.md b/guide/src/gateway/index.md index e60d9be0..c8a8858e 100644 --- a/guide/src/gateway/index.md +++ b/guide/src/gateway/index.md @@ -5,8 +5,9 @@ - [Remote Models and Endpoints](03-remote-models.md) - [Local Models](04-local-models.md) - [Speech-to-Text](05-speech.md) -- [Profiles and Switching](06-profiles.md) -- [Dominions and Queues](07-dominions.md) -- [Editing Configuration Safely](08-editing-configuration.md) -- [The Configuration UI](09-config-ui.md) -- [Serving and Observing](10-serving-and-observing.md) +- [Speech Synthesis](06-speech-synthesis.md) +- [Profiles and Switching](07-profiles.md) +- [Dominions and Queues](08-dominions.md) +- [Editing Configuration Safely](09-editing-configuration.md) +- [The Configuration UI](10-config-ui.md) +- [Serving and Observing](11-serving-and-observing.md) diff --git a/tools/gateway-tts-live.mjs b/tools/gateway-tts-live.mjs new file mode 100644 index 00000000..31d9500a --- /dev/null +++ b/tools/gateway-tts-live.mjs @@ -0,0 +1,448 @@ +// Live probe for the gateway speech surface. Dev-only; never in CI. +// +// Always builds the gateway first (`cargo build -p gateway`), so a stale +// binary can never be probed against a recorded current commit. Boots the +// fresh binary on an ephemeral loopback port with a throwaway +// Together-backed profile and asserts the speech and voices surfaces +// through the gateway's own responses. +// +// Invariant A19: the script never calls a vendor directly. The vendor key +// comes from the process environment only (no dotenv parsing, no .env +// reading) and is ferried only to the gateway subprocess environment; the +// throwaway config carries the `api_key = "${TOGETHER_API_KEY}"` +// interpolation reference. The key is never printed and never written to +// any file by this script. +// +// Usage: `node tools/gateway-tts-live.mjs`. Exit 0 on skip (no key in the +// environment) or when every assertion passes, 1 otherwise. + +import { spawn, spawnSync } from "node:child_process"; +import { + closeSync, + existsSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPOSITORY_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +const KEY_ENV_VAR = "TOGETHER_API_KEY"; +const TOGETHER_MODEL = "canopylabs/orpheus-3b-0.1-ft"; +const GATEWAY_MODEL = "orpheus"; +const PROFILE_NAME = "tts-live"; +const VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]; +// Local-only shared secret for the throwaway config's [server] api_key; not +// a real credential, never leaves the loopback listener. +const GATEWAY_KEY = "tts-live-throwaway-local-key"; +const PROBE_TEXT = + "PromptForge gateway live probe: the quick brown fox jumps over the lazy dog."; +const EMOTION_TEXT = + "Angle-bracket emotion tags must reach the provider untouched. "; +const READY_TIMEOUT_MS = 90_000; +const READY_POLL_MS = 250; +const CALL_TIMEOUT_MS = 180_000; +const STOP_GRACE_MS = 10_000; +const LOG_TAIL_CHARS = 2_000; + +function sleep(ms) { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + +export function renderConfig({ port }) { + const voices = VOICES.map((voice) => JSON.stringify(voice)).join(", "); + return `config-version = 2 + +[server] +bind = "127.0.0.1:${port}" +api_key = "${GATEWAY_KEY}" + +[[endpoint]] +id = "together" +protocol = "openai" +base_url = "https://api.together.xyz/v1" +api_key = "\${TOGETHER_API_KEY}" + +[[model]] +name = "${GATEWAY_MODEL}" +kind = "speech" +description = "Orpheus 3B conversational speech synthesis (live probe)" +upstream = "${TOGETHER_MODEL}" +endpoints = ["together"] +context = 8192 +voices = [${voices}] + +[[profile]] +name = "${PROFILE_NAME}" +models = ["${GATEWAY_MODEL}"] +`; +} + +function gatewayBinaryPath(repo) { + const name = + process.platform === "win32" ? "promptforge-gateway.exe" : "promptforge-gateway"; + return join(repo, "target", "debug", name); +} + +function buildGateway({ repo }) { + const result = spawnSync("cargo", ["build", "-p", "gateway"], { + cwd: repo, + stdio: "inherit", + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error(`cargo build -p gateway exited with status ${result.status}`); + } +} + +function spawnGatewayBinary({ binary, configPath, profile, env, logPath }) { + const logFd = openSync(logPath, "a"); + try { + return spawn( + binary, + ["--config", configPath, "--profile", profile, "--no-tray"], + { env, stdio: ["ignore", logFd, logFd] }, + ); + } finally { + closeSync(logFd); + } +} + +function freePort() { + return new Promise((resolvePort, reject) => { + const probe = createServer(); + probe.once("error", reject); + probe.listen(0, "127.0.0.1", () => { + const { port } = probe.address(); + probe.close(() => resolvePort(port)); + }); + }); +} + +function logTail(logPath) { + try { + return readFileSync(logPath, "utf8").slice(-LOG_TAIL_CHARS); + } catch { + return ""; + } +} + +async function waitReady({ port, proc, fetchImpl, timeoutMs, pollMs, readTail }) { + const deadline = Date.now() + timeoutMs; + const url = `http://127.0.0.1:${port}/v1/models`; + while (Date.now() < deadline) { + if (proc.exitCode !== null || proc.signalCode !== null) { + throw new Error( + `gateway exited during boot (code ${proc.exitCode ?? proc.signalCode}); log tail:\n${readTail()}`, + ); + } + try { + const response = await fetchImpl(url, { + headers: { authorization: `Bearer ${GATEWAY_KEY}` }, + signal: AbortSignal.timeout(2_000), + }); + if (response.status === 200) { + return; + } + } catch { + // Not up yet: connection refused or a timed-out poll attempt. + } + await sleep(pollMs); + } + throw new Error( + `gateway did not serve within ${Math.round(timeoutMs / 1000)}s; log tail:\n${readTail()}`, + ); +} + +async function stopGateway(proc) { + if (proc.exitCode !== null || proc.signalCode !== null) { + return; + } + const exited = new Promise((resolveExit) => + proc.once("exit", () => resolveExit(true)), + ); + proc.kill("SIGTERM"); + const graceful = await Promise.race([ + exited, + sleep(STOP_GRACE_MS).then(() => false), + ]); + if (graceful || proc.exitCode !== null || proc.signalCode !== null) { + return; + } + proc.kill("SIGKILL"); + await Promise.race([exited, sleep(STOP_GRACE_MS)]); +} + +async function speechCreate({ baseUrl, body, fetchImpl, timeoutMs }) { + try { + const response = await fetchImpl(`${baseUrl}/audio/speech`, { + method: "POST", + headers: { + authorization: `Bearer ${GATEWAY_KEY}`, + "content-type": "application/json", + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); + const audio = Buffer.from(await response.arrayBuffer()); + return { + ok: response.ok, + status: response.status, + contentType: response.headers.get("content-type") ?? "", + transferEncoding: response.headers.get("transfer-encoding") ?? "", + contentLength: response.headers.get("content-length") ?? "", + bytes: audio.length, + body: audio, + }; + } catch (error) { + return { ok: false, status: null, error: String(error).slice(0, 300) }; + } +} + +async function voicesCall({ baseUrl, fetchImpl, timeoutMs }) { + try { + const response = await fetchImpl(`${baseUrl}/audio/voices`, { + headers: { authorization: `Bearer ${GATEWAY_KEY}` }, + signal: AbortSignal.timeout(timeoutMs), + }); + let payload = null; + try { + payload = await response.json(); + } catch { + payload = null; + } + return { ok: response.ok, status: response.status, json: payload }; + } catch { + return { ok: false, status: null, json: null }; + } +} + +function isMp3(body) { + return ( + body.subarray(0, 3).toString("latin1") === "ID3" || + (body.length > 1 && body[0] === 0xff && (body[1] & 0xe0) === 0xe0) + ); +} + +function isWav(body) { + return ( + body.length > 11 && + body.subarray(0, 4).toString("latin1") === "RIFF" && + body.subarray(8, 12).toString("latin1") === "WAVE" + ); +} + +function describeSpeech(result) { + if (!result.ok) { + return `status=${result.status} error=${JSON.stringify(result.error ?? "")}`; + } + return ( + `status=${result.status} content-type=${JSON.stringify(result.contentType)} ` + + `transfer-encoding=${JSON.stringify(result.transferEncoding)} ` + + `content-length=${JSON.stringify(result.contentLength)} ` + + `bytes=${result.bytes} magic=${result.body.subarray(0, 4).toString("hex")}` + ); +} + +function createReporter(log) { + const failures = []; + return { + failures, + check(label, condition, detail) { + log(`${condition ? "PASS" : "FAIL"} ${label}: ${detail}`); + if (!condition) { + failures.push(label); + } + }, + observe(label, detail) { + log(`NOTE ${label}: ${detail}`); + }, + }; +} + +async function runSpeechSurface({ + baseUrl, + model, + fetchImpl, + callTimeoutMs, + reporter, +}) { + const call = (body) => + speechCreate({ baseUrl, body, fetchImpl, timeoutMs: callTimeoutMs }); + + const defaultCall = await call({ model, voice: "tara", input: PROBE_TEXT }); + reporter.observe("speech default format", describeSpeech(defaultCall)); + reporter.check( + "default format is mp3", + defaultCall.ok && + defaultCall.contentType.split(";")[0].trim() === "audio/mpeg" && + defaultCall.bytes > 0 && + isMp3(defaultCall.body), + describeSpeech(defaultCall), + ); + reporter.check( + "default response is streamed", + defaultCall.ok && defaultCall.contentLength === "", + `content-length=${JSON.stringify(defaultCall.contentLength ?? "")} ` + + `transfer-encoding=${JSON.stringify(defaultCall.transferEncoding ?? "")}`, + ); + + const wav = await call({ + model, + voice: "tara", + input: PROBE_TEXT, + response_format: "wav", + }); + reporter.observe("speech wav", describeSpeech(wav)); + reporter.check( + "wav format maps to audio/wav with a RIFF body", + wav.ok && + wav.contentType.split(";")[0].trim() === "audio/wav" && + wav.bytes > 0 && + isWav(wav.body), + describeSpeech(wav), + ); + + const emotion = await call({ model, voice: "tara", input: EMOTION_TEXT }); + reporter.observe("emotion-tag input", describeSpeech(emotion)); + reporter.check( + "emotion-tag input returns 200 with audio", + emotion.ok && emotion.bytes > 0, + describeSpeech(emotion), + ); + + const voices = await voicesCall({ baseUrl, fetchImpl, timeoutMs: callTimeoutMs }); + const entries = + voices.json && Array.isArray(voices.json.voices) ? voices.json.voices : null; + const ids = entries ? entries.map((entry) => entry?.id) : []; + const sorted = [...VOICES].sort(); + reporter.check( + "voices union shape", + voices.ok && + entries !== null && + JSON.stringify(ids) === JSON.stringify(sorted) && + entries.every( + (entry) => entry && typeof entry === "object" && entry.name === entry.id, + ), + `status=${voices.status} ids=${JSON.stringify(ids)}`, + ); + + // `instructions` is a named optional wire field; `sample_rate` and the + // bogus field ride the verbatim passthrough. All three must come back 2xx. + for (const [field, value] of [ + ["instructions", "Speak with a calm tone."], + ["sample_rate", 44100], + ["promptforge_probe", 1], + ]) { + const probe = await call({ model, voice: "tara", input: PROBE_TEXT, [field]: value }); + reporter.check(`field '${field}' tolerated`, probe.ok, describeSpeech(probe)); + } + + reporter.observe( + "429/503 envelopes", + "not provoked against the paid provider; covered by the Rust integration suite", + ); +} + +export async function runProbe({ + env = process.env, + repo = REPOSITORY_ROOT, + log = console.log, + build = buildGateway, + spawnGateway = spawnGatewayBinary, + fetchImpl = fetch, + readyTimeoutMs = READY_TIMEOUT_MS, + readyPollMs = READY_POLL_MS, + callTimeoutMs = CALL_TIMEOUT_MS, +} = {}) { + const key = env[KEY_ENV_VAR]; + if (!key) { + log( + `SKIP: ${KEY_ENV_VAR} is not set in the process environment; the live speech probe did not run.`, + ); + return 0; + } + + const reporter = createReporter(log); + let scratch; + try { + await build({ repo }); + const binary = gatewayBinaryPath(repo); + if (!existsSync(binary)) { + throw new Error( + `cargo build finished but the gateway binary is missing: ${binary}`, + ); + } + const port = await freePort(); + scratch = mkdtempSync(join(tmpdir(), "promptforge-tts-live-")); + const configPath = join(scratch, "gateway.toml"); + writeFileSync(configPath, renderConfig({ port }), "utf8"); + const logPath = join(scratch, "gateway.log"); + const proc = spawnGateway({ + binary, + configPath, + profile: PROFILE_NAME, + env: { ...env, [KEY_ENV_VAR]: key }, + logPath, + }); + try { + await waitReady({ + port, + proc, + fetchImpl, + timeoutMs: readyTimeoutMs, + pollMs: readyPollMs, + readTail: () => logTail(logPath), + }); + log(`gateway up on 127.0.0.1:${port} (throwaway profile '${PROFILE_NAME}')`); + await runSpeechSurface({ + baseUrl: `http://127.0.0.1:${port}/v1`, + model: GATEWAY_MODEL, + fetchImpl, + callTimeoutMs, + reporter, + }); + } finally { + await stopGateway(proc); + } + } catch (error) { + log(`FAIL: ${error instanceof Error ? error.message : String(error)}`); + return 1; + } finally { + if (scratch) { + rmSync(scratch, { force: true, recursive: true, maxRetries: 5, retryDelay: 100 }); + } + } + + if (reporter.failures.length > 0) { + log( + `LIVE FAIL: ${reporter.failures.length} assertion(s) failed: ${reporter.failures.join(", ")}`, + ); + return 1; + } + log("LIVE OK: every assertion passed"); + return 0; +} + +if ( + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + runProbe().then( + (code) => { + process.exitCode = code; + }, + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }, + ); +} diff --git a/tools/gateway-tts-live.test.mjs b/tools/gateway-tts-live.test.mjs new file mode 100644 index 00000000..5cad628b --- /dev/null +++ b/tools/gateway-tts-live.test.mjs @@ -0,0 +1,262 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { + closeSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { renderConfig, runProbe } from "./gateway-tts-live.mjs"; + +// A fake credential: the secret-free test needs a canary value to hunt for, +// and the ferrying test needs a value the fake gateway can compare against. +const CANARY = "ttest-canary-not-a-real-key-9f8e7d6c"; + +// The gateway double: a Node script that parses --config for its bind port +// and serves canned speech, voices, and readiness responses on loopback. +// Behavior modes arrive through its environment, exactly where the real +// gateway would find TOGETHER_API_KEY. +const FAKE_GATEWAY_SOURCE = ` +import { readFileSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; + +const configPath = process.argv[process.argv.indexOf("--config") + 1]; +const config = readFileSync(configPath, "utf8"); +const port = Number(config.match(/bind = "127\\.0\\.0\\.1:(\\d+)"/)[1]); +const mode = process.env.FAKE_GATEWAY_MODE ?? "ok"; +if (process.env.FAKE_KEY_SINK) { + const seen = process.env.TOGETHER_API_KEY; + const expected = process.env.FAKE_EXPECT_KEY; + const verdict = seen === undefined ? "absent" : seen === expected ? "match" : "mismatch"; + writeFileSync(process.env.FAKE_KEY_SINK, verdict); +} +if (mode === "exit-now") { + console.error("fake gateway refusing to boot"); + process.exit(1); +} +const MP3 = Buffer.from([0x49, 0x44, 0x33, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10]); +const WAV = Buffer.concat([ + Buffer.from("RIFF", "latin1"), + Buffer.alloc(4), + Buffer.from("WAVE", "latin1"), + Buffer.alloc(8), +]); +const VOICE_IDS = ["dan", "jess", "leah", "leo", "mia", "tara", "zac", "zoe"]; + +if (mode === "never-listen") { + setInterval(() => {}, 1000); +} else { + const server = createServer((request, response) => { + const url = new URL(request.url, "http://127.0.0.1"); + if (url.pathname === "/v1/models") { + response.writeHead(200, { "content-type": "application/json" }); + response.end("{}"); + return; + } + if (url.pathname === "/v1/audio/voices") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ voices: VOICE_IDS.map((id) => ({ id, name: id })) })); + return; + } + if (url.pathname === "/v1/audio/speech" && request.method === "POST") { + let body = ""; + request.on("data", (chunk) => { + body += chunk; + }); + request.on("end", () => { + if (mode === "error500") { + response.writeHead(500, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: { message: "upstream exploded" } })); + return; + } + const parsed = JSON.parse(body); + const wav = parsed.response_format === "wav" || mode === "wrong-format"; + response.writeHead(200, { + "content-type": wav ? "audio/wav" : "audio/mpeg", + "transfer-encoding": "chunked", + }); + response.end(wav ? WAV : MP3); + }); + return; + } + response.writeHead(404, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: { message: "not found" } })); + }); + server.listen(port, "127.0.0.1"); +} +`; + +function makeHarness(t, mode, { withKey = true } = {}) { + const dir = mkdtempSync(join(tmpdir(), "tts-live-test-")); + t.after(() => rmSync(dir, { force: true, recursive: true, maxRetries: 5, retryDelay: 100 })); + const fakePath = join(dir, "fake-gateway.mjs"); + writeFileSync(fakePath, FAKE_GATEWAY_SOURCE, "utf8"); + const sinkPath = join(dir, "key-sink.txt"); + const repo = join(dir, "repo"); + const binaryDir = join(repo, "target", "debug"); + mkdirSync(binaryDir, { recursive: true }); + const binaryName = + process.platform === "win32" ? "promptforge-gateway.exe" : "promptforge-gateway"; + writeFileSync(join(binaryDir, binaryName), "fake binary", "utf8"); + + const lines = []; + const events = []; + const spawned = []; + const env = { + ...process.env, + FAKE_GATEWAY_MODE: mode, + FAKE_KEY_SINK: sinkPath, + FAKE_EXPECT_KEY: CANARY, + }; + if (withKey) { + env.TOGETHER_API_KEY = CANARY; + } else { + delete env.TOGETHER_API_KEY; + } + const deps = { + env, + repo, + log: (line) => lines.push(line), + build: async () => { + events.push("build"); + }, + spawnGateway: ({ configPath, profile, env: childEnv, logPath }) => { + events.push("spawn"); + const logFd = openSync(logPath, "a"); + const child = spawn( + process.execPath, + [fakePath, "--config", configPath, "--profile", profile, "--no-tray"], + { env: childEnv, stdio: ["ignore", logFd, logFd] }, + ); + closeSync(logFd); + spawned.push(child); + return child; + }, + readyTimeoutMs: 5_000, + readyPollMs: 50, + callTimeoutMs: 5_000, + }; + return { lines, events, spawned, deps, sinkPath }; +} + +function assertAllDead(spawned) { + assert.ok(spawned.length > 0, "at least one gateway child was spawned"); + for (const child of spawned) { + assert.notEqual( + child.exitCode ?? child.signalCode, + null, + "the gateway subprocess was terminated", + ); + } +} + +test("passes against a conforming gateway and ferries the key only to the subprocess", async (t) => { + const { lines, spawned, deps, sinkPath } = makeHarness(t, "ok"); + const code = await runProbe(deps); + assert.equal(code, 0); + const output = lines.join("\n"); + assert.match(output, /gateway up on 127\.0\.0\.1:\d+/); + assert.match(output, /PASS default format is mp3/); + assert.match(output, /PASS default response is streamed/); + assert.match(output, /PASS wav format maps to audio\/wav with a RIFF body/); + assert.match(output, /PASS emotion-tag input returns 200 with audio/); + assert.match(output, /PASS voices union shape/); + assert.match(output, /PASS field 'instructions' tolerated/); + assert.match(output, /PASS field 'sample_rate' tolerated/); + assert.match(output, /PASS field 'promptforge_probe' tolerated/); + assert.match(output, /LIVE OK: every assertion passed/); + assert.equal(readFileSync(sinkPath, "utf8"), "match"); + assertAllDead(spawned); +}); + +test("always builds the gateway before booting it", async (t) => { + const { events, deps } = makeHarness(t, "ok"); + const code = await runProbe(deps); + assert.equal(code, 0); + assert.deepEqual(events.slice(0, 2), ["build", "spawn"]); + assert.equal(events.filter((event) => event === "build").length, 1); +}); + +test("fails loudly when the gateway exits during boot", async (t) => { + const { lines, spawned, deps } = makeHarness(t, "exit-now"); + const code = await runProbe(deps); + assert.equal(code, 1); + const output = lines.join("\n"); + assert.match(output, /gateway exited during boot/); + assert.match(output, /refusing to boot/); + assertAllDead(spawned); +}); + +test("fails when the gateway never serves within the readiness deadline", async (t) => { + const { lines, spawned, deps } = makeHarness(t, "never-listen"); + deps.readyTimeoutMs = 600; + const code = await runProbe(deps); + assert.equal(code, 1); + assert.match(lines.join("\n"), /did not serve within/); + assertAllDead(spawned); +}); + +test("fails when the speech surface answers an error envelope", async (t) => { + const { lines, spawned, deps } = makeHarness(t, "error500"); + const code = await runProbe(deps); + assert.equal(code, 1); + const output = lines.join("\n"); + assert.match(output, /FAIL default format is mp3: status=500/); + assert.match(output, /LIVE FAIL: \d+ assertion\(s\) failed/); + assertAllDead(spawned); +}); + +test("fails when a response violates the speech contract", async (t) => { + const { lines, spawned, deps } = makeHarness(t, "wrong-format"); + const code = await runProbe(deps); + assert.equal(code, 1); + const output = lines.join("\n"); + assert.match(output, /FAIL default format is mp3/); + assert.match(output, /PASS wav format maps to audio\/wav with a RIFF body/); + assertAllDead(spawned); +}); + +test("terminates the gateway subprocess on success and on failure", async (t) => { + for (const mode of ["ok", "error500", "exit-now"]) { + const { spawned, deps } = makeHarness(t, mode); + await runProbe(deps); + assertAllDead(spawned); + } +}); + +test("skips with exit 0 when TOGETHER_API_KEY is absent from the environment", async (t) => { + const { lines, deps } = makeHarness(t, "ok", { withKey: false }); + deps.build = async () => { + throw new Error("build must not run without a key"); + }; + deps.spawnGateway = () => { + throw new Error("spawn must not run without a key"); + }; + const code = await runProbe(deps); + assert.equal(code, 0); + assert.match(lines.join("\n"), /SKIP: TOGETHER_API_KEY is not set/); +}); + +test("never prints the key and the config carries only the interpolation reference", async (t) => { + const { lines, deps, sinkPath } = makeHarness(t, "ok"); + const code = await runProbe(deps); + assert.equal(code, 0); + assert.ok( + !lines.join("\n").includes(CANARY), + "the key never appears in the probe output", + ); + const config = renderConfig({ port: 12345 }); + assert.ok(!config.includes(CANARY), "the config never contains the key value"); + assert.ok( + config.includes('api_key = "${TOGETHER_API_KEY}"'), + "the config carries only the interpolation reference", + ); + assert.equal(readFileSync(sinkPath, "utf8"), "match"); +}); diff --git a/vibe-ledger.md b/vibe-ledger.md index 259272eb..2a8a5a3c 100644 --- a/vibe-ledger.md +++ b/vibe-ledger.md @@ -56,3 +56,18 @@ - Async STT boot reset Step 5: Notify the browser when STT needs restart - Config UI typecheck, build, and complete suite (142 passed). Decision: toast kind `info`; catalog order ignored; membership filtered through each document's own catalog; combined process-owned plus STT Apply shows both the existing banner and one STT toast. Falsifier: reordering `[[stt_model]]` changes what boot loads, or a profile naming a non-STT model is a speech change. - Async STT boot reset Step 6: Reconcile documentation and qualify - Full verification passed: build, fmt, warnings-denied clippy (portable and staged Workshop), portable workspace and doc tests, feature-disabled Gateway, both UI suites, staged Workshop tests with guaranteed cleanup, both Miri lanes, all five native Whisper lanes with hash-pinned fixtures, guide assembler with zero drift, mdBook build, and `cargo workshop` package construction. Operator confirmed physical microphone hypothesis revision, authoritative completion, and second take on the release binaries. + +## 2026-09-07-2-gateway-tts-phase-1 + +- Run re-seeded 2026-09-09 on branch `add-tts-phase-1` (rewrite base `5edcb3c9`, current cppalliance master). Plan: `vibe/2026-09-07-2-gateway-tts-phase-1.md` (corrected per the maintainer review of PR #21 before the rewrite; the correction commit lives on `backup/tts-phase-1-pre-rewrite`). This run rewrites the original 2026-09-07 run's history; the original series is preserved on the backup branch and under `d:\_cppalliance\promptforge-backups\2026-09-09-tts-rewrite\`. +- Decision (run-level): verify cadence. Every step gets a Verify dispatch running the build plus that step's focused tests; the full CI-exact gate suite runs at component ends (steps 2, 4, 6, 7, and 9) and after any fix round that changes the commit. Falsifier: a step lands red that its focused run could not catch, which indicts the cadence rather than the step. +- Decision (run-level): decomposition not re-run. The plan's steps are carried from the original run's executed plan with the review amendments folded in; the Run Mode defect pass re-validated ordering and ambiguity instead of a decomposition rewrite, which would have risked dropping the amendments. Falsifier: a step that cannot name what it receives from earlier steps; none found. +- Step 1 (Speech model kind and voices capability): `cargo build` + `cargo test -p gateway-config` green at `acfaa939` (log `vibe/verify-step-1.log`); config-UI npm gates (typecheck, build, 126 tests) green. Review: clean, 0 findings. Amendment beyond the original diff: `speech` added to both `kind()` doc comments in `accessors.rs` (review F12). Decisions made alone: none. +- Step 2 (launch_options goes fallible and refuses unknown kinds in gateway-local): component-ending verify at `eb7d3f25` green — fmt, clippy `-D warnings`, workspace tests + doctests, headless check, `cargo doc -D warnings`, `cargo deny` (0.20.2, no waiver), config-UI npm chain (log `vibe/verify-step-2.log`). Review: 1 Minor (`start_impl` provisioned the shared server before the per-model kind preflight), fixed by hoisting the preflight so the server is provisioned only when a launchable model exists, closed. Amendment beyond the original diff: the A6 preflight (`serve_mode_for` before any side effect in `start_impl` and `provision_artifacts_impl`) plus its no-side-effect tests (review F1). Decisions made alone: none. +- Step 3 (SpeechRequest wire type): `cargo build` + `cargo test -p shared-protocol` green at `b6cbf9d9` (log `vibe/verify-step-3.log`). Review: 1 Minor (the 4096-char cap table exercised only ASCII, so a byte-counting regression would pass untested), fixed with multi-byte boundary rows (4,096 x U+00E9 accepted at 8,192 bytes, 4,097 rejected), closed. Amendment beyond the original diff: the closed-format rejection test now pins Together's `raw` (review F8). Decisions made alone: none. +- Step 4 (Speech upstream and audio streaming client): component-ending verify at `e59f051a` green — fmt, clippy `-D warnings`, workspace tests + doctests, headless check, `cargo doc -D warnings`, `cargo deny` (manifests touched, so it really ran) (log `vibe/verify-step-4.log`). Review: 1 Important (the first-response deadline was untested — removing it kept all tests green), fixed with a cfg(test)-scaled budget and a stalled-headers rejection test, closed. Amendment beyond the original diff: the F4 deadline split (first-response ~120 s vs per-read body idle 30 s). Decision (made alone): both deadlines live in `send_speech`, not on the client, because reqwest 0.12 arms `read_timeout` during the header wait (`PendingRequest::poll`); the plan's Technical Design, Decision Record, and step text were revised in the same change naming the forcing behavior, per the plan's execution rule. Falsifier: a reqwest release that stops arming `read_timeout` during the header wait, which would let the idle budget move back onto the client. +- Step 5 (POST /v1/audio/speech route): focused verify green against the tree committed at `3cefa207` - fmt, clippy `-D warnings` (gateway, all-targets/all-features), full `cargo test -p gateway` (449 passed, 0 failed: 134 it + lib + doctests), warnings-denied docs over gateway/gateway-routing/shared-protocol, featureless `cargo check -p gateway --no-default-features`. Review: 3 findings (1 Important, 2 Minor), all closed: the vacuous permit-held guard became a bounded negative wait (`ADMISSION_GRACE` 100 ms, sized under the relay's scaled 200 ms upstream-idle budget, red-verified by dropping the permit at relay start); the queued-race `tokio::select!` is wrapped in `PHASE_TIMEOUT`; boot.rs's `not(test-fixtures)` rendezvous test records its retirement in its comment. Amendments beyond the original diff: the F2 bounded background relay (spawned task owning the upstream body, dominion permit, and cancellation guard; bounded channel; four named cfg(test)-scaled bounds; every terminal path emits one Err item and releases the permit), F3 cancellation emits a synthesized body error instead of clean EOF, F5 residual SSE-aware MIME fallback, F8 route pins (unknown-model 404 envelope, profile-switch mid-stream truncation, `` passthrough, `stream_format:"sse"` forwarding), F12 kind-list doc comments. Decision (made alone): a self dev-dependency (`gateway = { path = ".", features = ["test-fixtures"], default-features = false }`) lets the it harness drive the scaled relay bounds, retiring boot.rs's `not(test-fixtures)` test from its last runner (recorded in the test's comment). Falsifier: a gateway test invocation without `test-fixtures` reaches CI or local documentation, or the self dev-dep is removed, either of which re-enables the boot.rs test. +- Step 6 (GET /v1/audio/voices route): component-ending verify green against the tree committed at `6bded8ca` - fmt, clippy `-D warnings` (same workshop/workshop-server exclude substitution as steps 2 and 4, for the pre-existing missing Tauri sidecar binary), workspace tests + doctests (2843 + 275, 0 failures, 36 pre-existing env-gated ignores), featureless gateway check, warnings-denied docs, cargo deny, config-UI npm chain 126/126 (log `vibe/verify-step-6.log`). Review: clean, 0 findings. Amendment beyond the original diff: the unauthenticated-401 test on the voices route (review F8), red-verified against a commented-out `check_auth`. Decisions made alone: none. +- Step 7 (Live gateway speech probe and verification note): component-ending verify green against the tree committed at `aa138792` - fmt, clippy `-D warnings` (same workshop/workshop-server exclude substitution as steps 2/4/6), workspace tests + doctests (983 + 275; one gateway-logging timing assertion failed the first parallel run and passed isolated and full-package reruns - a flake, this commit carries zero Rust changes), featureless gateway check, warnings-denied docs, cargo deny, config-UI npm chain 126/126, `node --check` and `node --test tools/gateway-tts-live.test.mjs` 9/9 (log `vibe/verify-step-7.log`). Review: clean, 0 findings. Fresh work replacing the original run's parity step: the Python script never enters this history; the probe is the zero-dependency Node script `tools/gateway-tts-live.mjs` with paired offline tests, process-environment key ferrying only (F6/A19), always-builds-before-boot (F7), and the rewritten gateway-only verification note with Phase 3 provenance placeholders (F9/F10). Decisions made alone: none. +- Step 8 (Documentation): focused verify green against the tree committed at `447ba9e5` - fmt, `cargo run -p build-user-guide` byte-stable regen, F5/MIME-fallback inspection of README + speech chapter + export (log `vibe/verify-step-8.log`). Review: 1 Minor (the Content-Type fallback clause omitted the code's SSE-first `text/event-stream` path), fixed in the chapter, README, and regenerated export, closed. Amendments beyond the original diff: the F5 SSE wording (Together SSE is unrequestable because `raw` is rejected at the wire, phase-1 Together speech is non-streaming, `stream_format` forwarding is forward-looking) plus the SSE-first MIME fallback. Decisions made alone: none. +- Step 9 (As-built design document): component-ending verify green against the tree committed at `f44faebb` - fmt, clippy `-D warnings` (same workshop/workshop-server exclude), workspace tests + doctests (2568 + 275; one gateway-local readiness timing assertion fail-fasted the first parallel run and passed isolated and `--no-fail-fast` completion - a flake, this commit carries only the as-built markdown and the plan's F13 frontmatter), featureless gateway check, warnings-denied docs, cargo deny, config-UI npm chain 126/126, guide regen no-diff, `node --check` and `node --test tools/gateway-tts-live.test.mjs` 9/9 (log `vibe/verify-step-9.log`). Review: clean, 0 findings. Amendments beyond the original diff: the as-built reconciled with the bounded relay, the split send_speech deadlines, the serve_mode_for preflight, the Node live probe, A19 numbering, and the rewritten series SHAs (F9); the six frontmatter todos flipped to completed (F13). Decisions made alone: none. diff --git a/vibe/2026-09-07-2-gateway-tts-phase-1.md b/vibe/2026-09-07-2-gateway-tts-phase-1.md new file mode 100644 index 00000000..f2bd33fd --- /dev/null +++ b/vibe/2026-09-07-2-gateway-tts-phase-1.md @@ -0,0 +1,219 @@ +--- +name: Gateway TTS phase 1 +overview: "Implement phase 1 of the former design/report-gateway-tts-endpoint.md (removed from this repository when design records moved out; the as-built supersedes it) on branch add-tts-phase-1: a ModelKind::Speech config variant, the OpenAI-shaped POST /v1/audio/speech route, Upstream::send_speech streaming binary audio, a voices capability, GET /v1/audio/voices, remote passthrough, config-UI speech support, and docs. Phase 2 (local Orpheus engine) is out of scope." +todos: + - id: config-kind + content: Add ModelKind::Speech, voices capability, kind-scope validation plus empty/duplicate voice rejection at load, fallible launch_options with refuse-unknown wildcard, config-UI speech kind + voices chips, config and UI tests + status: completed + - id: wire-types + content: SpeechRequest with validate() and serde-defaulted response_format enum, StreamedAudio, Upstream::send_speech + OpenAiUpstream impl, audio streaming client, upstream status-shape tests + status: completed + - id: routes + content: POST /v1/audio/speech and GET /v1/audio/voices handlers and registration, speech-only 429/503 GatewayError variants, admin status row, speech.rs integration tests incl. no-leak, permit-held, reverse kind_mismatch, mid-stream Err + status: completed + - id: remote-verify + content: Dev-only zero-dependency Node live probe driving the gateway speech surface on a Together-backed throwaway profile (gateway-only per A19, vendor key from the process environment, ferried only to the gateway subprocess), with observed-dialect record, or documented deferral + status: completed + - id: docs + content: Gateway README speech section, guide chapter 06-speech-synthesis (git mv renumber + build-user-guide assembler), kind lists in guide 03/04, gateway.local.example.toml, crate READMEs + status: completed + - id: design-doc + content: "Final step: spawn generator subagent to write design/design-gateway-tts-phase-1.md from the finished work" + status: completed +isProject: false +--- + +# Gateway TTS Endpoint, Phase 1: Routed Speech Kind with Remote Passthrough + +## Product Requirements + +- Problem and users: the gateway routes chat, embedding, and classifier models but has no speech synthesis; users are prompt pipelines that bind a speech model by name and voice, and later the Workshop playing assistant replies. +- Goals: `kind = "speech"` catalog models; an OpenAI-shaped `POST /v1/audio/speech` that streams binary audio; a per-model `voices` capability surfaced on `GET /v1/models`; a `GET /v1/audio/voices` union route; remote passthrough (Together AI serving Orpheus 3B, OpenAI `tts-1`/`gpt-4o-mini-tts`); config-UI support (`speech` in the Kind options, a `voices` chips field shown only for speech models); documentation. +- Non-goals: the local Orpheus engine and `ServeMode::Speech` (phase 2, gated on an engine spike); voice cloning; audio transcoding, encoders, and WAV-header policy; SSE-decoding Together's `stream=true` dialect; ElevenLabs or Baseten adapters; Workshop playback; a config-UI voice picker; technical-text conditioning. +- Success criteria: an OpenAI-shaped speech request returns streamed binary audio from the configured provider; speech models and their voices are discoverable; invalid auth, model, kind, and voice each produce the specified error envelope; the full gate suite is green. +- Constraints: + - OpenAI wire compatibility is a hard requirement, and credentials never leave the gateway (nothing above it holds a vendor key). + - No `unsafe` outside FFI crates; `cargo clippy -D warnings`; `cargo fmt`; `cargo check -p gateway --no-default-features` stays green (this path adds no native or feature-gated dependency). + - Every new public type, function, and module carries a `///` doc comment; non-doc comments state a non-obvious why, with an upstream issue URL wherever code works around external behavior. + - `input` is capped at 4096 characters, and angle-bracket content in `input` is never sanitized or escaped anywhere on the path: Orpheus emotion tags (``, ``, ...) arrive inline and tag-stripping silently destroys the feature. +- Open questions: none blocking; field-tunable values are listed under Decision Record notes. + +## Functional Specification + +- Actors and workflows: an external client with a bearer token POSTs a speech request; the gateway authenticates before parsing the body, validates the request, resolves the routed model, guards the kind, validates the voice before queue admission (a 400 never burns a queue slot), admits through the dominion queue (cancellable), calls the upstream provider, and streams audio bytes back while holding the queue permit for the stream's life; the same client may enumerate voices via `GET /v1/audio/voices` and discover speech models via `GET /v1/models`. +- Inputs and outputs: + - Request JSON: `model` (required), `input` (required, non-empty, at most 4096 characters), `voice` (required; string or `{"id": "..."}` object), `response_format` (optional closed enum `mp3`|`opus`|`aac`|`flac`|`wav`|`pcm`; a serde default resolves an omitted field to `mp3` at deserialization, so the pin cannot be forgotten by any route), `speed` (optional, 0.25-4.0), `instructions` (optional string), `stream_format` (optional `sse`|`audio`); fields the gateway does not name pass through to the provider verbatim. + - Speech response: 200 with the upstream `Content-Type` (fallback: the format-to-MIME mapping `mp3`->`audio/mpeg`, `wav`->`audio/wav`, `pcm`->`audio/pcm`, `opus`->`audio/ogg`, `flac`->`audio/flac`, `aac`->`audio/aac`) and chunked binary audio; `Content-Length` is never set, so hyper emits `Transfer-Encoding: chunked`. + - Voices response: `{"voices": [{"id", "name"}]}`, the union of the active profile's speech models' `voices`, deduplicated and sorted; entries are id-first objects because ecosystem clients (Open WebUI, Talemate) read the `id` key, so the entry shape is itself a compatibility surface and is test-pinned. + - Config: `[[model]]` with `kind = "speech"`, an optional `voices` list (empty entries and duplicates rejected at load), and `context` required and nonzero (documented as maximum input text tokens), exactly as every other kind. +- States and validation: wire `validate()` rejects empty `model`, empty or over-cap `input`, and out-of-range `speed`; the kind guard returns 400 `kind_mismatch`; a `voice` absent from the model's non-empty `voices` list returns 400 `invalid_request_error` with code `invalid_voice` naming the valid voices; an auth failure returns 401 before any 400 can fire. +- Errors and recovery: upstream 429 maps to 429 `upstream_rate_limited` and upstream 503 to 503 `upstream_unavailable` on the speech path; other upstream 4xx pass through as `upstream_client_error` and 5xx as 502 `upstream_error`; a full dominion queue returns 503; a mid-stream transport failure or a profile-switch cancellation fails the client's body read with a stream error item (no JSON envelope can follow 200 plus audio bytes, and a clean EOF would read as a complete response); a `kind = "speech"` local model fails before any provisioning side effect with a "local speech models are not yet supported" error. +- Security and privacy behavior: bearer auth runs before body extraction so an unauthorized caller never makes the gateway parse a body; provider keys live only in endpoint configuration; audio bytes and synthesis text are never inspected, rewritten, or buffered whole; the default request body limit suffices because `input` is capped at 4096 characters. +- Acceptance criteria: byte-identical passthrough of canned audio; content-type forwarding and per-format fallback mapping; client disconnect drops the upstream stream and releases the permit; queue-full 503; the voices-union shape; 401-before-400 ordering. + +## Technical Design + +- Architecture: speech is a fourth routed model kind beside chat, embedding, and classifier, with catalog membership, routing-table visibility, dominion queue admission, and a per-kind route guard; the handler lives inline in `crates/gateway/src/lib.rs` beside `embeddings` and `rerank`, with no new crate and no feature gate. +- Modules and interfaces: + - `ModelKind::Speech` in `crates/gateway-config/src/config.rs` (enum ~529-540, serde and `Display` spelling `"speech"`). + - `Capabilities.voices: Vec` with `#[serde(default)]` in the same file (~562-588); `list_models` (lib.rs ~923-947) clones capabilities verbatim into `ModelInfo`, so `GET /v1/models` needs no change. + - `validate_kind_scope` (`crates/gateway-config/src/config/validate.rs` 714-745; call sites 313-323 remote and 397-414 local) already rejects chat-only fields for non-chat kinds; add the symmetric rejection of a non-empty `voices` on non-speech models. Empty voice entries and duplicates are rejected in `validate_capabilities` (validate.rs 671-705, beside the `effort_levels` content checks); an empty `voices` list stays valid and skips the route voice check. + - `SpeechRequest` with `validate() -> Result<(), &'static str>` in `crates/shared-protocol/src/wire.rs` beside `EmbeddingRequest` (~246), mirroring `ChatRequest::validate` (wire.rs:73); `voice` is an untagged string-or-object enum; `response_format` is a closed enum with `#[serde(default)]` resolving an omitted field to `mp3`; a flattened `rest` with `RESERVED` naming the seven known fields preserves verbatim passthrough. + - `StreamedAudio { content_type: String, body: BoxStream<'static, Result> }` beside `StreamedChunks` (`crates/shared-protocol/src/upstream.rs` 21-40); `Upstream::send_speech` beside `send_embeddings`/`shutdown` (44-148), defaulting to `ProtocolError::ModelUnavailable`; `OpenAiUpstream::send_speech` runs over the raw `post` helper (202-230), substituting `upstream_model` and forwarding the body otherwise verbatim. + - `audio_streaming_client()` in `crates/shared-protocol/src/http_util.rs` beside `streaming_client()` (38-44): connect timeout 10 s and `tcp_keepalive` 60 s, and deliberately no `read_timeout` — reqwest arms `read_timeout` during the wait for response headers, which would cap time-to-headers at the body-idle budget; both deadlines live in `send_speech` instead: a named first-response deadline (~120 s, a module-level constant) wraps the header wait, and a named per-read idle deadline (30 s) guards the opened body. `OpenAiUpstream` gains a third client field and the chat SSE client is untouched. + - Speech-only `GatewayError::UpstreamRateLimited` (429 `rate_limit_error`/`upstream_rate_limited`) and `GatewayError::UpstreamUnavailable` (503 `server_error`/`upstream_unavailable`) in `crates/gateway/src/error.rs` beside `QueueRejected` (~56-60), with two arms in the gateway's own exhaustive `classify()` (294-442) and two rows in `gateway_error_classify_is_table_driven` (487-570); the `audio_speech` handler matches `ProtocolError::UpstreamStatus { status: 429 | 503, .. }` into them and forwards every other error through the unchanged `Protocol` arm (321). `crates/shared-protocol/src/error.rs` is not touched: `ProtocolError::classify` (109-145) and its table test (184-239) keep chat, embedding, and rerank envelopes bit-identical. + - `audio_speech` and `audio_voices` handlers in `crates/gateway/src/lib.rs`, registered in `build_router` (464-479); `audio_speech` is cloned from `embeddings` (853-882) with the speech deltas; `audio_voices` follows the `list_models` pattern over `routing.models()` (routing.rs:84). An unconditional `endpoint_status` row for `/v1/audio/speech` joins the `admin_status` rows (lib.rs 1080-1102); the stt-gated `with_speech_endpoint` helper (lib.rs 1012-1026) is not reused. + - `launch_options` (`crates/gateway-local/src/runtime.rs` 699-720) becomes fallible, returning `Result` with the already-fallible wrapper `launch_options_for` (722-731) propagating: the Chat/Embedding/Classifier arms stay, `ModelKind::Speech` errors "local speech models are not yet supported" (a new `LocalError` variant beside `UnsupportedPlatform`, error.rs ~12-17), and the wildcard becomes refuse-unknown (`_ => Err(...)`) instead of `_ => ServeMode::Chat` (714) — `ModelKind` is `#[non_exhaustive]`, so a wildcard must remain but it never maps to chat. The `tool_dialect` wildcard (493-496) stays; its default is deliberate. The kind mapping is extracted into a side-effect-free `serve_mode_for(kind)` preflight that `start_impl` and `provision_artifacts_impl` run before any server, model, companion, or cache side effect, so an unsupported kind never downloads artifacts or writes metadata before refusal (A6). + - Config UI: `speech` joins the shared Kind options list (`crates/gateway-config-ui/ui/src/views/models-view.ts` 647-648, one list rendered for every non-STT entry remote or local), and a `voices` chips field reuses the existing `type: "chips"` editor (`components/chip-input.ts`) with `visibleWhen: kind === "speech"` (the registry's mechanism, settings-registry.ts:56). Discover's hardcoded `kind: "chat"` (discover-view.ts:742) stays a named gap. +- File and public API changes: `gateway-config` (config.rs, validate.rs), `gateway-local` (runtime.rs, error.rs), `shared-protocol` (wire.rs, upstream.rs, http_util.rs, Cargo.toml with `bytes.workspace = true` beside the root `bytes = "1"` pin, README where it enumerates `Upstream`; error.rs explicitly untouched), `gateway` (lib.rs, error.rs, tests/it/speech.rs, tests/it/main.rs, README), `gateway-config-ui` (models-view.ts, settings registry, model-detail tests), the guide (new `guide/src/gateway/06-speech-synthesis.md`, following chapters moved with `git mv` highest-first, `guide/src/SUMMARY.md` regenerated by `cargo run -p build-user-guide`, kind lists in 03 and 04), and `gateway.local.example.toml` (a commented speech `[[model]]`). +- Data, persistence, failure, security, and privacy constraints: + - Byte passthrough is the one departure from the gateway's typed-relay norm: audio frames are opaque bytes, so the handler forwards them unread and justifies the departure in its doc comment the way `relay_sse` justifies its design. + - A bounded background relay task owns the upstream body, the dominion permit, and the cancellation guard, feeding a small bounded channel to the HTTP body; named static constants bound total lifetime (60 min), response bytes (1 GiB), upstream read idle after headers (30 s), and blocked downstream delivery (60 s); every terminal path emits an `Err` item and releases the permit, so cancellation and failure never surface as a clean EOF and no path holds a permit unbounded (A5, A27). + - The route must stay out of any future `CompressionLayer` or whole-request `TimeoutLayer`, both of which buffer or kill long-lived streams; the handler's doc comment carries the warning. + - Nothing persists: the feature writes no artifacts, caches, or database rows. + +## Testing Plan + +- Unit: wire validation table (empty `input`, over-cap `input`, out-of-range `speed`, unknown `response_format`, and the intentional rejection of Together's `raw`) plus the serde-default `mp3` resolution; config parse/serialize round-trip of `kind = "speech"` with `voices`; each chat-only field rejected on a speech model; `voices` rejected on chat, embedding, and classifier models; empty voice entries and duplicate voices rejected at load; the `launch_options` speech arm and the refuse-unknown wildcard error rather than launching as chat; the provisioning preflight (an all-speech profile touches neither the server provisioner nor the model store, and a mixed profile provisions only supported models); slow-headers versus idle-body timeout separation (headers arriving after the body-idle budget but within the first-response budget are accepted; an opened body that stalls fails); two rows for the new variants in the gateway's own `gateway_error_classify_is_table_driven` (`crates/gateway/src/error.rs` 487-570) — the shared-protocol classify table (shared-protocol error.rs 184-239) is not touched; upstream tests mirroring the `serve_once`/`serve_stalled` module (upstream.rs ~412-439, ~970) for model-name substitution, bearer forwarding, status and transport mapping (asserting the `UpstreamStatus` shape for 429/503, never a client envelope), and untransformed byte streaming; config-UI tests: the dropdown-values pin updated to the four-kind list (`model-detail.test.mjs` ~78-82) and a pick-speech/add-chip/save test asserting the PUT body. +- Integration and end-to-end: new `crates/gateway/tests/it/speech.rs` mirroring `embeddings.rs` and `rerank.rs` on the `support.rs` harness (`spawn_backend` takes any axum `Router`, support.rs:163-169, so canned binary bytes work today), covering 401-before-malformed-JSON, `kind_mismatch`, `invalid_voice` naming valid voices, an empty `voices` list skipping the voice check, byte-identical passthrough, content-type forwarding and fallback, queue-full 503, disconnect dropping the upstream stream, and the voices union; the permit held for the stream's lifetime under `max_concurrency = 1` (copied from `stream_permit_is_held_until_the_stream_ends`, chat.rs:943 — the disconnect test at chat.rs:994 is a different property); reverse `kind_mismatch` rows (a speech model on the chat, embeddings, and rerank routes) beside the existing per-route tests (chat.rs:136, embeddings.rs:259, rerank.rs:168); mid-stream upstream `Err` after HTTP 200 (the client's body read fails and no JSON envelope appears in the audio; the harness yields an Err-chunk `Body::from_stream` per the chat.rs:1017 pattern); `GET /v1/models` showing `kind: "speech"` and `voices` (shape assertions per chat.rs:261-292); the speech-route 429/503 envelopes; and the no-leak assertion that an upstream error body carrying provider internals (stack text, internal hosts, request ids) never reaches the client; an unknown model on the speech route returning 404 `model_not_found`; an unauthenticated `GET /v1/audio/voices` returning 401; a profile switch during an open speech body failing the client's next read; `` emotion-tag passthrough in the offline suite; `stream_format: "sse"` forwarded into the outbound body at the route; the bounded relay's boundary tests (exact byte limit, one byte over, total deadline, sub-idle upstream drip, saturated downstream channel, upstream idle after headers, delayed-but-accepted headers, client disconnect, profile cancellation, upstream body error), each proving permit release and admission of a subsequent request; route-level auth tests beside (not inside) the stt-gated `transcription_auth_tests` module (lib.rs ~2772) — the speech tests stay ungated because the route is unconditional. +- Live probe (gated, dev-only): a zero-dependency Node script (`tools/gateway-tts-live.mjs`, built-in `fetch`, paired `.test.mjs` offline tests) boots a freshly built gateway on a throwaway Together-backed profile and asserts the speech and voices surfaces through the gateway's responses; the vendor key comes from the process environment and reaches only the gateway subprocess (A19: the script never calls a vendor directly); it never runs in CI, keeping the workspace suite offline by default, and provider drift surfaces as a failing gateway-side assertion rather than a stale doc. +- Regression, security, and performance (the exact CI gates, ci.yml): `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`; `cargo test --locked --workspace --exclude workshop --exclude workshop-server --all-features` plus doctests; `cargo fmt --all --check`; `cargo check -p gateway --no-default-features`; `cargo deny check`; the config-UI gates in `crates/gateway-config-ui/ui`: `npm ci`, then `npm run typecheck`, `npm run build`, `npm test` (build first, the tests import built dist). +- Exit criteria: every gate green; every acceptance criterion covered by a named test; the live-provider check recorded or explicitly deferred. + +## Decision Record + +- Decisions: + - Rebuild fresh on master rather than adopt the existing branch: branch `gate-way-tts-phase-1` (commit `a2bd9355`, 2026-09-07 10:50) already carries a full phase-1 implementation (~2,750 lines across 48 files) built around a `gateway-tts` service crate; the user chose "Discard it and rebuild fresh on master from the report"; the branch stays untouched and its code is not ported. The implementation base later moved from master to branch `add-tts-phase-1` (which carries PR #18's STT rework); the discard decision is unchanged. + - Routed model kind over an STT-style engine slot: speech calls are routable model calls that pipelines bind by name and voice, remote providers are first-class backends, and `GET /v1/models` discovery matters; the `[[stt_model]]` pattern has no endpoint, credential, or upstream concept because STT deliberately never leaves the machine. + - Inline handler over a service crate: the template is the `embeddings` handler in lib.rs, no module ceiling governs the file (only workshop-server has one), and a crate split adds machinery without a boundary to enforce; the branch's `AGENTS.md` rule blessing a `gateway-tts` crate exists only on that branch, so master carries no conflicting rule. + - Auth before body extraction: the `embeddings` handler's `Json` extractor parses the body before `check_auth` runs; speech instead takes the ungated `Caller` parts-extractor (auth.rs:54) and a raw `Request`, runs `check_auth` first, then extracts `Json` manually (re-adding an ungated `FromRequest` import — axum is an unconditional dependency, so the headless check stays green), preserving the clean 401-before-400 test order. Master's in-handler transcription extraction is gone: PR #18 moved the STT routes into gateway-stt behind the stt-gated `authorize_stt_route` middleware (lib.rs 610-625), which also owns `begin_inference`, the cancellation select, and the realtime Origin gate — not reusable by an unconditional speech route, and not cloned. + - `voice` as an untagged string-or-`{"id"}` enum: OpenAI's object form stays representable, and voice validation stays at the route against per-model catalog data because voice sets are per-checkpoint, never a shared constant. + - Closed `response_format` enum with a structural `mp3` pin: OpenAI defaults to mp3 while Together defaults to wav, so an unpinned default silently changes the wire for a Together-backed model; the pin lives in the type (`#[serde(default)]` on the enum resolves an omitted field to `mp3` at deserialization), so no route can forget it; Together-only `raw`/`mulaw` stay unrepresentable until the enum is deliberately widened. + - A dedicated audio streaming client: `streaming_client()` is connect-timeout-only and shared with chat SSE, where a `read_timeout` could kill long thinking pauses; speech gets its own client so a dead upstream cannot hold a dominion permit forever. The first-response deadline is separate from the body-idle deadline: a 4,096-character batch generation can legitimately exceed 30 s to first headers, so the header wait carries its own ~120 s budget and the 30 s per-read idle guards only the opened body. Both deadlines are applied in `send_speech` rather than on the client, because reqwest arms a client-level `read_timeout` during the header wait (reqwest 0.12 `PendingRequest::poll`), which would cap time-to-headers at the idle budget; the client carries only connect timeout and keepalive. + - A bounded background relay owns the speech stream's resources: the permit and cancellation guard live in a spawned task feeding a bounded channel to the HTTP body, with static named bounds on total lifetime (60 min), response bytes (1 GiB), upstream read idle after headers (30 s), and blocked downstream delivery (60 s), so neither a drip-feeding provider nor a stalled client can hold a dominion slot indefinitely (A5, A27); configurability waits for demonstrated operator need. The same shape is a candidate for `relay_sse` later; this plan does not change chat. + - Distinct 429/503 codes on the speech path only, at a named seam: `ProtocolError::classify` today renders upstream 429 as `upstream_client_error` and 503 as 502 `upstream_error`, and it and its tests stay frozen — chat, embedding, and rerank envelopes are bit-identical. The seam is two speech-only `GatewayError` variants (`UpstreamRateLimited` → 429 `rate_limit_error`/`upstream_rate_limited`, `UpstreamUnavailable` → 503 `server_error`/`upstream_unavailable`) beside `QueueRejected`, whose exhaustive gateway `classify()` makes the new arms compiler-forced; the `audio_speech` handler matches `ProtocolError::UpstreamStatus` into them and forwards everything else through the unchanged `Protocol` arm. New `ProtocolError` variants were rejected: shared-protocol's exhaustive `classify()` cannot compile them without editing the frozen function, and they would offer the codes to every route. A no-leak test on the speech route pins the negative half: an upstream error body carrying provider internals (stack text, internal hosts, request ids) never reaches the client. If a later implementer cannot do speech-only without touching `classify`, they stop and revise this plan; they do not silently globalize. + - Live verification is a repeatable gateway-only probe, not a one-shot manual check and not a provider-direct comparison: provider dialect drift is a standing watch item, and drift surfaces through the gateway's own responses, which is the surface clients actually see. The probe is a zero-dependency Node script (the repo's tooling language, so no unmanaged Python environment, no third-party imports, and offline `.test.mjs` coverage per the tools convention); it boots a freshly built gateway with a throwaway Together-backed profile, takes the vendor key from the process environment, and ferries it only to the gateway subprocess, whose `api_key = "${VAR}"` interpolation is the standard credential-delivery mechanism (A19 stays absolute: the script never calls a vendor). It never runs in CI. + - `voices` is speech-only and rejected on other kinds: symmetric with the chat-only-field discipline, so a stale or misplaced list fails loudly at load time naming the field. When present, the list is content-validated at load (no empty entries, no duplicates, beside the `effort_levels` checks in `validate_capabilities`); an empty list stays valid and skips the route voice check. + - The voices union reads the live routing table: the `list_models` pattern; `LiveState.model_allowlist` is informational only and consulted by no route. + - Byte passthrough with header mapping, and a body error on mid-stream failure or cancellation: audio cannot be re-validated per chunk, and no JSON envelope can follow 200 plus audio bytes, but a clean EOF would read as a complete response, so the relay terminates failed or cancelled streams with a stream error item that fails the client's body read. + - `launch_options` goes fallible and refuses unknown kinds: the `_ => ServeMode::Chat` catch-all (runtime.rs:714) would compile clean and launch a speech model as a chat server; `server/support.rs` matches `ServeMode` exhaustively, so the hazard is solely `launch_options` never producing a speech mode. `launch_options` returns `Result` and the already-fallible `launch_options_for` propagates, the speech arm errors "local speech models are not yet supported" through a new `LocalError` variant, and because `ModelKind` is `#[non_exhaustive]` the wildcard remains but becomes `_ => Err(...)` — any future kind fails loudly instead of launching as chat. The `tool_dialect` wildcard (runtime.rs 493-496) stays; its chat-default is deliberate and harmless for a kind that never launches. The kind check runs as a side-effect-free preflight before any provisioning: `start_impl` and `provision_artifacts_impl` refuse an unsupported kind before server provisioning, model download, or metadata writes, so a misdeclared local speech model never downloads gigabytes or reports successful preparation (A6). + - Voice validation precedes dominion admission: the design sequenced the voice check after queue admission, but a 400 must not burn a queue slot; the workflow validates voice before admission and this entry records the deviation. + - Config examples use `api_key` with `"${VAR}"` interpolation (03-remote-models.md:14, syntax in 02-configuration-file.md:50-56); the design's `secret = "env:..."` example predates the config schema and is not followed. + - Config UI is in scope, the voice picker is not: `speech` joins the Kind options and `voices` gets a chips field visible only for speech models (config-data editing, the same control class as `effort_levels`); a request-time voice picker stays a non-goal, and Discover's hardcoded `kind: "chat"` (discover-view.ts:742) is a named gap left for the discover flow's own pass. + - `validate()` returning `&'static str` matches `ChatRequest::validate` (wire.rs:73) exactly; it is conformance, not a deviation. + - Speech-model sampling defaults are pinned in phase 2, not phase 1: the design's Risks section asks the gateway to pin them (greedy decoding is unstable for Orpheus; the working recipe is temperature ~0.6 with top-k and repetition_penalty >= 1.1), but neither OpenAI's nor Together's speech dialect carries sampling fields, so the remote passthrough has no landing site and a phase-1 pin would only invent unpassable fields; the pin lands with the local engine that actually generates. Deferred per operator ruling. +- Rejected alternatives: + - Adopt-and-verify the branch: rejected by the user's fork choice; revisit only if the rebuild stalls, since the branch remains as a reference. + - A `gateway-tts` service crate: rejected with the inline-handler decision; revisit if lib.rs gains a module ceiling or phase 2 gives speech a lifecycle to isolate. + - An engine-slot `[[tts_model]]` catalog: no remote passthrough, no per-model routing, no `GET /v1/models` visibility; phase 2's local engine still follows the gateway-stt crate trio as its structural template. + - Clients calling providers directly: breaks credential isolation, the gateway's founding invariant. + - A generic byte-passthrough route with no `ModelKind`: skips the kind guard, catalog metadata, and queue admission every other workload gets. + - ElevenLabs and Baseten adapters: structurally divergent (voice id in the URL path, `xi-api-key` auth, format as a query parameter); a separate feature if ever wanted. +- Assumptions, risks, and notes: + - Field-tunable values: the per-read body idle deadline ships at 30 s (anywhere in 30-60 s satisfies the design), the first-response deadline at ~120 s, `tcp_keepalive` at 60 s, the `invalid_voice` code spelling, and voices entries as `{"id", "name"}` objects. + - Provider dialect drift: Together's speech dialect differs from OpenAI's in defaults and streaming shape (SSE of base64 PCM with `stream=true` and `response_format=raw` only), so the gated live probe records the observed dialect rather than trusting provider docs. Together's SSE mode is unrequestable in phase 1 because `raw` is rejected at the wire boundary; `stream_format = "sse"` passthrough is forward-looking for SSE-capable OpenAI-compatible providers, and phase-1 Together speech is non-streaming. + - Codebase anchors were verified against master `d539a6d9` on 2026-09-07 and re-anchored to branch `add-tts-phase-1` (`f8e07fb6`, carrying PR #18) the same day: the STT routes live in gateway-stt behind the `authorize_stt_route` middleware, the guide already has `05-speech.md` (Speech-to-Text), and no `FromRequest` import remains in the gateway crate. Line numbers are approximate and cited with symbols. + - Branch `gate-way-tts-phase-1` stays in the repository untouched; deleting it is a separate explicit action. + +## Project survey + +Surveyed 2026-09-07 on branch `add-tts-phase-1` at its original base `f8e07fb6` (carrying PR #18's STT rework); re-validated 2026-09-09 against the rebased branch base `5edcb3c9` (current cppalliance master), whose tooling, CI, and layout this survey records. Architecture anchor: `vibe/archdoc.md`, read in full; its component list and invariants A1-A30 anchor the component map below. The invariants this plan leans on: A2 (each credential, connection record, lifecycle, and persisted state has exactly one owning subsystem), A3 (explicit dependency direction), A5 (bound every loop, queue, wait, stream, retry, and tool invocation), A6 (validate capabilities and semantics before queue admission or side effects), A19 (vendor and remote-service credentials stay inside the gateway), A27 (hold stream permits until body termination and expire stalled reads). + +- Build: + - `cargo build` builds the gateway, the workspace default member (`default-members = ["crates/gateway"]`; the binary is `promptforge-gateway`). `cargo build -p workshop` builds the Tauri desktop app (Windows: Visual Studio C++ workload; Linux: webkit2gtk/ssl system packages). `cargo workshop` (a `.cargo/config.toml` alias for `run -p build-workshop`) builds the gateway and desktop app together. + - Prerequisites: Rust 1.89 (pinned by `rust-toolchain.toml`, also the MSRV and `rust-version`; edition 2024) and Node.js 22. After cloning, `npm ci` once in each UI folder (`crates/workshop-server/ui`, `crates/gateway-config-ui/ui`); crate build scripts bundle the UIs with esbuild into `OUT_DIR`, and nothing UI-built is committed. + - `cargo check -p gateway --no-default-features` is the headless gate and must stay green. Gateway default features are `local`, `web-search`, `config-ui`, `stt`; `test-fixtures` is the opt-in feature supplying deterministic local-runtime bindings for cross-crate behavior tests. + - `cargo run -p build-user-guide` regenerates `guide/src/SUMMARY.md` and the per-part `index.md` files (that crate owns them; never hand-edited). +- Focused test command pattern: `cargo test -p ` for one crate's tests; `cargo test -p --test it ` for the integration harness binaries (CI's boundary guard: `cargo test -p gateway-stt --test it architecture`). UI: `npm run typecheck && npm run build && npm test` inside the relevant `ui/` directory. Tools scripts: `node --test tools/.test.mjs`. +- Full-suite test command: `cargo test --locked --workspace --exclude workshop --exclude workshop-server --all-features`, plus doctests via the same invocation with `--doc`. Desktop: `cargo test --locked -p workshop -p workshop-server`. Bare `cargo test` at the root covers only the gateway default member. +- Linters and formatters: `cargo fmt --all --check` (`rustfmt.toml`: `style_edition = "2024"`); `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings` (the workshop pair is linted separately on Windows CI: `cargo clippy -p workshop -p workshop-server --all-targets -- -D warnings`); `cargo deny check` and `cargo audit` (supply-chain job, `deny.toml`); `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server` with `RUSTDOCFLAGS=-D warnings`; STT native-policy gates (`cargo rustc -F unsafe-code` on `gateway-stt-engine`, `gateway-stt-backend-whisper`, and `gateway-stt`; `cargo check -p gateway-whisper-ffi` for the FFI crate's own lint policy); the product dependency-boundary guard `cargo test -p gateway-stt --test it architecture`. Opt-in git hooks in `.githooks/` (enable with `git config core.hooksPath .githooks`): pre-commit runs fmt, pre-push runs the headless check, clippy, and cargo deny. +- Test placement and naming: Rust unit tests live in `#[cfg(test)]` modules beside the code. Gateway integration tests live in `crates/gateway/tests/it/` as a single harness binary (`main.rs`) with shared scaffolding in `support.rs` and one module per surface (`boot.rs`, `chat.rs`, `embeddings.rs`, `profiles.rs`, `progress.rs`, `queue.rs`, `rerank.rs`, `sidecar.rs`, `surface.rs`; `cache.rs`/`cuda.rs`/`local.rs` gated on `feature = "local"`, `icon.rs` Windows-only, `realtime_stt` gated on `feature = "stt"`, `web_search.rs` gated on `feature = "web-search"`). The suite runs a fake OpenAI backend behind the real gateway on a caller-owned ephemeral listener, with rendezvous shutdown and arrivals channels plus per-request release handles instead of sleeps. `gateway-stt` has its own `tests/it/` harness; its `architecture.rs` module is the CI dependency-boundary guard. `product-integration-tests` holds the boundary-neutral cross-product compatibility tests. UI tests are colocated `src/**/*.test.mjs` run by `node --test` (the workshop UI adds `test/**/*.mjs`). Every `tools/*.mjs` pairs with a `.test.mjs`. `clippy.toml` allows unwrap/expect in tests only. +- Directory map: `crates/` holds the 34 Rust workspace members (`members = ["crates/*"]`; `crates/shared-ui` is TypeScript/CSS only and excluded from the glob). `design/` on this branch holds the as-built and verification note (`design-gateway-tts-phase-1.md`, `note-gateway-tts-phase-1-verification.md`); the original endpoint report `report-gateway-tts-endpoint.md` was removed from this repository when design records moved out. `guide/` is the mdBook user guide (`src/` split into `agent/`, `gateway/`, `language/`, `workshop/`; generated `SUMMARY.md`; the gateway part runs 01-10, with `05-speech.md` for STT). `prompts/` holds example pipelines. `tools/` holds repo tooling: `stage-gateway-sidecar.mjs` with its paired test, `check-stt-native-workflow.test.mjs` (tests the STT native workflow files), `validate-rust-1.89.0.ps1`, and `document.md` (the guide-rebuild tool). `vibe/` holds dated run records plus `archdoc.md` and `archdoc-next.md`. `images/` holds README artwork. `.github/workflows/` carries CI (`ci.yml` jobs: check, check-workshop, check-workshop-linux, ui, msrv, supply-chain) plus the release, nightly, guide, STT Miri, whisper-lib, and installer-smoke workflows (`dist-ci/` holds the shared dist build setup). `.githooks/`; `.cargo/config.toml` (Windows static CRT, the `cargo workshop` alias). Root files: `AGENTS.md`, `Cargo.toml`/`Cargo.lock`, `clippy.toml`, `rustfmt.toml`, `rust-toolchain.toml`, `deny.toml`, `dist-workspace.toml`, `gateway.local.example.toml`, `README.md`, `LICENSE` (BSL-1.0), `vibe-ledger.md`. +- Component boundaries (archdoc components mapped to crate prefixes): + - executor (parses and executes prompt pipelines and agent programs): `promptforge` facade (integrator library, lib-only; no standalone CLI binary crate exists at this commit), `promptforge-core`, `promptforge-agent`, `promptforge-parser`, `promptforge-lua` (the Lua VM boundary), `promptforge-store` (the run-scoped store), `promptforge-model-client`, `promptforge-tools`, `promptforge-tool-picker`, `promptforge-webfetch`, `promptforge-web-search`, `promptforge-core-support`. + - gateway (owns model routing, provider access, and local inference lifecycle; sole credential holder per A2/A19): `gateway`, `gateway-config`, `gateway-config-ui`, `gateway-local`, `gateway-routing`, `gateway-logging`, `gateway-web-search`, and the STT stack `gateway-stt`, `gateway-stt-engine`, `gateway-stt-backend-whisper`, `gateway-whisper-ffi`. + - workshop UI (desktop shell plus in-process server): `workshop`, `workshop-server`. + - shared substrate (progress, loopback discovery, protocol, sidecar): `shared-progress`, `shared-loopback`, `shared-protocol`, `shared-sidecar`, `shared-ui`. + - build tooling (linked into no deliverable): `build-llama-cuda`, `build-ui`, `build-user-guide`, `build-workshop`. + - cross-product tests: `product-integration-tests` (`publish = false`; nothing depends on it). + - Dependency directions per archdoc (A3): executor depends on gateway, store, Lua VM boundary, shared substrate; the CLI shell and workshop UI depend on executor, gateway, store, shared substrate; the Lua VM boundary depends on gateway, store, shared substrate; gateway depends on shared substrate; store and shared substrate depend on nothing. The root AGENTS.md's four cross-product dependency rules encode the same directions. +- Conventions summary: crate name prefixes encode product membership (`gateway*`, `promptforge*`, `workshop*`, `shared-*`, `build-*`). Prefer types and compiler checks, then behavior tests and deterministic fault injection; structural enforcement (parsers, snapshots, allowlists, ceilings, topology checks) enters only with explicit plan approval, which is why the old `tools/check-*` guard scripts are gone (vibe run 2026-09-08-1). Behavior changes ship with tests in the same change, and refactors preserve product and behavior tests. Every public type, function, and module carries a `///` doc comment (`missing_docs` warns workspace-wide; `cargo doc` is the project documentation). `unsafe_code` is forbidden workspace-wide, with FFI isolated in `gateway-whisper-ffi` where every unsafe block documents its invariants; clippy `all` is deny, `pedantic` warn, `unwrap_used`/`expect_used` denied outside tests. Library and serve paths never call `process::exit`, install process-global state, or invoke compilers and build tools. Long-running work reports through `shared-progress` hubs. Cargo features gate only real constraints (`local`, `stt`, `web-search`, `config-ui`, `cuda`), and feature-disabled builds must not leak optional types into core paths. Shared external dependencies are pinned once in root `[workspace.dependencies]`, with comments explaining resolver-driven version choices. Non-doc comments state a non-obvious why and cite upstream issue URLs for external workarounds. Nested AGENTS.md files bind their subtrees. CI enforces a clean tree after builds (no build step may write into the repo). +- Rules manifest (each file governs its containing directory's subtree): `AGENTS.md` (whole workspace); `crates/gateway/AGENTS.md`; `crates/gateway-config/AGENTS.md`; `crates/gateway-local/AGENTS.md`; `crates/gateway-logging/AGENTS.md`; `crates/gateway-routing/AGENTS.md`; `crates/gateway-stt/AGENTS.md`; `crates/gateway-stt-engine/AGENTS.md`; `crates/gateway-web-search/AGENTS.md`; `crates/gateway-whisper-ffi/AGENTS.md`; `crates/promptforge/AGENTS.md`; `crates/promptforge-agent/AGENTS.md`; `crates/promptforge-core/AGENTS.md`; `crates/promptforge-core-support/AGENTS.md`; `crates/promptforge-lua/AGENTS.md`; `crates/promptforge-model-client/AGENTS.md`; `crates/promptforge-parser/AGENTS.md`; `crates/promptforge-store/AGENTS.md`; `crates/promptforge-tools/AGENTS.md`; `crates/promptforge-web-search/AGENTS.md`; `crates/promptforge-webfetch/AGENTS.md`; `crates/shared-loopback/AGENTS.md`; `crates/shared-progress/AGENTS.md`; `crates/shared-protocol/AGENTS.md`; `crates/shared-sidecar/AGENTS.md`; `crates/shared-ui/AGENTS.md`; `crates/workshop/AGENTS.md`; `crates/workshop/icons/AGENTS.md`; `crates/workshop-server/AGENTS.md`; `crates/workshop-server/ui/AGENTS.md`. Crates without their own AGENTS.md (root rules only): `build-llama-cuda`, `build-ui`, `build-user-guide`, `build-workshop`, `gateway-config-ui`, `gateway-stt-backend-whisper`, `product-integration-tests`, `promptforge-tool-picker`. + +## Execution Instructions + +Execution rules: tests land with each behavior change in the same change; every step ends with the build plus that step's focused tests green, and the full gate suite (per the Testing Plan) runs at each component's end and at the final step; where implementation contradicts or extends a Decision Record entry, revise this plan in the same change, naming what forced it. + +Components, in dependency order: + +1. Config kind (`gateway-config`, `gateway-config-ui`, then `gateway-local`): `ModelKind::Speech` is the type every later component references, the config UI's Kind list and chips ride the same schema change, and `gateway-local` matches on the kind, so its arm builds immediately after the enum exists. +2. Wire and upstream (`shared-protocol`): the wire types name no config types, so this component is independent of component 1; it builds second because routes compile against both and the config change carries the config-UI work and its gates, which are cheapest to run first. Pieces build sequentially: `SpeechRequest` before the upstream that consumes it. +3. Routes (`gateway`): compiles against components 1 and 2. Pieces build sequentially: the speech route first (it carries the streaming core and the first `build_router` edit), the voices route second (it needs only the voices capability but shares `build_router` and the `speech.rs` integration file with the speech route). +4. Live verification: needs a working route from component 3. +5. Documentation: describes as-built, verified behavior; the verification note feeds the guide chapter. +6. As-built design document: reconciles the design against the finished work, so it is final. + +### Step 1: Speech model kind and voices capability in gateway-config and the config UI [completed] + +- `crates/gateway-config/src/config.rs`: add `ModelKind::Speech` (serde and `Display` spelling `"speech"`); add `Capabilities.voices: Vec` with `#[serde(default)]`. +- `crates/gateway-config/src/config/validate.rs`: confirm `validate_kind_scope` covers speech models under the existing chat-only-field rejection, and add the symmetric rejection of a non-empty `voices` list on non-speech models, naming the field; in `validate_capabilities` (671-705, beside the `effort_levels` checks) reject empty voice entries and duplicate voices at load — an empty `voices` list stays valid and skips the route voice check. +- Config UI (`crates/gateway-config-ui/ui`): add `speech` to the shared Kind options list (`src/views/models-view.ts` 647-648, one list for every non-STT entry remote or local); add a `voices` chips field with `visibleWhen: kind === "speech"` reusing the existing `type: "chips"` editor (`components/chip-input.ts`, registry mechanism at `settings-registry.ts:56`). Discover's hardcoded `kind: "chat"` (`discover-view.ts:742`) stays a named gap. +- Tests: config parse/serialize round-trip of `kind = "speech"` with `voices`; each chat-only field rejected on a speech model; `voices` rejected on chat, embedding, and classifier models; empty voice entries and duplicates rejected at load; UI: update the dropdown-values pin to the four-kind list (`model-detail.test.mjs` ~78-82) and add a pick-speech/add-chip/save test asserting the PUT body (save-then-PUT pattern at `model-detail.test.mjs` ~409-436). +- Doc comments: the kind enumerations in `crates/gateway-config/src/config/accessors.rs` (~882-883 remote `kind()`, ~1223-1224 local `kind()`) gain `speech`. +- Verification adds the config-UI gates exactly as CI runs them (`npm ci`, then `npm run typecheck`, `npm run build`, `npm test` in `crates/gateway-config-ui/ui`; build first, the tests import built dist). + +### Step 2: launch_options goes fallible and refuses unknown kinds in gateway-local [completed] + +- `crates/gateway-local/src/runtime.rs`: make `launch_options` (699-720) return `Result`; the already-fallible wrapper `launch_options_for` (722-731) propagates. The Chat/Embedding/Classifier arms stay; `ModelKind::Speech` errors "local speech models are not yet supported" through a new `LocalError` variant (beside `UnsupportedPlatform`, error.rs ~12-17); the wildcard becomes refuse-unknown (`_ => Err(...)`) instead of `_ => ServeMode::Chat` (714) — `ModelKind` is `#[non_exhaustive]`, so a wildcard must remain but it never maps to chat. Leave the `tool_dialect` wildcard (493-496); its chat-default is deliberate. +- Preflight before side effects (A6): extract the kind mapping into a side-effect-free `serve_mode_for(kind) -> Result`; `start_impl` runs it at the top of the per-model closure, before `ensure_model_with_cancellation` downloads anything and before `maybe_write_sidecar` writes metadata; `provision_artifacts_impl` checks every model's kind before provisioning the shared server or any model, so an all-speech profile fails without a single side effect; a mixed profile keeps supported-model progress while returning a per-model failure for each unsupported kind. +- Tests: the speech arm errors rather than launching as chat; an all-speech profile touches neither the server provisioner nor the model store; a mixed profile provisions only supported models. + +### Step 3: SpeechRequest wire type [completed] + +- `crates/shared-protocol/src/wire.rs`: add `SpeechRequest` beside `EmbeddingRequest` with `validate() -> Result<(), &'static str>` mirroring `ChatRequest::validate`; `voice` is an untagged string-or-`{"id"}` enum; `response_format` is a closed enum with `#[serde(default)]` resolving an omitted field to `mp3`, so the pin is structural and no route can forget it; a flattened `rest` with `RESERVED` naming the seven known fields preserves verbatim passthrough. +- Tests: the wire validation table (empty `model`, empty `input`, over-cap `input` past 4096 characters, out-of-range `speed`, unknown `response_format`), the intentional rejection of Together's `raw` (the closed enum stays unrepresentable, so the exclusion is pinned rather than incidental), the serde-default `mp3` resolution, and verbatim passthrough of unnamed fields. + +### Step 4: Speech upstream and audio streaming client [completed] + +- `crates/shared-protocol/src/upstream.rs`: add `StreamedAudio { content_type, body }` beside `StreamedChunks`; add `Upstream::send_speech` defaulting to `ProtocolError::ModelUnavailable`; implement `OpenAiUpstream::send_speech` over the raw `post` helper, substituting `upstream_model` and forwarding the body otherwise verbatim. `post` already returns `ProtocolError::UpstreamStatus` with a capped body (upstream.rs 218-228, `MAX_ERROR_BODY` at http_util.rs:12), so no error-path work exists here. The `send()` await gains its own named first-response deadline (~120 s, a module-level constant), separate from the body-idle `read_timeout`, so a slow batch generation is not killed by the stalled-body budget. +- `crates/shared-protocol/src/http_util.rs`: add `audio_streaming_client()` beside `streaming_client()` (connect timeout 10 s, `tcp_keepalive` 60 s, and deliberately no `read_timeout` — reqwest arms it during the header wait, so both deadlines live in `send_speech`: a named first-response deadline of ~120 s for the header wait and a named 30 s per-read idle deadline for the opened body); `OpenAiUpstream` gains a third client field and the chat SSE client is untouched. +- `crates/shared-protocol/src/error.rs` is not touched: `ProtocolError::classify` and its table test stay frozen (the Decision Record names the speech-only seam in the gateway crate). +- Root `Cargo.toml`: pin `bytes = "1"` in `[workspace.dependencies]` (the house convention: every shared external dependency is workspace-pinned); `crates/shared-protocol/Cargo.toml`: `bytes.workspace = true`. `crates/shared-protocol/README.md`: update where it enumerates `Upstream`. +- Tests: upstream tests mirroring the `serve_once`/`serve_stalled` module (model-name substitution, bearer forwarding, status and transport mapping asserting the `UpstreamStatus` shape for upstream 429/503 — never a client envelope — and untransformed byte streaming), plus the deadline separation: headers arriving after the 30 s body-idle budget but within the first-response budget are accepted, and an opened body that stalls past 30 s fails. The 429/503 envelope tests and the no-leak assertion live with the route in Step 5. + +### Step 5: POST /v1/audio/speech route [completed] + +- `crates/gateway/src/lib.rs`: add the `audio_speech` handler cloned from `embeddings` (853-882) with the speech deltas: auth before body extraction (the handler takes the ungated `Caller` parts-extractor and a raw `Request`, runs `check_auth` first, then extracts `Json` manually, re-adding an ungated `FromRequest` import; the stt-gated `authorize_stt_route` middleware is not reused — it owns `begin_inference`, the cancellation select, and the realtime Origin gate for the STT routes), the kind guard returning 400 `kind_mismatch`, voice validation before queue admission returning 400 `invalid_voice` naming the valid voices (an empty `voices` list skips the check), dominion queue admission, upstream `Content-Type` forwarding with the format-to-MIME fallback mapping (the fallback takes the framing selector, so an SSE upstream with a missing or invalid Content-Type is labeled `text/event-stream`, never audio), and `Content-Length` never set. The doc comment justifies the byte-passthrough departure and warns against any future `CompressionLayer` or whole-request `TimeoutLayer` on the route. Register `/v1/audio/speech` in `build_router`. +- Bounded relay: a spawned background task owns the upstream body, the dominion permit, and the `InFlightGuard` cancellation guard, feeding a small bounded channel to the HTTP body, so the permit's lifetime never depends on downstream polling. Named static constants bound the stream: 60 min total lifetime, 1 GiB response bytes, 30 s upstream read idle after headers, 60 s blocked on downstream delivery. Every terminal path (limit exceeded, deadline reached, upstream error, profile-switch cancellation, downstream gone) emits one `Err` item so the client's body read fails, then releases the permit; cancellation never surfaces as a clean EOF (the `relay_sse` RequestCancelled envelope at lib.rs 852-855 is the precedent for failing rather than truncating). +- `crates/gateway/src/error.rs`: add the speech-only `GatewayError::UpstreamRateLimited` (429 `rate_limit_error`/`upstream_rate_limited`) and `GatewayError::UpstreamUnavailable` (503 `server_error`/`upstream_unavailable`) variants beside `QueueRejected` (~56-60), with their two arms in the gateway's exhaustive `classify()` (294-442); the handler matches `ProtocolError::UpstreamStatus { status: 429 | 503, .. }` into them and forwards every other error through the unchanged `Protocol` arm (321). `shared-protocol/src/error.rs` is not touched. +- Admin status and crate docs: add an unconditional `endpoint_status` row for `/v1/audio/speech` beside the existing rows (lib.rs 1080-1102; do not reuse the stt-gated `with_speech_endpoint` helper at 1012-1026); list both new routes in the crate-doc "What ships" header (lib.rs 8-77). Doc comments: the kind enumerations in `crates/shared-protocol/src/wire.rs` (~592), `crates/gateway-routing/src/model.rs` (~46), and `crates/gateway/src/model_info.rs` (~48) gain `speech`. +- `crates/gateway/tests/it/speech.rs` (new, on the `support.rs` harness) and `crates/gateway/tests/it/main.rs`; route-level auth tests beside (not inside) the stt-gated `transcription_auth_tests` module in lib.rs — the speech tests stay ungated because the route is unconditional. +- Tests: 401-before-malformed-JSON, `kind_mismatch`, `invalid_voice` naming valid voices, an empty `voices` list skipping the check, byte-identical passthrough, content-type forwarding and per-format fallback, queue-full 503, client disconnect dropping the upstream stream and releasing the permit, the permit held for the stream's lifetime under `max_concurrency = 1` (copied from `stream_permit_is_held_until_the_stream_ends`, chat.rs:943), reverse `kind_mismatch` rows (a speech model on the chat, embeddings, and rerank routes, beside chat.rs:136, embeddings.rs:259, rerank.rs:168), mid-stream upstream `Err` after HTTP 200 (the client's body read fails and no JSON envelope appears in the audio), `GET /v1/models` showing `kind: "speech"` and `voices`, the 429/503 envelopes, the no-leak assertion that an upstream error body carrying provider internals never reaches the client, an unknown model returning 404 `model_not_found` on the speech route, a profile switch during an open body failing the client's next read and freeing the permit, `` emotion-tag passthrough in the offline suite, `stream_format: "sse"` forwarded into the outbound body, the bounded relay's boundary tests (exact byte limit, one byte over, total deadline, sub-idle upstream drip, saturated downstream channel, upstream idle after headers, delayed-but-accepted headers, disconnect, cancellation, upstream error — each proving permit release and admission of a subsequent request), and two rows for the new variants in `gateway_error_classify_is_table_driven` (error.rs 487-570). + +### Step 6: GET /v1/audio/voices route [completed] + +- `crates/gateway/src/lib.rs`: add the `audio_voices` handler following the `list_models` pattern over `routing.models()`, taking the union of the active profile's speech models' `voices`, deduplicated and sorted, entries as `{"id", "name"}` objects; register `/v1/audio/voices` in `build_router`. +- Tests in `crates/gateway/tests/it/speech.rs`: the voices-union shape (the id-first entry shape is a compatibility surface and is test-pinned), deduplication and ordering, an empty union when the profile has no speech models, non-speech models contributing nothing, and an unauthenticated request rejected with 401 before any union is computed. + +### Step 7: Live gateway speech probe and verification note [completed] + +- `tools/gateway-tts-live.mjs` (new, dev-only, never in CI): a zero-dependency Node script (built-in `fetch`, the repo's tooling language) that builds the gateway fresh (`cargo build -p gateway`, so a stale binary can never be tested against a recorded current hash), boots it on an ephemeral loopback port with a throwaway Together-backed profile, and asserts the speech and voices surfaces through the gateway's responses: the default-format call proving the mp3 pin reached the provider, a `wav` call, byte-nonempty audio, the voices union shape, and the 429/503 envelopes if provocable. The vendor key comes from the process environment only (no dotenv parsing) and is ferried to the gateway subprocess, whose `api_key = "${TOGETHER_API_KEY}"` interpolation is the standard credential-delivery mechanism; the script never calls a vendor directly (A19) and never prints or persists the key. With no key in the environment it prints a skip and exits 0. +- `tools/gateway-tts-live.test.mjs`: offline tests per the tools convention, covering gateway startup failure, readiness timeout, request failure, assertion failure, process cleanup, key-absent skip, and secret-free output. +- `design/note-gateway-tts-phase-1-verification.md`: record the observed dialect through the gateway (default-format `Content-Type` proving the mp3 pin reached the provider, chunked vs `Content-Length`, emotion-tag handling, 429/503 envelopes if provocable, rejected or ignored fields), naming the exact commit and tree hash exercised; when the key is absent, write the same note as a recipe with a deferral line and do not block on it. +- Verification: `node --test tools/gateway-tts-live.test.mjs` passes; the live run's gateway assertions pass when a key is present. + +### Step 8: Documentation [completed] + +- `crates/gateway/README.md`: add a "Speech synthesis models" section beside "Speech-to-text models" (:100) with example catalog entries using `protocol = "openai"` and `api_key = "${TOGETHER_API_KEY}"` (the config schema's credential field with `"${VAR}"` interpolation; the design's `secret = "env:..."` example predates the schema). The section states the streaming scope plainly: `stream_format` is forwarded to the provider, but Together's SSE mode requires `response_format = "raw"`, which the wire type rejects, so phase-1 Together speech is non-streaming. +- `guide/src/gateway/06-speech-synthesis.md`: new chapter (the STT chapter `05-speech.md` already exists); `git mv` the following chapters highest-first (10→11, 09→10, 08→09, 07→08, 06→07), then regenerate `guide/src/SUMMARY.md` with `cargo run -p build-user-guide` — the assembler owns SUMMARY.md and it is never hand-edited. The chapter covers the `[[model]]` speech fields, the `voices` capability, the `/v1/audio/speech` request shape (defaults, rejections, `instructions`), the `/v1/audio/voices` route (id-first `{"id", "name"}` entries, with a note that tolerant clients also accept plain strings — the entry shape is a compatibility surface), and the `stream_format=sse` caveat stated as a limitation: Together's `stream=true` SSE mode needs `response_format = "raw"`, which the wire enum rejects, so Together SSE cannot be requested in phase 1 and `stream_format` forwarding is forward-looking for SSE-capable OpenAI-compatible providers. +- Update the kind lists in `guide/src/gateway/03-remote-models.md` (:40 and :71) and `guide/src/gateway/04-local-models.md` (:44) to include `speech`. +- `gateway.local.example.toml`: add a commented speech `[[model]]` block beside the existing commented chat block (:76-92). +- Root README only if it enumerates gateway endpoints or capabilities. +- Verification: the gate suite green and the guide builds. + +### Step 9: As-built design document [completed] + +- `design/design-gateway-tts-phase-1.md`: spawn a generator subagent to write the design as built: a title stating what was built, a standalone executive summary, and a numbered list of the key design choices, reconciled against the finished work and this plan's Decision Record (including the bounded relay, the split deadlines, the provisioning preflight, the Node live probe, and the A19 credential numbering). +- Metadata synchronization: the six frontmatter todo statuses flip to `completed` in this final execution commit, and the as-built's commit and tree references name the objects the finished history actually contains. + +Deferred and out of scope: the phase-2 local engine (a spike choosing between a managed CrispASR child and llama-server plus in-process SNAC decode, scored on time-to-first-audio, real-time factor under concurrent load, and an ASR-roundtrip quality gate); `ServeMode::Speech`; encoder and WAV-header policy; speech-model sampling-default pins (the design's Risks recipe targets Orpheus generation and lands with the local engine; no remote speech dialect carries sampling fields); Together SSE reframing; Workshop playback; the config-UI voice picker; text conditioning; ElevenLabs and Baseten adapters. diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 00000000..c5500f60 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +2026-09-07-2-gateway-tts-phase-1