diff --git a/sentry_streams/sentry_streams/adapters/arroyo/adapter.py b/sentry_streams/sentry_streams/adapters/arroyo/adapter.py
index a3f829bc..d7a53f59 100644
--- a/sentry_streams/sentry_streams/adapters/arroyo/adapter.py
+++ b/sentry_streams/sentry_streams/adapters/arroyo/adapter.py
@@ -36,6 +36,7 @@
from sentry_streams.adapters.stream_adapter import (
PipelineConfig,
RuntimeState,
+ StartOptions,
StreamAdapter,
)
from sentry_streams.config_types import (
@@ -326,7 +327,7 @@ def create_processors(self) -> None:
for source, consumer in self.__consumers.items()
}
- def _run(self) -> None:
+ def _run(self, options: StartOptions) -> None:
"""
Starts the pipeline
"""
diff --git a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py
index 9c0ff705..ce816430 100644
--- a/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py
+++ b/sentry_streams/sentry_streams/adapters/arroyo/rust_arroyo.py
@@ -30,6 +30,7 @@
from sentry_streams.adapters.stream_adapter import (
PipelineConfig,
RuntimeState,
+ StartOptions,
StreamAdapter,
)
from sentry_streams.config_types import (
@@ -591,13 +592,15 @@ def routing_function(msg: Message[Any]) -> str:
)
return build_branches(stream, step.routing_table.values())
- def _run(self) -> None:
+ def _run(self, options: StartOptions) -> None:
"""
Starts the pipeline
"""
# TODO: Support multiple consumers
assert len(self.__consumers) == 1, "Multiple consumers not supported yet"
consumer = next(iter(self.__consumers.values()))
+ if options.group_instance_id is not None:
+ consumer.set_group_instance_id(options.group_instance_id)
self._set_status(RuntimeState.CONSUMING)
consumer.run()
diff --git a/sentry_streams/sentry_streams/adapters/stream_adapter.py b/sentry_streams/sentry_streams/adapters/stream_adapter.py
index 29954bdc..4845e4cf 100644
--- a/sentry_streams/sentry_streams/adapters/stream_adapter.py
+++ b/sentry_streams/sentry_streams/adapters/stream_adapter.py
@@ -78,6 +78,11 @@ def can_transition_to(self, state: RuntimeState) -> bool:
return self.rank < state.rank
+@dataclass(frozen=True)
+class StartOptions:
+ group_instance_id: str | None = None
+
+
@dataclass(frozen=True)
class RuntimeStatus:
state: RuntimeState
@@ -126,7 +131,7 @@ def begin_start(self) -> RuntimeStatus:
self.__status = RuntimeStatus(RuntimeState.STARTING)
return self.__status
- def run(self) -> None:
+ def run(self, options: StartOptions | None = None) -> None:
state = self.status.state
if state is RuntimeState.STOPPING:
self._set_status(RuntimeState.STOPPED)
@@ -135,7 +140,7 @@ def run(self) -> None:
raise RuntimeStateError(f"cannot run runtime while it is {state}")
try:
- self._run()
+ self._run(options if options is not None else StartOptions())
except Exception as exc:
self._set_status(RuntimeState.ERRORED, exc)
raise
@@ -251,7 +256,7 @@ def broadcast(
raise NotImplementedError
@abstractmethod
- def _run(self) -> None:
+ def _run(self, options: StartOptions) -> None:
"""
Starts the pipeline
"""
diff --git a/sentry_streams/sentry_streams/control.py b/sentry_streams/sentry_streams/control.py
index 458fe9bf..ca16d54a 100644
--- a/sentry_streams/sentry_streams/control.py
+++ b/sentry_streams/sentry_streams/control.py
@@ -6,6 +6,7 @@
from sentry_streams.adapters.stream_adapter import (
RuntimeStatus,
+ StartOptions,
StreamAdapter,
)
@@ -33,14 +34,14 @@ def __init__(self, runtime: StreamAdapter[Any, Any]) -> None:
def snapshot(self) -> RuntimeStatus:
return self._runtime.status
- def request_start(self) -> RuntimeStatus:
+ def request_start(self, options: StartOptions | None = None) -> RuntimeStatus:
"""
Ask the pipeline to start (non-blocking).
"""
with self._lock:
status = self._runtime.begin_start()
if self._run_future is None:
- self._run_future = self._executor.submit(self._runtime.run)
+ self._run_future = self._executor.submit(self._runtime.run, options)
self._run_future.add_done_callback(self._run_finished)
return status
diff --git a/sentry_streams/sentry_streams/dummy/dummy_adapter.py b/sentry_streams/sentry_streams/dummy/dummy_adapter.py
index b7651e3e..89e34489 100644
--- a/sentry_streams/sentry_streams/dummy/dummy_adapter.py
+++ b/sentry_streams/sentry_streams/dummy/dummy_adapter.py
@@ -1,6 +1,10 @@
from typing import Any, Callable, Optional, Self, Sequence, Type, TypeVar, cast
-from sentry_streams.adapters.stream_adapter import PipelineConfig, StreamAdapter
+from sentry_streams.adapters.stream_adapter import (
+ PipelineConfig,
+ StartOptions,
+ StreamAdapter,
+)
from sentry_streams.pipeline.function_template import (
InputType,
OutputType,
@@ -95,7 +99,7 @@ def router(self, step: Router[RoutingFuncReturnType, Any], stream: Any) -> Any:
ret[branch.root.name] = branch
return ret
- def _run(self) -> None:
+ def _run(self, options: StartOptions) -> None:
pass
def _shutdown(self) -> None:
diff --git a/sentry_streams/sentry_streams/rust_streams.pyi b/sentry_streams/sentry_streams/rust_streams.pyi
index e4c359cf..27b2c567 100644
--- a/sentry_streams/sentry_streams/rust_streams.pyi
+++ b/sentry_streams/sentry_streams/rust_streams.pyi
@@ -159,6 +159,7 @@ class ArroyoConsumer:
) -> None: ...
def add_threadpool(self, step_name: str, thread_count: int) -> None: ...
def add_step(self, step: RuntimeOperator) -> None: ...
+ def set_group_instance_id(self, group_instance_id: str) -> None: ...
def run(self) -> None: ...
def shutdown(self) -> None: ...
@property
diff --git a/sentry_streams/sentry_streams/server/control_server.py b/sentry_streams/sentry_streams/server/control_server.py
index 1f764918..6fe5637b 100644
--- a/sentry_streams/sentry_streams/server/control_server.py
+++ b/sentry_streams/sentry_streams/server/control_server.py
@@ -9,17 +9,48 @@
from sentry_streams.adapters.stream_adapter import (
RuntimeStateError,
RuntimeStatus,
+ StartOptions,
)
from sentry_streams.control import PipelineController
logger = logging.getLogger(__name__)
+class InvalidRequest(Exception):
+ """Raised when a request body cannot be read."""
+
+
class ControlHandler(BaseHTTPRequestHandler):
@property
def _controller(self) -> PipelineController:
return cast(ControlServer, self.server).controller
+ def _read_start_options(self) -> StartOptions | None:
+ length = int(self.headers.get("Content-Length") or 0)
+ raw = self.rfile.read(length) if length > 0 else b""
+ if not raw.strip():
+ return None
+
+ try:
+ payload = json.loads(raw)
+ except json.JSONDecodeError as exc:
+ raise InvalidRequest(f"body is not valid JSON: {exc}") from exc
+
+ if not isinstance(payload, dict):
+ raise InvalidRequest("body must be a JSON object")
+
+ unknown = set(payload) - {"group_instance_id"}
+ if unknown:
+ raise InvalidRequest(f"unknown fields: {', '.join(sorted(unknown))}")
+
+ group_instance_id = payload.get("group_instance_id")
+ if group_instance_id is not None and (
+ not isinstance(group_instance_id, str) or not group_instance_id
+ ):
+ raise InvalidRequest("group_instance_id must be a non-empty string")
+
+ return StartOptions(group_instance_id=group_instance_id)
+
def log_message(self, format: str, *args: Any) -> None:
logger.debug("control-server %s - %s", self.address_string(), format % args)
@@ -49,7 +80,8 @@ def do_POST(self) -> None:
try:
path = urlparse(self.path).path
if path == "/start":
- self._respond(202, self._controller.request_start().as_dict())
+ options = self._read_start_options()
+ self._respond(202, self._controller.request_start(options).as_dict())
elif path == "/stop":
self._respond_to_stop(self._controller.request_stop())
else:
@@ -62,7 +94,10 @@ def _respond_to_stop(self, snapshot: RuntimeStatus) -> None:
self._respond(code, snapshot.as_dict())
def _respond_to_failure(self, exc: Exception) -> None:
- if isinstance(exc, RuntimeStateError):
+ if isinstance(exc, InvalidRequest):
+ logger.info("control-server could not read %s: %s", self.path, exc)
+ self._respond(400, {"error": str(exc)})
+ elif isinstance(exc, RuntimeStateError):
logger.info("control-server rejected %s: %s", self.path, exc)
self._respond(409, {"error": str(exc)})
else:
diff --git a/sentry_streams/src/consumer.rs b/sentry_streams/src/consumer.rs
index 5ef2aa52..82cf299e 100644
--- a/sentry_streams/src/consumer.rs
+++ b/sentry_streams/src/consumer.rs
@@ -91,6 +91,9 @@ pub struct ArroyoConsumer {
/// processor is not lost. `run()` checks it before entering the run loop.
shutdown_requested: AtomicBool,
+ /// Static membership id provided by controller.
+ group_instance_id: Mutex