diff --git a/.sampo/changesets/multimodal-capture-correctness.md b/.sampo/changesets/multimodal-capture-correctness.md new file mode 100644 index 00000000..663f3526 --- /dev/null +++ b/.sampo/changesets/multimodal-capture-correctness.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +AI capture now records multimodal and structured content (thinking blocks, tool calls, media, and Responses API output items) faithfully across all providers and streaming paths, and redacts base64 media structurally without leaking raw bytes or over-redacting legitimate values. diff --git a/posthog/ai/anthropic/anthropic.py b/posthog/ai/anthropic/anthropic.py index fac04b81..a5a2299b 100644 --- a/posthog/ai/anthropic/anthropic.py +++ b/posthog/ai/anthropic/anthropic.py @@ -22,7 +22,6 @@ handle_anthropic_tool_delta, finalize_anthropic_tool_input, ) -from posthog.ai.sanitization import sanitize_anthropic from posthog.client import Client as PostHogClient from posthog import setup @@ -169,7 +168,7 @@ def generator(): if block: content_blocks.append(block) - if block.get("type") == "text": + if block.get("type") in ("text", "thinking"): current_text_block = block else: current_text_block = None @@ -248,16 +247,14 @@ def _capture_streaming_event( ) from posthog.ai.utils import capture_streaming_event - # Prepare standardized event data formatted_input = format_anthropic_streaming_input(kwargs) - sanitized_input = sanitize_anthropic(formatted_input, self._client._ph_client) event_data = StreamingEventData( provider="anthropic", model=kwargs.get("model", "unknown"), base_url=str(self._client.base_url), kwargs=kwargs, - formatted_input=sanitized_input, + formatted_input=formatted_input, formatted_output=format_anthropic_streaming_output_complete( content_blocks, accumulated_content ), diff --git a/posthog/ai/anthropic/anthropic_async.py b/posthog/ai/anthropic/anthropic_async.py index bebf2f69..8ea07840 100644 --- a/posthog/ai/anthropic/anthropic_async.py +++ b/posthog/ai/anthropic/anthropic_async.py @@ -24,7 +24,6 @@ handle_anthropic_tool_delta, finalize_anthropic_tool_input, ) -from posthog.ai.sanitization import sanitize_anthropic from posthog.client import Client as PostHogClient @@ -170,7 +169,7 @@ async def generator(): if block: content_blocks.append(block) - if block.get("type") == "text": + if block.get("type") in ("text", "thinking"): current_text_block = block else: current_text_block = None @@ -249,16 +248,14 @@ async def _capture_streaming_event( ) from posthog.ai.utils import capture_streaming_event - # Prepare standardized event data formatted_input = format_anthropic_streaming_input(kwargs) - sanitized_input = sanitize_anthropic(formatted_input, self._client._ph_client) event_data = StreamingEventData( provider="anthropic", model=kwargs.get("model", "unknown"), base_url=str(self._client.base_url), kwargs=kwargs, - formatted_input=sanitized_input, + formatted_input=formatted_input, formatted_output=format_anthropic_streaming_output_complete( content_blocks, accumulated_content ), diff --git a/posthog/ai/anthropic/anthropic_converter.py b/posthog/ai/anthropic/anthropic_converter.py index 3c611cac..ab5503fc 100644 --- a/posthog/ai/anthropic/anthropic_converter.py +++ b/posthog/ai/anthropic/anthropic_converter.py @@ -8,6 +8,7 @@ import json from typing import Any, Dict, List, Optional, Tuple +from posthog.ai.media import to_plain from posthog.ai.types import ( FormattedContentItem, FormattedFunctionCall, @@ -69,6 +70,30 @@ def format_anthropic_response(response: Any) -> List[FormattedMessage]: } content.append(function_call) + elif getattr(choice, "type", None) == "thinking": + content.append( + { + "type": "thinking", + "thinking": getattr(choice, "thinking", None), + "signature": getattr(choice, "signature", None), + } + ) + + elif getattr(choice, "type", None) == "redacted_thinking": + content.append( + { + "type": "redacted_thinking", + "data": getattr(choice, "data", None), + } + ) + + else: + # Catches blocks the branches above skip on falsy values (e.g. TextBlock(text="")) + # so empty-but-valid content survives instead of being silently dropped. + plain = to_plain(choice) + if isinstance(plain, dict) and plain.get("type"): + content.append(plain) + if content: message: FormattedMessage = { "role": "assistant", @@ -102,10 +127,15 @@ def format_anthropic_input( # Add user messages if messages: for msg in messages: - # Messages are already in the correct format, just ensure type safety + raw_content = msg.get("content", "") + content: Any = ( + [to_plain(item) for item in raw_content] + if isinstance(raw_content, list) + else raw_content + ) formatted_msg: FormattedMessage = { "role": msg.get("role", "user"), - "content": msg.get("content", ""), + "content": content, } formatted_messages.append(formatted_msg) @@ -161,6 +191,23 @@ def format_anthropic_streaming_content( } ) + elif block.get("type") == "thinking": + formatted.append( + { + "type": "thinking", + "thinking": block.get("thinking") or "", + "signature": block.get("signature"), + } + ) + + elif block.get("type") == "redacted_thinking": + formatted.append( + { + "type": "redacted_thinking", + "data": block.get("data"), + } + ) + return formatted @@ -325,6 +372,23 @@ def handle_anthropic_content_block_start( tool_in_progress: ToolInProgress = {"block": tool_block, "input_string": ""} return tool_block, tool_in_progress + elif block.type == "thinking": + thinking_block: StreamingContentBlock = { + "type": "thinking", + "thinking": getattr(block, "thinking", "") or "", + } + signature = getattr(block, "signature", None) + if signature: + thinking_block["signature"] = signature + return thinking_block, None + + elif block.type == "redacted_thinking": + redacted_block: StreamingContentBlock = { + "type": "redacted_thinking", + "data": getattr(block, "data", None), + } + return redacted_block, None + return None, None @@ -332,11 +396,15 @@ def handle_anthropic_text_delta( event: Any, current_block: Optional[StreamingContentBlock] ) -> Optional[str]: """ - Handle text delta event from Anthropic streaming. + Handle text, thinking, and signature delta events from Anthropic streaming. + + Thinking and signature deltas are accumulated into current_block in place but, + unlike text deltas, are not returned — the caller's accumulated_content is a + plain-text fallback and thinking output must not leak into it. Args: event: Delta event - current_block: Current text block being accumulated + current_block: Current block being accumulated Returns: Text delta if present @@ -354,6 +422,24 @@ def handle_anthropic_text_delta( return delta_text + if hasattr(event, "delta") and hasattr(event.delta, "thinking"): + delta_thinking = event.delta.thinking or "" + + if current_block is not None and current_block.get("type") == "thinking": + thinking_val = current_block.get("thinking") + current_block["thinking"] = (thinking_val or "") + delta_thinking + + return None + + if hasattr(event, "delta") and hasattr(event.delta, "signature"): + delta_signature = event.delta.signature or "" + + if current_block is not None and current_block.get("type") == "thinking": + signature_val = current_block.get("signature") + current_block["signature"] = (signature_val or "") + delta_signature + + return None + return None diff --git a/posthog/ai/claude_agent_sdk/client.py b/posthog/ai/claude_agent_sdk/client.py index 689fe3db..f9a718d8 100644 --- a/posthog/ai/claude_agent_sdk/client.py +++ b/posthog/ai/claude_agent_sdk/client.py @@ -23,6 +23,10 @@ "Please install the Claude Agent SDK to use this feature: 'pip install claude-agent-sdk'" ) +from posthog.ai.claude_agent_sdk.formatting import ( + format_assistant_blocks, + format_tool_result_content, +) from posthog.ai.claude_agent_sdk.processor import ( PostHogClaudeAgentProcessor, _GenerationTracker, @@ -159,7 +163,6 @@ async def receive_response(self): self._tracker.current_span_id or self._current_generation_span_id ) - output_content: List[Dict[str, Any]] = [] for block in message.content: if isinstance(block, ToolUseBlock): self._processor._emit_tool_span( @@ -171,17 +174,7 @@ async def receive_response(self): self._privacy, self._groups, ) - output_content.append( - { - "type": "function", - "function": { - "name": block.name, - "arguments": block.input, - }, - } - ) - elif hasattr(block, "text"): - output_content.append({"type": "text", "text": block.text}) + output_content = format_assistant_blocks(message.content) if output_content: self._pending_output = [ {"role": "assistant", "content": output_content} @@ -199,9 +192,9 @@ async def receive_response(self): { "type": "tool_result", "tool_use_id": block.tool_use_id, - "content": str(block.content)[:500] - if block.content - else None, + "content": format_tool_result_content( + block, self._processor._client + ), } ) elif hasattr(block, "text"): diff --git a/posthog/ai/claude_agent_sdk/formatting.py b/posthog/ai/claude_agent_sdk/formatting.py new file mode 100644 index 00000000..eea13c26 --- /dev/null +++ b/posthog/ai/claude_agent_sdk/formatting.py @@ -0,0 +1,55 @@ +from typing import Any, Dict, List + +try: + from claude_agent_sdk import ToolUseBlock +except ImportError: + raise ModuleNotFoundError( + "Please install the Claude Agent SDK to use this feature: 'pip install claude-agent-sdk'" + ) + +from posthog.ai.media import to_plain +from posthog.ai.sanitization import redact_media + + +def format_tool_result_content(block: Any, ph_client: Any = None) -> Any: + """Structured, redacted tool-result content (replaces str(block.content)[:500]). + + Calls redact_media directly (not finalize_ai_content) for the + max_string_len truncation, which finalize_ai_content doesn't support. + The processor.py/client.py emit sites re-run finalize_ai_content on the + already-redacted result when building $ai_input; that's a no-op since + placeholders don't re-match. + """ + content = block.content + if isinstance(content, list): + content = [to_plain(c) for c in content] + return redact_media(content, max_string_len=5000, ph_client=ph_client) + + +def format_assistant_blocks(blocks: Any) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + for block in blocks: + if getattr(block, "thinking", None) is not None: + thinking_block: Dict[str, Any] = { + "type": "thinking", + "thinking": block.thinking, + } + signature = getattr(block, "signature", None) + if signature is not None: + thinking_block["signature"] = signature + out.append(thinking_block) + elif isinstance(block, ToolUseBlock): + out.append( + { + "type": "function", + "function": { + "name": block.name, + "arguments": block.input, + }, + } + ) + elif getattr(block, "text", None) is not None: + out.append({"type": "text", "text": block.text}) + else: + out.append(to_plain(block)) + return out diff --git a/posthog/ai/claude_agent_sdk/processor.py b/posthog/ai/claude_agent_sdk/processor.py index f2375d5f..28217f9f 100644 --- a/posthog/ai/claude_agent_sdk/processor.py +++ b/posthog/ai/claude_agent_sdk/processor.py @@ -25,7 +25,12 @@ ) from posthog import setup -from posthog.ai.utils import _capture_ai_event +from posthog.ai.claude_agent_sdk.formatting import ( + format_assistant_blocks, + format_tool_result_content, +) +from posthog.ai.media import ensure_serializable as _ensure_serializable +from posthog.ai.utils import _capture_ai_event, finalize_ai_content from posthog.client import Client log = logging.getLogger("posthog") @@ -322,8 +327,6 @@ async def query( # would be stale (from the previous turn). tracker.current_span_id gives us # the correct in-progress generation. parent_id = tracker.current_span_id or current_generation_span_id - # Build output content from assistant blocks - output_content: List[Dict[str, Any]] = [] for block in message.content: if isinstance(block, ToolUseBlock): self._emit_tool_span( @@ -335,17 +338,7 @@ async def query( privacy, groups, ) - output_content.append( - { - "type": "function", - "function": { - "name": block.name, - "arguments": block.input, - }, - } - ) - elif hasattr(block, "text"): - output_content.append({"type": "text", "text": block.text}) + output_content = format_assistant_blocks(message.content) if output_content: pending_output = [ {"role": "assistant", "content": output_content} @@ -365,9 +358,9 @@ async def query( { "type": "tool_result", "tool_use_id": block.tool_use_id, - "content": str(block.content)[:500] - if block.content - else None, + "content": format_tool_result_content( + block, self._client + ), } ) elif hasattr(block, "text"): @@ -439,11 +432,19 @@ def _emit_generation( if input_messages is not None: properties["$ai_input"] = ( - None if privacy else self._with_privacy_mode(input_messages) + None + if privacy + else self._with_privacy_mode( + finalize_ai_content(input_messages, self._client) + ) ) if output_choices is not None: properties["$ai_output_choices"] = ( - None if privacy else self._with_privacy_mode(output_choices) + None + if privacy + else self._with_privacy_mode( + finalize_ai_content(output_choices, self._client) + ) ) if gen.cache_read_input_tokens: @@ -498,11 +499,19 @@ def _emit_generation_from_result( if input_messages is not None: properties["$ai_input"] = ( - None if privacy else self._with_privacy_mode(input_messages) + None + if privacy + else self._with_privacy_mode( + finalize_ai_content(input_messages, self._client) + ) ) if output_choices is not None: properties["$ai_output_choices"] = ( - None if privacy else self._with_privacy_mode(output_choices) + None + if privacy + else self._with_privacy_mode( + finalize_ai_content(output_choices, self._client) + ) ) cache_read = usage.get("cache_read_input_tokens", 0) @@ -548,7 +557,9 @@ def _emit_tool_span( if not privacy and not ( hasattr(self._client, "privacy_mode") and self._client.privacy_mode ): - properties["$ai_input_state"] = _ensure_serializable(block.input) + properties["$ai_input_state"] = finalize_ai_content( + _ensure_serializable(block.input), self._client + ) if resolved_id is None: properties["$process_person_profile"] = False @@ -613,16 +624,3 @@ def _resolve_distinct_id( return str(override) # Fall back to instance default return self._get_distinct_id(result) - - -def _ensure_serializable(obj: Any) -> Any: - """Ensure an object is JSON-serializable.""" - if obj is None: - return None - try: - import json - - json.dumps(obj) - return obj - except (TypeError, ValueError): - return str(obj) diff --git a/posthog/ai/gemini/gemini.py b/posthog/ai/gemini/gemini.py index 4d533783..09c0d764 100644 --- a/posthog/ai/gemini/gemini.py +++ b/posthog/ai/gemini/gemini.py @@ -18,6 +18,7 @@ call_llm_and_track_usage, _capture_ai_event, capture_streaming_event, + finalize_ai_content, merge_usage_stats, ) from posthog.ai.gemini.gemini_converter import ( @@ -28,7 +29,6 @@ format_gemini_streaming_output, ) from posthog.ai.utils import with_privacy_mode -from posthog.ai.sanitization import sanitize_gemini from posthog.client import Client as PostHogClient @@ -321,10 +321,10 @@ def generator(): merge_usage_stats(usage_stats, chunk_usage, mode="cumulative") # Extract content from chunk (now returns content blocks) - content_block = extract_gemini_content_from_chunk(chunk) + content_blocks = extract_gemini_content_from_chunk(chunk) - if content_block is not None: - accumulated_content.append(content_block) + if content_blocks is not None: + accumulated_content.extend(content_blocks) # Extract stop reason from chunk chunk_stop_reason = extract_gemini_stop_reason_from_chunk(chunk) @@ -369,16 +369,14 @@ def _capture_streaming_event( output: Any, stop_reason: Optional[str] = None, ): - # Prepare standardized event data formatted_input = self._format_input(contents, **kwargs) - sanitized_input = sanitize_gemini(formatted_input, self._ph_client) event_data = StreamingEventData( provider="gemini", model=model, base_url=self._base_url, kwargs=kwargs, - formatted_input=sanitized_input, + formatted_input=formatted_input, formatted_output=format_gemini_streaming_output(output), usage_stats=usage_stats, latency=latency, @@ -507,7 +505,11 @@ def embed_content( event_properties = { "$ai_provider": "gemini", "$ai_model": model, - "$ai_input": with_privacy_mode(self._ph_client, privacy_mode, contents), + "$ai_input": with_privacy_mode( + self._ph_client, + privacy_mode, + finalize_ai_content(contents, self._ph_client), + ), "$ai_http_status": http_status, "$ai_input_tokens": input_tokens, "$ai_latency": latency, diff --git a/posthog/ai/gemini/gemini_async.py b/posthog/ai/gemini/gemini_async.py index b5977edf..a7497f4e 100644 --- a/posthog/ai/gemini/gemini_async.py +++ b/posthog/ai/gemini/gemini_async.py @@ -19,6 +19,7 @@ call_llm_and_track_usage_async, _capture_ai_event, capture_streaming_event, + finalize_ai_content, merge_usage_stats, ) from posthog.ai.gemini.gemini_converter import ( @@ -29,7 +30,6 @@ format_gemini_streaming_output, ) from posthog.ai.utils import with_privacy_mode -from posthog.ai.sanitization import sanitize_gemini from posthog.client import Client as PostHogClient @@ -325,10 +325,10 @@ async def async_generator(): merge_usage_stats(usage_stats, chunk_usage, mode="cumulative") # Extract content from chunk (now returns content blocks) - content_block = extract_gemini_content_from_chunk(chunk) + content_blocks = extract_gemini_content_from_chunk(chunk) - if content_block is not None: - accumulated_content.append(content_block) + if content_blocks is not None: + accumulated_content.extend(content_blocks) # Extract stop reason from chunk chunk_stop_reason = extract_gemini_stop_reason_from_chunk(chunk) @@ -373,16 +373,14 @@ def _capture_streaming_event( output: Any, stop_reason: Optional[str] = None, ): - # Prepare standardized event data formatted_input = self._format_input(contents, **kwargs) - sanitized_input = sanitize_gemini(formatted_input, self._ph_client) event_data = StreamingEventData( provider="gemini", model=model, base_url=self._base_url, kwargs=kwargs, - formatted_input=sanitized_input, + formatted_input=formatted_input, formatted_output=format_gemini_streaming_output(output), usage_stats=usage_stats, latency=latency, @@ -511,7 +509,11 @@ async def embed_content( event_properties = { "$ai_provider": "gemini", "$ai_model": model, - "$ai_input": with_privacy_mode(self._ph_client, privacy_mode, contents), + "$ai_input": with_privacy_mode( + self._ph_client, + privacy_mode, + finalize_ai_content(contents, self._ph_client), + ), "$ai_http_status": http_status, "$ai_input_tokens": input_tokens, "$ai_latency": latency, diff --git a/posthog/ai/gemini/gemini_converter.py b/posthog/ai/gemini/gemini_converter.py index efac3243..9ad711d7 100644 --- a/posthog/ai/gemini/gemini_converter.py +++ b/posthog/ai/gemini/gemini_converter.py @@ -5,8 +5,9 @@ into standardized formats for PostHog tracking. """ -from typing import Any, Dict, List, Optional, TypedDict, Union +from typing import Any, Dict, List, Optional, TypedDict, Union, cast +from posthog.ai.media import bytes_to_base64, normalize_part_keys, to_plain from posthog.ai.types import ( FormattedContentItem, FormattedMessage, @@ -14,6 +15,62 @@ ) from posthog.ai.utils import serialize_raw_usage +_MEDIA_KINDS = {"image": "image", "video": "video", "audio": "audio"} +_BARE_PART_KEYS = { + "text", + "inline_data", + "file_data", + "function_call", + "function_response", +} + + +def _kind_from_mime(mime: Optional[str]) -> str: + if isinstance(mime, str): + prefix = mime.split("/", 1)[0] + if prefix in _MEDIA_KINDS: + return _MEDIA_KINDS[prefix] + return "file" + + +def _format_media_payload(payload: Any) -> Dict[str, Any]: + payload = to_plain(payload) + if isinstance(payload, dict) and isinstance(payload.get("data"), bytes): + payload = {**payload, "data": bytes_to_base64(payload["data"])} + return payload + + +def _format_part(part: Any) -> Optional[FormattedContentItem]: + if isinstance(part, str): + return {"type": "text", "text": part} + plain = to_plain(part) + if not isinstance(plain, dict): + return {"type": "unknown", "part": str(plain)} + plain = normalize_part_keys(plain) + if "text" in plain: + return {"type": "text", "text": plain["text"]} + if "inline_data" in plain: + media = _format_media_payload(plain["inline_data"]) + return {"type": _kind_from_mime(media.get("mime_type")), "inline_data": media} + if "file_data" in plain: + media = _format_media_payload(plain["file_data"]) + return {"type": _kind_from_mime(media.get("mime_type")), "file_data": media} + if "function_call" in plain: + return { + "type": "function_call", + "function_call": to_plain(plain["function_call"]), + } + if "function_response" in plain: + return { + "type": "function_response", + "function_response": to_plain(plain["function_response"]), + } + if not plain: + return None + key = next(iter(plain)) + fallback: Dict[str, Any] = {"type": key, key: to_plain(plain[key])} + return cast(FormattedContentItem, fallback) + class GeminiPart(TypedDict, total=False): """Represents a part in a Gemini message.""" @@ -43,63 +100,12 @@ def _format_parts_as_content_blocks(parts: List[Any]) -> List[FormattedContentIt Returns: List of formatted content blocks """ - content_blocks: List[FormattedContentItem] = [] - + blocks: List[FormattedContentItem] = [] for part in parts: - # Handle dict with text field - if isinstance(part, dict) and "text" in part: - content_blocks.append({"type": "text", "text": part["text"]}) - - # Handle string parts - elif isinstance(part, str): - content_blocks.append({"type": "text", "text": part}) - - # Handle dict with inline_data (images, documents, etc.) - elif isinstance(part, dict) and "inline_data" in part: - inline_data = part["inline_data"] - mime_type = inline_data.get("mime_type", "") - content_type = "image" if mime_type.startswith("image/") else "document" - - content_blocks.append( - { - "type": content_type, - "inline_data": inline_data, - } - ) - - # Handle object with text attribute - elif hasattr(part, "text"): - text_value = getattr(part, "text", "") - if text_value: - content_blocks.append({"type": "text", "text": text_value}) - - # Handle object with inline_data attribute - elif hasattr(part, "inline_data"): - inline_data = part.inline_data - # Convert to dict if needed - if hasattr(inline_data, "mime_type") and hasattr(inline_data, "data"): - # Determine type based on mime_type - mime_type = inline_data.mime_type - content_type = "image" if mime_type.startswith("image/") else "document" - - content_blocks.append( - { - "type": content_type, - "inline_data": { - "mime_type": mime_type, - "data": inline_data.data, - }, - } - ) - else: - content_blocks.append( - { - "type": "image", - "inline_data": inline_data, - } - ) - - return content_blocks + block = _format_part(part) + if block is not None: + blocks.append(block) + return blocks def _format_dict_message(item: Dict[str, Any]) -> FormattedMessage: @@ -132,6 +138,13 @@ def _format_dict_message(item: Dict[str, Any]) -> FormattedMessage: return {"role": item.get("role", "user"), "content": content} + if "role" not in item: + plain = to_plain(item) + if isinstance(plain, dict) and _BARE_PART_KEYS.intersection( + normalize_part_keys(plain) + ): + return {"role": "user", "content": _format_parts_as_content_blocks([item])} + # Handle dict with text field if "text" in item: return {"role": item.get("role", "user"), "content": item["text"]} @@ -162,6 +175,15 @@ def _format_object_message(item: Any) -> FormattedMessage: return {"role": role, "content": content_blocks} + # Handle a bare typed Part object (no parts/content/role attributes) — must + # be caught before the "text" branch below, which would otherwise treat an + # unset `.text` as empty content and drop any other part kind it carries + plain = to_plain(item) + if isinstance(plain, dict) and _BARE_PART_KEYS.intersection( + normalize_part_keys(plain) + ): + return {"role": "user", "content": _format_parts_as_content_blocks([item])} + # Handle object with text attribute if hasattr(item, "text"): role = getattr(item, "role", "user") if hasattr(item, "role") else "user" @@ -217,15 +239,16 @@ def format_gemini_response(response: Any) -> List[FormattedMessage]: if hasattr(candidate.content, "parts") and candidate.content.parts: for part in candidate.content.parts: - if hasattr(part, "text") and part.text: - content.append( - { - "type": "text", - "text": part.text, - } - ) + # Checked ahead of _format_part so loosely-specced MagicMock + # fixtures (which report a truthy .text but aren't real Part + # objects) still resolve to a text block — must stay ahead of + # the _format_part delegation below. + text = getattr(part, "text", None) + if isinstance(text, str) and text: + content.append({"type": "text", "text": text}) + continue - elif hasattr(part, "function_call") and part.function_call: + if hasattr(part, "function_call") and part.function_call: function_call = part.function_call content.append( { @@ -236,29 +259,11 @@ def format_gemini_response(response: Any) -> List[FormattedMessage]: }, } ) + continue - elif hasattr(part, "inline_data") and part.inline_data: - # Handle audio/media inline data - import base64 - - inline_data = part.inline_data - mime_type = getattr(inline_data, "mime_type", "audio/pcm") - raw_data = getattr(inline_data, "data", b"") - - # Encode binary data as base64 string for JSON serialization - if isinstance(raw_data, bytes): - data = base64.b64encode(raw_data).decode("utf-8") - else: - # Already a string (base64) - data = raw_data - - content.append( - { - "type": "audio", - "mime_type": mime_type, - "data": data, - } - ) + block = _format_part(part) + if block is not None: + content.append(block) if content: output.append( @@ -568,42 +573,49 @@ def extract_gemini_usage_from_chunk(chunk: Any) -> TokenUsage: return usage -def extract_gemini_content_from_chunk(chunk: Any) -> Optional[Dict[str, Any]]: +def extract_gemini_content_from_chunk( + chunk: Any, +) -> Optional[List[FormattedContentItem]]: """ - Extract content (text or function call) from a Gemini streaming chunk. + Extract all content blocks (text, media, function calls) from a Gemini + streaming chunk's parts, in order. Args: chunk: Streaming chunk from Gemini API Returns: - Content block dictionary if present, None otherwise + List of content block dictionaries if the chunk yields any blocks, + None otherwise """ - # Check for text content - if hasattr(chunk, "text") and chunk.text: - return {"type": "text", "text": chunk.text} + blocks: List[FormattedContentItem] = [] - # Check for function calls in candidates if hasattr(chunk, "candidates") and chunk.candidates: for candidate in chunk.candidates: if hasattr(candidate, "content") and candidate.content: if hasattr(candidate.content, "parts") and candidate.content.parts: for part in candidate.content.parts: - # Check for function_call part if hasattr(part, "function_call") and part.function_call: function_call = part.function_call - return { - "type": "function", - "function": { - "name": function_call.name, - "arguments": function_call.args, - }, - } - # Also check for text in parts - elif hasattr(part, "text") and part.text: - return {"type": "text", "text": part.text} + blocks.append( + { + "type": "function", + "function": { + "name": function_call.name, + "arguments": function_call.args, + }, + } + ) + continue - return None + block = _format_part(part) + if block is not None: + blocks.append(block) + + if not blocks and hasattr(chunk, "text") and chunk.text: + blocks.append({"type": "text", "text": chunk.text}) + + return blocks or None def format_gemini_streaming_output( @@ -659,6 +671,17 @@ def format_gemini_streaming_output( "function": item.get("function", {}), } ) + else: + if text_parts: + content.append( + { + "type": "text", + "text": "".join(text_parts), + } + ) + text_parts = [] + + content.append(item) # Add any remaining text if text_parts: diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 2c99d29b..f3564298 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -43,8 +43,12 @@ from posthog import setup from posthog.ai.gateway import warn_if_posthog_ai_gateway -from posthog.ai.sanitization import sanitize_langchain -from posthog.ai.utils import _capture_ai_event, get_model_params, with_privacy_mode +from posthog.ai.utils import ( + _capture_ai_event, + finalize_ai_content, + get_model_params, + with_privacy_mode, +) from posthog.client import Client log = logging.getLogger("posthog") @@ -538,7 +542,7 @@ def _capture_trace_or_span( "$ai_input_state": with_privacy_mode( self._ph_client, self._privacy_mode, - sanitize_langchain(run.input, self._ph_client), + finalize_ai_content(run.input, self._ph_client), ), "$ai_latency": run.latency, "$ai_span_name": run.name, @@ -563,7 +567,9 @@ def _capture_trace_or_span( elif outputs is not None: event_properties["$ai_output_state"] = with_privacy_mode( - self._ph_client, self._privacy_mode, outputs + self._ph_client, + self._privacy_mode, + finalize_ai_content(outputs, self._ph_client), ) if self._distinct_id is None: @@ -620,7 +626,7 @@ def _capture_generation( "$ai_input": with_privacy_mode( self._ph_client, self._privacy_mode, - sanitize_langchain(run.input, self._ph_client), + finalize_ai_content(run.input, self._ph_client), ), "$ai_http_status": 200, "$ai_latency": run.latency, @@ -678,7 +684,9 @@ def _capture_generation( for generation in generation_result ] event_properties["$ai_output_choices"] = with_privacy_mode( - self._ph_client, self._privacy_mode, completions + self._ph_client, + self._privacy_mode, + finalize_ai_content(completions, self._ph_client), ) # Extract stop reason from generation info diff --git a/posthog/ai/media.py b/posthog/ai/media.py new file mode 100644 index 00000000..6c4b4f00 --- /dev/null +++ b/posthog/ai/media.py @@ -0,0 +1,86 @@ +import base64 +import dataclasses +import json +from typing import Any + +_CAMEL_TO_SNAKE = { + "inlineData": "inline_data", + "fileData": "file_data", + "mimeType": "mime_type", + "fileUri": "file_uri", + "functionCall": "function_call", + "functionResponse": "function_response", + "videoMetadata": "video_metadata", +} + + +def to_plain(obj: Any) -> Any: + if isinstance(obj, dict): + return {k: v for k, v in obj.items() if v is not None} + model_dump = getattr(obj, "model_dump", None) + if callable(model_dump): + try: + return model_dump(exclude_none=True) + except TypeError: + return {k: v for k, v in model_dump().items() if v is not None} + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + return {k: v for k, v in dataclasses.asdict(obj).items() if v is not None} + return obj + + +def ensure_serializable(obj: Any) -> Any: + """Ensure an object is JSON-serializable, converting to str as fallback. + + Recurses into dicts/lists/tuples so one non-serializable leaf doesn't + collapse the whole structure to a string. Bytes pass through untouched - + finalize_ai_content is responsible for redacting or base64-encoding them. + Non-string dict keys are coerced to str so json.dumps can't TypeError at + send time on tuple/object keys. Guards against reference cycles the same + way redact_media does, so a self-referencing structure can't blow the + stack. + """ + stack: set = set() + + def walk(node: Any) -> Any: + if node is None or isinstance(node, bytes): + return node + if isinstance(node, dict): + if id(node) in stack: + return "" + stack.add(id(node)) + try: + return { + (k if isinstance(k, str) else str(k)): walk(v) + for k, v in node.items() + } + finally: + stack.discard(id(node)) + if isinstance(node, (list, tuple)): + if id(node) in stack: + return "" + stack.add(id(node)) + try: + return [walk(v) for v in node] + finally: + stack.discard(id(node)) + try: + json.dumps(node) + return node + except (TypeError, ValueError): + return str(node) + + return walk(obj) + + +def bytes_to_base64(data: bytes) -> str: + return base64.b64encode(data).decode("utf-8") + + +def normalize_part_keys(d: dict) -> dict: + out = {} + for key, value in d.items(): + snake = _CAMEL_TO_SNAKE.get(key, key) + if isinstance(value, dict): + value = {_CAMEL_TO_SNAKE.get(k, k): v for k, v in value.items()} + out[snake] = value + return out diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index 719c878d..56f46de8 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -15,6 +15,7 @@ call_llm_and_track_usage, _capture_ai_event, extract_available_tool_calls, + finalize_ai_content, merge_usage_stats, with_privacy_mode, ) @@ -24,7 +25,6 @@ extract_openai_tool_calls_from_chunk, accumulate_openai_tool_calls, ) -from posthog.ai.sanitization import sanitize_openai, sanitize_openai_response from posthog.client import Client as PostHogClient from posthog import setup from posthog.ai.openai.wrapper_utils import _OpenAIWrapperResource @@ -155,7 +155,7 @@ def _create_streaming( ): start_time = time.time() usage_stats: TokenUsage = TokenUsage() - final_content = [] + final_content: List[Any] = [] model_from_response: Optional[str] = None stop_reason: Optional[str] = None response = self._original.create(**kwargs) @@ -181,11 +181,10 @@ def generator(): if chunk_usage: merge_usage_stats(usage_stats, chunk_usage) - # Extract content from chunk content = extract_openai_content_from_chunk(chunk, "responses") if content is not None: - final_content.append(content) + final_content.extend(content) # Capture stop reason from response.completed event if ( @@ -243,11 +242,7 @@ def _capture_streaming_event( ) from posthog.ai.utils import capture_streaming_event - # Prepare standardized event data formatted_input = format_openai_streaming_input(kwargs, "responses") - sanitized_input = sanitize_openai_response( - formatted_input, self._client._ph_client - ) # Use model from kwargs, fallback to model from response model = kwargs.get("model") or model_from_response or "unknown" @@ -257,7 +252,7 @@ def _capture_streaming_event( model=model, base_url=str(self._client.base_url), kwargs=kwargs, - formatted_input=sanitized_input, + formatted_input=formatted_input, formatted_output=format_openai_streaming_output(output, "responses"), usage_stats=usage_stats, latency=latency, @@ -411,7 +406,7 @@ def _create_streaming( ): start_time = time.time() usage_stats: TokenUsage = TokenUsage() - accumulated_content = [] + accumulated_content: List[Any] = [] accumulated_tool_calls: Dict[int, Dict[str, Any]] = {} model_from_response: Optional[str] = None stop_reason: Optional[str] = None @@ -514,9 +509,7 @@ def _capture_streaming_event( ) from posthog.ai.utils import capture_streaming_event - # Prepare standardized event data formatted_input = format_openai_streaming_input(kwargs, "chat") - sanitized_input = sanitize_openai(formatted_input, self._client._ph_client) # Use model from kwargs, fallback to model from response model = kwargs.get("model") or model_from_response or "unknown" @@ -526,7 +519,7 @@ def _capture_streaming_event( model=model, base_url=str(self._client.base_url), kwargs=kwargs, - formatted_input=sanitized_input, + formatted_input=formatted_input, formatted_output=format_openai_streaming_output(output, "chat", tool_calls), usage_stats=usage_stats, latency=latency, @@ -593,7 +586,7 @@ def create( "$ai_input": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - sanitize_openai_response(kwargs.get("input"), self._client._ph_client), + finalize_ai_content(kwargs.get("input"), self._client._ph_client), ), "$ai_http_status": 200, "$ai_input_tokens": usage_stats.get("prompt_tokens", 0), diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 09833740..6ce8cc80 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -17,6 +17,7 @@ call_llm_and_track_usage_async, _capture_ai_event, extract_available_tool_calls, + finalize_ai_content, get_model_params, merge_usage_stats, with_privacy_mode, @@ -26,9 +27,9 @@ extract_openai_content_from_chunk, extract_openai_tool_calls_from_chunk, accumulate_openai_tool_calls, + format_openai_streaming_input, format_openai_streaming_output, ) -from posthog.ai.sanitization import sanitize_openai, sanitize_openai_response from posthog.client import Client as PostHogClient from posthog.ai.openai.wrapper_utils import _OpenAIWrapperResource @@ -158,7 +159,7 @@ async def _create_streaming( ): start_time = time.time() usage_stats: TokenUsage = TokenUsage() - final_content = [] + final_content: List[Any] = [] model_from_response: Optional[str] = None stop_reason: Optional[str] = None response = await self._original.create(**kwargs) @@ -184,11 +185,10 @@ async def async_generator(): if chunk_usage: merge_usage_stats(usage_stats, chunk_usage) - # Extract content from chunk content = extract_openai_content_from_chunk(chunk, "responses") if content is not None: - final_content.append(content) + final_content.extend(content) # Capture stop reason from response.completed event if ( @@ -253,12 +253,18 @@ async def _capture_streaming_event( "$ai_input": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - sanitize_openai_response(kwargs.get("input"), self._client._ph_client), + finalize_ai_content( + format_openai_streaming_input(kwargs, "responses"), + self._client._ph_client, + ), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - format_openai_streaming_output(output, "responses"), + finalize_ai_content( + format_openai_streaming_output(output, "responses"), + self._client._ph_client, + ), ), "$ai_http_status": 200, "$ai_input_tokens": usage_stats.get("input_tokens", 0), @@ -441,7 +447,7 @@ async def _create_streaming( ): start_time = time.time() usage_stats: TokenUsage = TokenUsage() - accumulated_content = [] + accumulated_content: List[Any] = [] accumulated_tool_calls: Dict[int, Dict[str, Any]] = {} model_from_response: Optional[str] = None stop_reason: Optional[str] = None @@ -549,12 +555,18 @@ async def _capture_streaming_event( "$ai_input": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - sanitize_openai(kwargs.get("messages"), self._client._ph_client), + finalize_ai_content( + format_openai_streaming_input(kwargs, "chat"), + self._client._ph_client, + ), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - format_openai_streaming_output(output, "chat", tool_calls), + finalize_ai_content( + format_openai_streaming_output(output, "chat", tool_calls), + self._client._ph_client, + ), ), "$ai_http_status": 200, "$ai_input_tokens": usage_stats.get("input_tokens", 0), @@ -650,7 +662,7 @@ async def create( "$ai_input": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - sanitize_openai_response(kwargs.get("input"), self._client._ph_client), + finalize_ai_content(kwargs.get("input"), self._client._ph_client), ), "$ai_http_status": 200, "$ai_input_tokens": usage_stats.get("input_tokens", 0), diff --git a/posthog/ai/openai/openai_converter.py b/posthog/ai/openai/openai_converter.py index 95090723..1ce20900 100644 --- a/posthog/ai/openai/openai_converter.py +++ b/posthog/ai/openai/openai_converter.py @@ -6,8 +6,9 @@ Chat Completions API and Responses API formats. """ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, cast +from posthog.ai.media import to_plain from posthog.ai.types import ( FormattedContentItem, FormattedFunctionCall, @@ -19,6 +20,101 @@ from posthog.ai.utils import serialize_raw_usage +def _item_attr(item: Any, name: str, default: Any = None) -> Any: + if isinstance(item, dict): + return item.get(name, default) + return getattr(item, name, default) + + +def _format_responses_output_items(items: Any) -> List[FormattedContentItem]: + content: List[FormattedContentItem] = [] + + for item in items: + item_type = _item_attr(item, "type") + + if item_type == "message": + message_content = _item_attr(item, "content") + if isinstance(message_content, list): + for content_item in message_content: + content_item_type = _item_attr(content_item, "type") + content_item_text = _item_attr(content_item, "text") + content_item_refusal = _item_attr(content_item, "refusal") + + if content_item_type == "output_text" and ( + content_item_text is not None + ): + content.append({"type": "text", "text": content_item_text}) + + elif content_item_type == "refusal" and ( + content_item_refusal is not None + ): + content.append( + {"type": "refusal", "refusal": content_item_refusal} + ) + + elif content_item_text is not None: + content.append({"type": "text", "text": content_item_text}) + + elif content_item_type == "input_image" and ( + _item_attr(content_item, "image_url") is not None + ): + image_content: FormattedImageContent = { + "type": "image", + "image": _item_attr(content_item, "image_url"), + } + content.append(image_content) + + elif message_content is not None: + content.append({"type": "text", "text": str(message_content)}) + + elif item_type == "function_call": + call_id = _item_attr(item, "call_id") + if call_id is None: + call_id = _item_attr(item, "id", "") + content.append( + { + "type": "function", + "id": call_id, + "function": { + "name": _item_attr(item, "name"), + "arguments": _item_attr(item, "arguments", {}), + }, + } + ) + + elif item_type == "reasoning": + content.append(to_plain(item)) + + elif item_type == "image_generation_call": + content.append( + { + "type": "image_generation_call", + "result": _item_attr(item, "result"), + "status": _item_attr(item, "status"), + } + ) + + elif item_type is not None: + plain_item = to_plain(item) + content.append( + plain_item if isinstance(plain_item, dict) else {"type": item_type} + ) + + return content + + +def _responses_output_role(items: Any) -> str: + role = "assistant" + + for item in items: + if _item_attr(item, "type") == "message": + item_role = _item_attr(item, "role") + if item_role is not None: + role = item_role + + return role + + def format_openai_response(response: Any) -> List[FormattedMessage]: """ Format an OpenAI response into standardized message format. @@ -84,56 +180,8 @@ def format_openai_response(response: Any) -> List[FormattedMessage]: # Handle Responses API format if hasattr(response, "output"): - content = [] - role = "assistant" - - for item in response.output: - if item.type == "message": - role = item.role - - if hasattr(item, "content") and isinstance(item.content, list): - for content_item in item.content: - if ( - hasattr(content_item, "type") - and content_item.type == "output_text" - and hasattr(content_item, "text") - ): - content.append( - { - "type": "text", - "text": content_item.text, - } - ) - - elif hasattr(content_item, "text"): - content.append({"type": "text", "text": content_item.text}) - - elif ( - hasattr(content_item, "type") - and content_item.type == "input_image" - and hasattr(content_item, "image_url") - ): - image_content: FormattedImageContent = { - "type": "image", - "image": content_item.image_url, - } - content.append(image_content) - - elif hasattr(item, "content"): - text_content = {"type": "text", "text": str(item.content)} - content.append(text_content) - - elif hasattr(item, "type") and item.type == "function_call": - content.append( - { - "type": "function", - "id": getattr(item, "call_id", getattr(item, "id", "")), - "function": { - "name": item.name, - "arguments": getattr(item, "arguments", {}), - }, - } - ) + content = _format_responses_output_items(response.output) + role = _responses_output_role(response.output) if content: output.append( @@ -164,20 +212,41 @@ def format_openai_input( formatted_messages: List[FormattedMessage] = [] - # Handle Chat Completions API format if messages is not None: for msg in messages: - formatted_messages.append( - { - "role": msg.get("role", "user"), - "content": msg.get("content", ""), - } - ) + plain = to_plain(msg) + if not isinstance(plain, dict): + plain = {"role": "user", "content": str(plain)} + + formatted: Dict[str, Any] = { + "role": plain.get("role", "user"), + "content": plain.get("content"), + } + + for key in ("tool_calls", "tool_call_id", "name", "audio", "refusal"): + if plain.get(key) is not None: + formatted[key] = ( + to_plain(plain[key]) if key == "audio" else plain[key] + ) + + formatted_messages.append(cast(FormattedMessage, formatted)) # Handle Responses API format if input_data is not None: if isinstance(input_data, list): for item in input_data: + if not isinstance(item, (dict, str)): + item = to_plain(item) + + if ( + isinstance(item, dict) + and "type" in item + and "role" not in item + and "content" not in item + ): + formatted_messages.append(cast(FormattedMessage, to_plain(item))) + continue + role = "user" content = "" @@ -551,7 +620,7 @@ def extract_openai_usage_from_chunk( def extract_openai_content_from_chunk( chunk: Any, provider_type: str = "chat" -) -> Optional[str]: +) -> Optional[Any]: """ Extract content from an OpenAI streaming chunk. @@ -562,7 +631,9 @@ def extract_openai_content_from_chunk( provider_type: Either "chat" or "responses" to handle different API formats Returns: - Text content if present, None otherwise + For "chat": text content (str), or an audio/refusal delta block (dict), + if present. For "responses": the full `response.output` list on the + `response.completed` event. None otherwise. """ if provider_type == "chat": @@ -572,18 +643,30 @@ def extract_openai_content_from_chunk( and chunk.choices and len(chunk.choices) > 0 and chunk.choices[0].delta - and chunk.choices[0].delta.content ): - return chunk.choices[0].delta.content + delta = chunk.choices[0].delta + + if delta.content: + return delta.content + + audio_delta = getattr(delta, "audio", None) + if audio_delta is not None: + plain_audio = to_plain(audio_delta) + if isinstance(plain_audio, dict): + return {"type": "audio", **plain_audio} + return {"type": "audio"} + + refusal_delta = getattr(delta, "refusal", None) + if refusal_delta: + return {"type": "refusal", "refusal": refusal_delta} elif provider_type == "responses": # Responses API format if hasattr(chunk, "type") and chunk.type == "response.completed": if hasattr(chunk, "response") and chunk.response: res = chunk.response - if res.output and len(res.output) > 0: - # Return the full output for responses - return res.output[0] + if res.output: + return res.output return None @@ -708,10 +791,44 @@ def format_openai_streaming_output( if isinstance(accumulated_content, str) and accumulated_content: content_items.append({"type": "text", "text": accumulated_content}) elif isinstance(accumulated_content, list): - # If it's a list of strings, join them - text = "".join(str(item) for item in accumulated_content if item) - if text: - content_items.append({"type": "text", "text": text}) + text_parts: List[str] = [] + audio_id: Optional[str] = None + audio_data_parts: List[str] = [] + audio_transcript_parts: List[str] = [] + refusal_parts: List[str] = [] + + for item in accumulated_content: + if isinstance(item, str): + if item: + text_parts.append(item) + elif isinstance(item, dict) and item.get("type") == "audio": + if audio_id is None and item.get("id"): + audio_id = item["id"] + if item.get("data"): + audio_data_parts.append(item["data"]) + if item.get("transcript"): + audio_transcript_parts.append(item["transcript"]) + elif isinstance(item, dict) and item.get("type") == "refusal": + if item.get("refusal"): + refusal_parts.append(item["refusal"]) + + if text_parts: + content_items.append({"type": "text", "text": "".join(text_parts)}) + + if audio_data_parts or audio_transcript_parts: + audio_block: Dict[str, Any] = {"type": "audio"} + if audio_id is not None: + audio_block["id"] = audio_id + if audio_data_parts: + audio_block["data"] = "".join(audio_data_parts) + if audio_transcript_parts: + audio_block["transcript"] = "".join(audio_transcript_parts) + content_items.append(audio_block) + + if refusal_parts: + content_items.append( + {"type": "refusal", "refusal": "".join(refusal_parts)} + ) # Add tool calls if present if tool_calls: @@ -732,10 +849,22 @@ def format_openai_streaming_output( return [{"role": "assistant", "content": []}] elif provider_type == "responses": - # Responses API: accumulated_content is a list of output items + if isinstance(accumulated_content, list) and not accumulated_content: + return [] if isinstance(accumulated_content, list) and accumulated_content: - # The output is already formatted, just return it - return accumulated_content + items: List[Any] = [] + for entry in accumulated_content: + if isinstance(entry, list): + items.extend(entry) + else: + items.append(entry) + + content_items = _format_responses_output_items(items) + role = _responses_output_role(items) + + if content_items: + return [{"role": role, "content": content_items}] + return [] elif isinstance(accumulated_content, str): return [ { diff --git a/posthog/ai/openai_agents/processor.py b/posthog/ai/openai_agents/processor.py index 0939aefd..92834f0b 100644 --- a/posthog/ai/openai_agents/processor.py +++ b/posthog/ai/openai_agents/processor.py @@ -1,4 +1,3 @@ -import json import logging import time from datetime import datetime @@ -21,28 +20,14 @@ ) from posthog import setup -from posthog.ai.utils import _capture_ai_event +from posthog.ai.media import ensure_serializable as _ensure_serializable +from posthog.ai.sanitization import _multimodal_capture_enabled, _placeholder +from posthog.ai.utils import _capture_ai_event, finalize_ai_content from posthog.client import Client log = logging.getLogger("posthog") -def _ensure_serializable(obj: Any) -> Any: - """Ensure an object is JSON-serializable, converting to str as fallback. - - Returns the original object if it's already serializable (dict, list, str, - int, etc.), or str(obj) for non-serializable types so that downstream - json.dumps() calls won't fail. - """ - if obj is None: - return None - try: - json.dumps(obj) - return obj - except (TypeError, ValueError): - return str(obj) - - def _parse_iso_timestamp(iso_str: Optional[str]) -> Optional[float]: """Parse ISO timestamp to Unix timestamp.""" if not iso_str: @@ -532,9 +517,13 @@ def _handle_generation_span( ), "$ai_model": span_data.model, "$ai_model_parameters": model_params if model_params else None, - "$ai_input": self._with_privacy_mode(_ensure_serializable(span_data.input)), + "$ai_input": self._with_privacy_mode( + finalize_ai_content(_ensure_serializable(span_data.input), self._client) + ), "$ai_output_choices": self._with_privacy_mode( - _ensure_serializable(span_data.output) + finalize_ai_content( + _ensure_serializable(span_data.output), self._client + ) ), "$ai_input_tokens": input_tokens, "$ai_output_tokens": output_tokens, @@ -579,10 +568,12 @@ def _handle_function_span( "$ai_span_name": span_data.name, "$ai_span_type": "tool", "$ai_input_state": self._with_privacy_mode( - _ensure_serializable(span_data.input) + finalize_ai_content(_ensure_serializable(span_data.input), self._client) ), "$ai_output_state": self._with_privacy_mode( - _ensure_serializable(span_data.output) + finalize_ai_content( + _ensure_serializable(span_data.output), self._client + ) ), } @@ -700,7 +691,9 @@ def _handle_response_span( ), "$ai_model": model, "$ai_response_id": response_id, - "$ai_input": self._with_privacy_mode(_ensure_serializable(span_data.input)), + "$ai_input": self._with_privacy_mode( + finalize_ai_content(_ensure_serializable(span_data.input), self._client) + ), "$ai_input_tokens": input_tokens, "$ai_output_tokens": output_tokens, "$ai_total_tokens": input_tokens + output_tokens, @@ -714,7 +707,9 @@ def _handle_response_span( output_items = getattr(response, "output", None) if output_items: properties["$ai_output_choices"] = self._with_privacy_mode( - _ensure_serializable(output_items) + finalize_ai_content( + _ensure_serializable(output_items), self._client + ) ) # Extract stop reason (status) from response @@ -789,18 +784,38 @@ def _handle_audio_span( if hasattr(span_data, "output_format"): properties["audio_output_format"] = span_data.output_format - # Add text input for TTS + # Capture the input. For speech (TTS) spans the input is the text to + # synthesize. For transcription spans the input is the base64-encoded + # audio — a bare string with no key/parent context the structural + # redactor can't recognize, so redact it here (or pass it through in + # multimodal mode) rather than leaking raw base64 into $ai_input. if ( hasattr(span_data, "input") and span_data.input and isinstance(span_data.input, str) ): - properties["$ai_input"] = self._with_privacy_mode(span_data.input) + if span_type == "transcription": + audio_input: Any = ( + span_data.input + if _multimodal_capture_enabled(self._client) + else _placeholder( + getattr(span_data, "input_format", None) or "audio" + ) + ) + properties["$ai_input"] = self._with_privacy_mode(audio_input) + else: + properties["$ai_input"] = self._with_privacy_mode( + finalize_ai_content( + _ensure_serializable(span_data.input), self._client + ) + ) - # Don't include audio data (base64) - just metadata if hasattr(span_data, "output") and isinstance(span_data.output, str): - # For transcription, output is the text - properties["$ai_output_state"] = self._with_privacy_mode(span_data.output) + properties["$ai_output_state"] = self._with_privacy_mode( + finalize_ai_content( + _ensure_serializable(span_data.output), self._client + ) + ) self._capture_event("$ai_span", properties, distinct_id) diff --git a/posthog/ai/sanitization.py b/posthog/ai/sanitization.py index 2c7722ba..8989c61a 100644 --- a/posthog/ai/sanitization.py +++ b/posthog/ai/sanitization.py @@ -1,9 +1,36 @@ import re -from typing import Any -from urllib.parse import urlparse +from typing import Any, Optional + +from posthog.ai.media import bytes_to_base64 REDACTED_IMAGE_PLACEHOLDER = "[base64 image redacted]" +_DATA_URL_RE = re.compile(r"^data:([^;,]+);base64,") +_BASE64_BODY_RE = re.compile(r"^[A-Za-z0-9+/_-]+={0,2}$") +_STRONG_CONTEXT_MIN_LEN = 200 + +_STRONG_KEYS = { + "data", + "image_url", + "imageUrl", + "video_url", + "videoUrl", + "audio", + "audio_data", + "audioData", + "inline_data", + "inlineData", + "file_data", + "fileData", +} +_MIME_HINT_KEYS = {"mime_type", "mimeType", "media_type", "mediaType", "format"} + +# Container keys whose nested {"url": ...} value carries base64 media in the +# Chat Completions / LangChain dict shape ({"image_url": {"url": }}). The +# inner "url" key is weak on its own, so it's only strong when its parent dict +# sits under one of these. +_MEDIA_URL_CONTAINER_KEYS = {"image_url", "imageUrl", "video_url", "videoUrl"} + def _multimodal_capture_enabled(ph_client: Any = None) -> bool: """Media passthrough: on only when the client opted into multimodal capture.""" @@ -12,237 +39,172 @@ def _multimodal_capture_enabled(ph_client: Any = None) -> bool: ) # is True: tolerate unspecced Mock clients whose auto-generated attrs are truthy -def is_base64_data_url(text: str) -> bool: - return re.match(r"^data:([^;]+);base64,", text) is not None - - -def is_valid_url(text: str) -> bool: - try: - result = urlparse(text) - return bool(result.scheme and result.netloc) - except Exception: - pass - - return text.startswith(("/", "./", "../")) - - -def is_raw_base64(text: str) -> bool: - if is_valid_url(text): - return False - - return len(text) > 20 and re.match(r"^[A-Za-z0-9+/]+=*$", text) is not None - - -def redact_base64_data_url(value: Any) -> Any: - if not isinstance(value, str): - return value - - if is_base64_data_url(value): - return REDACTED_IMAGE_PLACEHOLDER - - if is_raw_base64(value): - return REDACTED_IMAGE_PLACEHOLDER - - return value - - -def process_messages(messages: Any, transform_content_func) -> Any: - if not messages: - return messages - - def process_content(content: Any) -> Any: - if isinstance(content, str): - return content +_AUDIO_FORMATS = {"wav", "mp3", "pcm16", "pcm", "flac", "opus", "aac", "ogg", "m4a"} +_VIDEO_FORMATS = {"mp4", "mov", "webm", "avi", "mkv"} +_IMAGE_FORMATS = {"png", "jpg", "jpeg", "gif", "webp", "bmp", "svg"} - if not content: - return content - if isinstance(content, list): - return [transform_content_func(item) for item in content] +_BARE_MEDIA_WORDS = {"image", "video", "audio"} - return transform_content_func(content) - def process_message(msg: Any) -> Any: - if not isinstance(msg, dict) or "content" not in msg: - return msg - return {**msg, "content": process_content(msg["content"])} +def _media_word(mime: Optional[str]) -> str: + if not mime: + return "image" + if "/" in mime: + for prefix in ("image", "video", "audio"): + if mime.startswith(prefix + "/"): + return prefix + return "file" + lowered = mime.lower() + if lowered in _BARE_MEDIA_WORDS: + return lowered + if lowered in _AUDIO_FORMATS: + return "audio" + if lowered in _VIDEO_FORMATS: + return "video" + if lowered in _IMAGE_FORMATS: + return "image" + return "file" - if isinstance(messages, list): - return [process_message(msg) for msg in messages] - - return process_message(messages) - - -def sanitize_openai_image(item: Any) -> Any: - if not isinstance(item, dict): - return item - - if item.get("type") == "input_image" and isinstance(item.get("image_url"), str): - return { - **item, - "image_url": redact_base64_data_url(item["image_url"]), - } - - if ( - item.get("type") == "image_url" - and isinstance(item.get("image_url"), dict) - and "url" in item["image_url"] - ): - return { - **item, - "image_url": { - **item["image_url"], - "url": redact_base64_data_url(item["image_url"]["url"]), - }, - } - if item.get("type") == "audio" and "data" in item: - return {**item, "data": REDACTED_IMAGE_PLACEHOLDER} +def _placeholder(mime: Optional[str]) -> str: + return f"[base64 {_media_word(mime)} redacted]" - return item +def _sibling_mime(parent: Optional[dict]) -> Optional[str]: + if not isinstance(parent, dict): + return None + for key in _MIME_HINT_KEYS: + value = parent.get(key) + if isinstance(value, str): + return value + return None -def sanitize_openai_response_image(item: Any) -> Any: - if not isinstance(item, dict): - return item - if item.get("type") == "input_image" and "image_url" in item: - return { - **item, - "image_url": redact_base64_data_url(item["image_url"]), - } +def _word_hint(parent: Optional[dict]) -> Optional[str]: + mime = _sibling_mime(parent) + if mime is not None: + return mime + if isinstance(parent, dict) and parent.get("type") in _BARE_MEDIA_WORDS: + return parent["type"] + return None - return item +# Sibling-type hints: a key is strong context when its parent dict has one of +# these "type" values, without making the key itself universally strong (a +# bare "result" also shows up on tool outputs, where it must stay untouched). +_SIBLING_TYPE_STRONG_KEYS = { + "result": {"image_generation_call"}, +} -def sanitize_anthropic_image(item: Any) -> Any: - if not isinstance(item, dict): - return item - - if ( - item.get("type") == "image" - and isinstance(item.get("source"), dict) - and item["source"].get("type") == "base64" - and "data" in item["source"] - ): - return { - **item, - "source": { - **item["source"], - "data": REDACTED_IMAGE_PLACEHOLDER, - }, - } - - return item - - -def sanitize_gemini_part(part: Any) -> Any: - if not isinstance(part, dict): - return part - - if ( - "inline_data" in part - and isinstance(part["inline_data"], dict) - and "data" in part["inline_data"] - ): - return { - **part, - "inline_data": { - **part["inline_data"], - "data": REDACTED_IMAGE_PLACEHOLDER, - }, - } - - return part - - -def process_gemini_item(item: Any) -> Any: - if not isinstance(item, dict): - return item - - if "parts" in item and item["parts"]: - parts = item["parts"] - if isinstance(parts, list): - parts = [sanitize_gemini_part(part) for part in parts] - else: - parts = sanitize_gemini_part(parts) - - return {**item, "parts": parts} - - return item - - -def sanitize_langchain_image(item: Any) -> Any: - if not isinstance(item, dict): - return item +def _has_strong_sibling_type(key: Optional[str], parent: Optional[dict]) -> bool: + if not isinstance(parent, dict): + return False + sibling_types = _SIBLING_TYPE_STRONG_KEYS.get(key or "") + return sibling_types is not None and parent.get("type") in sibling_types + + +def _redact_string( + value: str, + key: Optional[str], + parent: Optional[dict], + parent_key: Optional[str], +) -> str: + match = _DATA_URL_RE.match(value) + if match: + return _placeholder(match.group(1)) + strong = ( + (key in _STRONG_KEYS) + or _has_strong_sibling_type(key, parent) + or (key == "url" and parent_key in _MEDIA_URL_CONTAINER_KEYS) + ) if ( - item.get("type") == "image_url" - and isinstance(item.get("image_url"), dict) - and "url" in item["image_url"] + strong + and len(value) >= _STRONG_CONTEXT_MIN_LEN + and _BASE64_BODY_RE.match(value) ): - return { - **item, - "image_url": { - **item["image_url"], - "url": redact_base64_data_url(item["image_url"]["url"]), - }, - } + return _placeholder(_word_hint(parent)) + return value - if item.get("type") == "image" and "data" in item: - return {**item, "data": redact_base64_data_url(item["data"])} - if ( - item.get("type") == "image" - and isinstance(item.get("source"), dict) - and "data" in item["source"] - ): - return { - **item, - "source": { - **item["source"], - "data": REDACTED_IMAGE_PLACEHOLDER, - }, - } +def redact_media( + value: Any, max_string_len: Optional[int] = None, ph_client: Any = None +) -> Any: + passthrough = _multimodal_capture_enabled(ph_client) + stack: set = set() + + def walk( + node: Any, + key: Optional[str], + parent: Optional[dict], + parent_key: Optional[str], + ) -> Any: + if isinstance(node, bytes): + if passthrough: + return bytes_to_base64(node) + return _placeholder(_word_hint(parent)) + if isinstance(node, str): + # Passthrough keeps media intact, so it must also skip truncation: + # a 5000-char cut through base64 corrupts it as surely as redaction. + if passthrough: + return node + out = _redact_string(node, key, parent, parent_key) + if max_string_len is not None and len(out) > max_string_len: + out = out[:max_string_len] + "... [truncated]" + return out + if isinstance(node, dict): + if id(node) in stack: + return None + stack.add(id(node)) + try: + return {k: walk(v, k, node, key) for k, v in node.items()} + finally: + stack.discard(id(node)) + if isinstance(node, (list, tuple)): + if id(node) in stack: + return None + stack.add(id(node)) + try: + return [walk(item, key, parent, parent_key) for item in node] + finally: + stack.discard(id(node)) + return node + + return walk(value, None, None, None) + + +def sanitize_messages( + data: Any, provider: Optional[str] = None, ph_client: Any = None +) -> Any: + """Back-compat entry point; provider is ignored — redaction is structural now.""" + return redact_media(data, ph_client=ph_client) - if item.get("type") == "media" and "data" in item: - return {**item, "data": redact_base64_data_url(item["data"])} - return item +def redact_base64_data_url(value: Any) -> Any: + if not isinstance(value, str): + return value + return _redact_string(value, "data", None, None) +# Back-compat wrappers for the old per-provider sanitizers — still imported by +# wrappers and langchain/callbacks.py; kept in the public API snapshot. They +# stay 2-arg-tolerant so callers threading the client keep working. def sanitize_openai(data: Any, ph_client: Any = None) -> Any: - if _multimodal_capture_enabled(ph_client): - return data - return process_messages(data, sanitize_openai_image) + return sanitize_messages(data, ph_client=ph_client) def sanitize_openai_response(data: Any, ph_client: Any = None) -> Any: - if _multimodal_capture_enabled(ph_client): - return data - return process_messages(data, sanitize_openai_response_image) + return sanitize_messages(data, ph_client=ph_client) def sanitize_anthropic(data: Any, ph_client: Any = None) -> Any: - if _multimodal_capture_enabled(ph_client): - return data - return process_messages(data, sanitize_anthropic_image) + return sanitize_messages(data, ph_client=ph_client) def sanitize_gemini(data: Any, ph_client: Any = None) -> Any: - if _multimodal_capture_enabled(ph_client): - return data - - if not data: - return data - - if isinstance(data, list): - return [process_gemini_item(item) for item in data] - - return process_gemini_item(data) + return sanitize_messages(data, ph_client=ph_client) def sanitize_langchain(data: Any, ph_client: Any = None) -> Any: - if _multimodal_capture_enabled(ph_client): - return data - return process_messages(data, sanitize_langchain_image) + return sanitize_messages(data, ph_client=ph_client) diff --git a/posthog/ai/types.py b/posthog/ai/types.py index a46ce3e4..6670f12a 100644 --- a/posthog/ai/types.py +++ b/posthog/ai/types.py @@ -7,6 +7,8 @@ from typing import Any, Dict, List, Optional, TypedDict, Union +from typing_extensions import NotRequired # For Python < 3.11 compatibility + class FormattedTextContent(TypedDict): """Formatted text content item.""" @@ -44,11 +46,18 @@ class FormattedMessage(TypedDict): Standardized message format for PostHog tracking. Used across all providers to ensure consistent message structure - when sending events to PostHog. + when sending events to PostHog. ``role`` and ``content`` are always + present; the remaining keys are provider-specific and only set when + the source message carried them (e.g. OpenAI tool-call messages). """ role: str content: Union[str, List[FormattedContentItem], Any] + tool_calls: NotRequired[List[Dict[str, Any]]] + tool_call_id: NotRequired[str] + name: NotRequired[str] + audio: NotRequired[Dict[str, Any]] + refusal: NotRequired[str] class TokenUsage(TypedDict, total=False): @@ -83,13 +92,16 @@ class StreamingContentBlock(TypedDict, total=False): """ Content block used during streaming to accumulate content. - Used for tracking text and function calls as they stream in. + Used for tracking text, function calls, and thinking blocks as they stream in. """ - type: str # "text" or "function" + type: str text: Optional[str] id: Optional[str] function: Optional[Dict[str, Any]] + thinking: Optional[str] + signature: Optional[str] + data: Optional[str] class ToolInProgress(TypedDict): diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 495260b5..cc4817cf 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -4,13 +4,8 @@ from posthog import get_tags, identify_context, new_context, tag, contexts from posthog.ai.gateway import warn_if_posthog_ai_gateway -from posthog.ai.sanitization import ( - _multimodal_capture_enabled, - sanitize_anthropic, - sanitize_gemini, - sanitize_langchain, - sanitize_openai, -) +from posthog.ai.sanitization import _multimodal_capture_enabled, redact_media +from posthog.ai.sanitization import sanitize_messages # noqa: F401 -- re-exported for back-compat from posthog.ai.types import FormattedMessage, StreamingEventData, TokenUsage from posthog.client import Client as PostHogClient @@ -425,7 +420,7 @@ def call_llm_and_track_usage( usage = get_usage(response, provider) messages = merge_system_prompt(kwargs, provider) - sanitized_messages = sanitize_messages(messages, provider, ph_client) + sanitized_messages = finalize_ai_content(messages, ph_client) tag("$ai_provider", provider) tag("$ai_model", kwargs.get("model") or getattr(response, "model", None)) @@ -437,7 +432,9 @@ def call_llm_and_track_usage( tag( "$ai_output_choices", with_privacy_mode( - ph_client, posthog_privacy_mode, format_response(response, provider) + ph_client, + posthog_privacy_mode, + finalize_ai_content(format_response(response, provider), ph_client), ), ) tag("$ai_http_status", http_status) @@ -576,7 +573,7 @@ async def call_llm_and_track_usage_async( usage = get_usage(response, provider) messages = merge_system_prompt(kwargs, provider) - sanitized_messages = sanitize_messages(messages, provider, ph_client) + sanitized_messages = finalize_ai_content(messages, ph_client) tag("$ai_provider", provider) tag("$ai_model", kwargs.get("model") or getattr(response, "model", None)) @@ -588,7 +585,9 @@ async def call_llm_and_track_usage_async( tag( "$ai_output_choices", with_privacy_mode( - ph_client, posthog_privacy_mode, format_response(response, provider) + ph_client, + posthog_privacy_mode, + finalize_ai_content(format_response(response, provider), ph_client), ), ) tag("$ai_http_status", http_status) @@ -666,17 +665,14 @@ async def call_llm_and_track_usage_async( return response -def sanitize_messages(data: Any, provider: str, ph_client: Any = None) -> Any: - """Sanitize messages using provider-specific sanitization functions.""" - if provider == "anthropic": - return sanitize_anthropic(data, ph_client) - elif provider == "openai": - return sanitize_openai(data, ph_client) - elif provider == "gemini": - return sanitize_gemini(data, ph_client) - elif provider == "langchain": - return sanitize_langchain(data, ph_client) - return data +def finalize_ai_content(value: Any, ph_client: Any = None) -> Any: + """Single choke point for AI content properties: structural media redaction + (or bytes->base64 passthrough when the client opted into multimodal capture). + + This is the ONLY function allowed to touch $ai_input / $ai_output_choices / + $ai_input_state / $ai_output_state values before capture. + """ + return redact_media(value, ph_client=ph_client) def with_privacy_mode(ph_client: PostHogClient, privacy_mode: bool, value: Any): @@ -717,12 +713,12 @@ def capture_streaming_event( "$ai_input": with_privacy_mode( ph_client, event_data["privacy_mode"], - event_data["formatted_input"], + finalize_ai_content(event_data["formatted_input"], ph_client), ), "$ai_output_choices": with_privacy_mode( ph_client, event_data["privacy_mode"], - event_data["formatted_output"], + finalize_ai_content(event_data["formatted_output"], ph_client), ), "$ai_http_status": 200, "$ai_input_tokens": event_data["usage_stats"].get("input_tokens", 0), diff --git a/posthog/test/ai/anthropic/test_anthropic.py b/posthog/test/ai/anthropic/test_anthropic.py index 0b5fc805..7566e452 100644 --- a/posthog/test/ai/anthropic/test_anthropic.py +++ b/posthog/test/ai/anthropic/test_anthropic.py @@ -205,6 +205,77 @@ def stream_generator(): return stream_generator() +@pytest.fixture +def mock_anthropic_stream_with_thinking(): + """Mock stream events for a thinking block followed by a text block.""" + + class MockMessage: + def __init__(self): + self.usage = MockUsage( + input_tokens=40, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ) + + def stream_generator(): + # Message start with usage + event = MockStreamEvent("message_start") + event.message = MockMessage() + yield event + + # Thinking block start + event = MockStreamEvent("content_block_start") + event.content_block = MockContentBlock("thinking", thinking="", signature="") + event.index = 0 + yield event + + # Thinking delta + event = MockStreamEvent("content_block_delta") + event.delta = MockDelta(thinking="Let me work through this.") + event.index = 0 + yield event + + # Signature delta + event = MockStreamEvent("content_block_delta") + event.delta = MockDelta(signature="sig-xyz") + event.index = 0 + yield event + + # Thinking block stop + event = MockStreamEvent("content_block_stop") + event.index = 0 + yield event + + # Text block start + event = MockStreamEvent("content_block_start") + event.content_block = MockContentBlock("text") + event.index = 1 + yield event + + # Text delta + event = MockStreamEvent("content_block_delta") + event.delta = MockDelta(text="The answer is 4.") + event.index = 1 + yield event + + # Text block stop + event = MockStreamEvent("content_block_stop") + event.index = 1 + yield event + + # Message delta with final usage + event = MockStreamEvent("message_delta") + event.usage = MockUsage(output_tokens=12) + event.delta = MockDelta(stop_reason="end_turn") + yield event + + # Message stop + event = MockStreamEvent("message_stop") + yield event + + return stream_generator() + + @pytest.fixture def mock_anthropic_response_with_cached_tokens(): # Create a mock Usage object with cached_tokens in input_tokens_details @@ -1079,6 +1150,96 @@ async def run_test(): assert props["$ai_cache_creation_input_tokens"] == 0 +def test_streaming_with_thinking(mock_client, mock_anthropic_stream_with_thinking): + """Test that thinking blocks are accumulated and captured alongside text in streaming mode.""" + with patch( + "anthropic.resources.Messages.create", + return_value=mock_anthropic_stream_with_thinking, + ): + client = Anthropic(api_key="test-key", posthog_client=mock_client) + response = client.messages.create( + model="claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "What's 2 + 2?"}], + stream=True, + posthog_distinct_id="test-id", + ) + + # Consume the stream - this triggers the finally block synchronously + list(response) + + assert mock_client.capture.call_count == 1 + + call_args = mock_client.capture.call_args[1] + props = call_args["properties"] + + output_choices = props["$ai_output_choices"] + assert len(output_choices) == 1 + + content = output_choices[0]["content"] + assert content == [ + { + "type": "thinking", + "thinking": "Let me work through this.", + "signature": "sig-xyz", + }, + {"type": "text", "text": "The answer is 4."}, + ] + + assert props["$ai_stop_reason"] == "end_turn" + + +def test_async_streaming_with_thinking( + mock_client, mock_anthropic_stream_with_thinking +): + """Test that thinking blocks are accumulated and captured alongside text in async streaming mode.""" + import asyncio + + async def mock_async_generator(): + for event in mock_anthropic_stream_with_thinking: + yield event + + async def mock_async_create(**kwargs): + return mock_async_generator() + + with patch( + "anthropic.resources.AsyncMessages.create", + side_effect=mock_async_create, + ): + async_client = AsyncAnthropic(api_key="test-key", posthog_client=mock_client) + + async def run_test(): + response = await async_client.messages.create( + model="claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "What's 2 + 2?"}], + stream=True, + posthog_distinct_id="test-id", + ) + + [event async for event in response] + + asyncio.run(run_test()) + + assert mock_client.capture.call_count == 1 + + call_args = mock_client.capture.call_args[1] + props = call_args["properties"] + + output_choices = props["$ai_output_choices"] + assert len(output_choices) == 1 + + content = output_choices[0]["content"] + assert content == [ + { + "type": "thinking", + "thinking": "Let me work through this.", + "signature": "sig-xyz", + }, + {"type": "text", "text": "The answer is 4."}, + ] + + assert props["$ai_stop_reason"] == "end_turn" + + def test_web_search_count(mock_client): """Test that web search count is properly tracked from Anthropic responses.""" diff --git a/posthog/test/ai/anthropic/test_anthropic_converter.py b/posthog/test/ai/anthropic/test_anthropic_converter.py new file mode 100644 index 00000000..15aaeaa6 --- /dev/null +++ b/posthog/test/ai/anthropic/test_anthropic_converter.py @@ -0,0 +1,159 @@ +import pytest + +try: + from anthropic.types import ( + Message, + RawContentBlockDeltaEvent, + RawContentBlockStartEvent, + SignatureDelta, + TextBlock, + ThinkingBlock, + ThinkingDelta, + ToolUseBlock, + Usage, + ) + + ANTHROPIC_AVAILABLE = True +except ImportError: + ANTHROPIC_AVAILABLE = False + +from posthog.ai.anthropic.anthropic_converter import ( + format_anthropic_input, + format_anthropic_response, + format_anthropic_streaming_content, + format_anthropic_streaming_output_complete, + handle_anthropic_content_block_start, + handle_anthropic_text_delta, +) + +pytestmark = pytest.mark.skipif( + not ANTHROPIC_AVAILABLE, reason="anthropic not available" +) + + +def _msg(content): + return Message( + id="m", + content=content, + model="claude-sonnet-4-5", + role="assistant", + stop_reason="end_turn", + stop_sequence=None, + type="message", + usage=Usage(input_tokens=1, output_tokens=1), + ) + + +def test_round_tripped_blocks_become_dicts(): + out = format_anthropic_input( + [ + { + "role": "assistant", + "content": [ + TextBlock(text="let me check", type="text"), + ToolUseBlock( + id="t1", name="search", input={"q": "x"}, type="tool_use" + ), + ], + }, + ] + ) + content = out[0]["content"] + assert content[0]["type"] == "text" and content[0]["text"] == "let me check" + assert content[1]["type"] == "tool_use" and content[1]["name"] == "search" + assert all(isinstance(block, dict) for block in content) + + +def test_thinking_blocks_survive_response(): + out = format_anthropic_response( + _msg( + [ + ThinkingBlock(thinking="hmm", signature="s", type="thinking"), + TextBlock(text="answer", type="text"), + ] + ) + ) + blocks = out[0]["content"] + assert blocks[0] == {"type": "thinking", "thinking": "hmm", "signature": "s"} + assert blocks[1] == {"type": "text", "text": "answer"} + + +def test_empty_text_block_survives_response(): + out = format_anthropic_response(_msg([TextBlock(text="", type="text")])) + assert out[0]["content"] == [{"type": "text", "text": ""}] + + +def test_unknown_block_kind_preserved(): + out = format_anthropic_input( + [ + { + "role": "user", + "content": [ + { + "type": "web_search_tool_result", + "content": [{"url": "https://x"}], + } + ], + } + ] + ) + assert out[0]["content"][0]["type"] == "web_search_tool_result" + + +def test_streaming_thinking_deltas_accumulate(): + start_event = RawContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ThinkingBlock(type="thinking", thinking="", signature=""), + ) + block, tool = handle_anthropic_content_block_start(start_event) + assert tool is None + content_blocks = [block] + current_block = ( + block + if block is not None and block.get("type") in ("text", "thinking") + else None + ) + + delta_events = [ + RawContentBlockDeltaEvent( + type="content_block_delta", + index=0, + delta=ThinkingDelta(type="thinking_delta", thinking="Let me think"), + ), + RawContentBlockDeltaEvent( + type="content_block_delta", + index=0, + delta=ThinkingDelta(type="thinking_delta", thinking=" it through."), + ), + RawContentBlockDeltaEvent( + type="content_block_delta", + index=0, + delta=SignatureDelta(type="signature_delta", signature="sig-123"), + ), + ] + for event in delta_events: + handle_anthropic_text_delta(event, current_block) + + formatted = format_anthropic_streaming_content(content_blocks) + assert formatted == [ + { + "type": "thinking", + "thinking": "Let me think it through.", + "signature": "sig-123", + } + ] + + output = format_anthropic_streaming_output_complete(content_blocks, "") + assert output == [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Let me think it through.", + "signature": "sig-123", + } + ], + } + ] diff --git a/posthog/test/ai/claude_agent_sdk/test_processor.py b/posthog/test/ai/claude_agent_sdk/test_processor.py index 0db0e865..06d35102 100644 --- a/posthog/test/ai/claude_agent_sdk/test_processor.py +++ b/posthog/test/ai/claude_agent_sdk/test_processor.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import logging from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, patch @@ -415,6 +416,64 @@ async def test_privacy_mode_redacts_tool_input(self, mock_client): assert "$ai_input_state" not in props +class TestMediaRedactionEndToEnd: + @pytest.mark.asyncio + async def test_data_url_prompt_and_tool_use_base64_are_redacted( + self, processor, mock_client + ): + """A data-URL image in the prompt ($ai_input) and base64 in a + tool_use block's input (relayed into $ai_output_choices via + format_assistant_blocks) must both be redacted end-to-end.""" + png_b64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 400).decode() + data_url = f"data:image/png;base64,{png_b64}" + + messages = [ + _make_message_start(), + _make_message_stop(), + _make_assistant_message( + tool_uses=[ + {"id": "tu_1", "name": "Read", "input": {"image": data_url}}, + ] + ), + _make_message_start(), + _make_message_stop(), + _make_result_message(), + ] + + with patch( + "posthog.ai.claude_agent_sdk.processor.original_query", + side_effect=lambda **kw: _fake_query(messages), + ): + async for _ in processor.query( + prompt=data_url, options=ClaudeAgentOptions() + ): + pass + + gen_calls = [ + c + for c in mock_client.capture.call_args_list + if (c.kwargs.get("event") or c[1].get("event")) == "$ai_generation" + ] + assert len(gen_calls) == 2 + + gen1_props = gen_calls[0].kwargs.get("properties") or gen_calls[0][1].get( + "properties" + ) + assert gen1_props["$ai_input"][0]["content"] == "[base64 image redacted]" + + gen2_props = gen_calls[1].kwargs.get("properties") or gen_calls[1][1].get( + "properties" + ) + output_content = gen2_props["$ai_output_choices"][0]["content"] + function_block = next( + item for item in output_content if item.get("type") == "function" + ) + assert ( + function_block["function"]["arguments"]["image"] + == "[base64 image redacted]" + ) + + class TestPersonlessMode: @pytest.mark.asyncio async def test_no_distinct_id_sets_process_person_profile_false(self, mock_client): diff --git a/posthog/test/ai/claude_agent_sdk/test_processor_content.py b/posthog/test/ai/claude_agent_sdk/test_processor_content.py new file mode 100644 index 00000000..1ad190b5 --- /dev/null +++ b/posthog/test/ai/claude_agent_sdk/test_processor_content.py @@ -0,0 +1,119 @@ +import base64 +import types +from dataclasses import dataclass + +import pytest + +try: + from posthog.ai.claude_agent_sdk import client as cas_client + from posthog.ai.claude_agent_sdk import processor as cas_processor + + CLAUDE_AGENT_SDK_AVAILABLE = True +except ImportError: + CLAUDE_AGENT_SDK_AVAILABLE = False + +pytestmark = pytest.mark.skipif( + not CLAUDE_AGENT_SDK_AVAILABLE, reason="Claude Agent SDK is not available" +) + +PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 400).decode() +IMAGE_BLOCK = { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}, +} + + +@dataclass +class FakeToolResultBlock: + tool_use_id: str = "t1" + content: object = None + + +@dataclass +class FakeThinkingBlock: + thinking: str = "let me reason" + signature: str = "sig" + + +@dataclass +class FakeTextBlock: + text: str = "" + + +@dataclass +class FakeThinkingWithEmptyTextBlock: + thinking: str = "let me reason" + text: str = "" + + +@pytest.mark.parametrize( + "mod", [cas_processor, cas_client] if CLAUDE_AGENT_SDK_AVAILABLE else [] +) +def test_tool_result_keeps_structure_and_redacts_image(mod): + block = FakeToolResultBlock( + content=[IMAGE_BLOCK, {"type": "text", "text": "x" * 6000}] + ) + out = mod.format_tool_result_content(block) + assert isinstance(out, list) + assert out[0]["source"]["data"] == "[base64 image redacted]" + assert out[1]["text"].endswith("... [truncated]") + assert len(out[1]["text"]) == 5000 + len("... [truncated]") + + +@pytest.mark.parametrize( + "mod", [cas_processor, cas_client] if CLAUDE_AGENT_SDK_AVAILABLE else [] +) +def test_empty_string_tool_content_not_none(mod): + assert mod.format_tool_result_content(FakeToolResultBlock(content="")) == "" + + +@pytest.mark.parametrize( + "mod", [cas_processor, cas_client] if CLAUDE_AGENT_SDK_AVAILABLE else [] +) +def test_passthrough_tool_result_media_not_truncated(mod): + # A client opted into multimodal capture must not get its tool-result media + # cut at max_string_len — a 5000-char slice through base64 corrupts it. + client = types.SimpleNamespace(_enable_multimodal_capture=True) + long_b64 = "A" * 6000 + block = FakeToolResultBlock( + content=[ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": long_b64, + }, + } + ] + ) + out = mod.format_tool_result_content(block, ph_client=client) + assert out[0]["source"]["data"] == long_b64 + + +@pytest.mark.parametrize( + "mod", [cas_processor, cas_client] if CLAUDE_AGENT_SDK_AVAILABLE else [] +) +def test_thinking_block_captured(mod): + out = mod.format_assistant_blocks([FakeThinkingBlock()]) + assert { + "type": "thinking", + "thinking": "let me reason", + "signature": "sig", + } in out + + +@pytest.mark.parametrize( + "mod", [cas_processor, cas_client] if CLAUDE_AGENT_SDK_AVAILABLE else [] +) +def test_empty_text_block_keeps_type_label(mod): + out = mod.format_assistant_blocks([FakeTextBlock()]) + assert out == [{"type": "text", "text": ""}] + + +@pytest.mark.parametrize( + "mod", [cas_processor, cas_client] if CLAUDE_AGENT_SDK_AVAILABLE else [] +) +def test_thinking_takes_precedence_over_empty_text(mod): + out = mod.format_assistant_blocks([FakeThinkingWithEmptyTextBlock()]) + assert out == [{"type": "thinking", "thinking": "let me reason"}] diff --git a/posthog/test/ai/gemini/test_gemini_converter.py b/posthog/test/ai/gemini/test_gemini_converter.py new file mode 100644 index 00000000..3991a8ef --- /dev/null +++ b/posthog/test/ai/gemini/test_gemini_converter.py @@ -0,0 +1,246 @@ +import base64 + +import pytest + +try: + from google.genai import types + + GEMINI_AVAILABLE = True +except ImportError: + GEMINI_AVAILABLE = False + +from posthog.ai.gemini.gemini_converter import ( + extract_gemini_content_from_chunk, + format_gemini_input, + format_gemini_response, + format_gemini_streaming_output, +) + +pytestmark = pytest.mark.skipif( + not GEMINI_AVAILABLE, reason="google-genai not available" +) + +PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 40 + + +class TestGeminiInputParts: + def test_typed_video_file_part_preserved(self): + contents = [ + types.Content( + role="user", + parts=[ + types.Part(text="watch this"), + types.Part( + file_data=types.FileData( + file_uri="https://files/abc", mime_type="video/mp4" + ) + ), + ], + ) + ] + out = format_gemini_input(contents) + assert out[0]["content"][0] == {"type": "text", "text": "watch this"} + assert out[0]["content"][1] == { + "type": "video", + "file_data": {"mime_type": "video/mp4", "file_uri": "https://files/abc"}, + } + + def test_typed_inline_image_part_bytes_base64d(self): + contents = [ + types.Content( + role="user", + parts=[ + types.Part(inline_data=types.Blob(data=PNG, mime_type="image/png")) + ], + ) + ] + out = format_gemini_input(contents) + block = out[0]["content"][0] + assert block["type"] == "image" + assert block["inline_data"]["data"] == base64.b64encode(PNG).decode() + + def test_function_call_and_response_turns_preserved(self): + contents = [ + types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall( + name="get_weather", args={"city": "SF"} + ) + ) + ], + ), + types.Content( + role="user", + parts=[ + types.Part( + function_response=types.FunctionResponse( + name="get_weather", response={"temp": "18C"} + ) + ) + ], + ), + ] + out = format_gemini_input(contents) + assert out[0]["content"][0]["type"] == "function_call" + assert out[0]["content"][0]["function_call"]["name"] == "get_weather" + assert out[1]["content"][0]["type"] == "function_response" + + def test_camel_case_dict_part_preserved(self): + out = format_gemini_input( + [ + { + "role": "user", + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": base64.b64encode(PNG).decode(), + } + } + ], + } + ] + ) + assert out[0]["content"][0]["type"] == "image" + assert out[0]["content"][0]["inline_data"]["mime_type"] == "image/png" + + def test_model_dumped_part_with_none_text_keeps_image(self): + out = format_gemini_input( + [ + { + "role": "user", + "parts": [ + { + "text": None, + "inline_data": {"mime_type": "image/png", "data": "AAAA"}, + } + ], + } + ] + ) + assert out[0]["content"] == [ + {"type": "image", "inline_data": {"mime_type": "image/png", "data": "AAAA"}} + ] + + def test_bare_part_list_same_as_wrapped(self): + bare = format_gemini_input( + [types.Part(inline_data=types.Blob(data=PNG, mime_type="image/png")), "hi"] + ) + assert bare[0]["content"][0]["type"] == "image" + assert bare[1] == {"role": "user", "content": "hi"} + + def test_unknown_part_kind_preserved_with_label(self): + out = format_gemini_input( + [ + { + "role": "user", + "parts": [ + {"executable_code": {"language": "PYTHON", "code": "1+1"}} + ], + } + ] + ) + assert out[0]["content"][0]["type"] == "executable_code" + + def test_empty_string_text_kept(self): + out = format_gemini_input([{"role": "user", "parts": [{"text": ""}]}]) + assert out[0]["content"] == [{"type": "text", "text": ""}] + + +class TestGeminiResponse: + def test_image_generation_output_labeled_image(self): + resp = types.GenerateContentResponse( + candidates=[ + types.Candidate( + content=types.Content( + role="model", + parts=[ + types.Part(text="here you go:"), + types.Part( + inline_data=types.Blob(data=PNG, mime_type="image/png") + ), + ], + ), + ) + ] + ) + out = format_gemini_response(resp) + blocks = out[0]["content"] + assert blocks[0] == {"type": "text", "text": "here you go:"} + assert blocks[1]["type"] == "image" + assert blocks[1]["inline_data"]["mime_type"] == "image/png" + assert blocks[1]["inline_data"]["data"] == base64.b64encode(PNG).decode() + + def test_streaming_keeps_inline_data_chunks(self): + chunks = [ + types.GenerateContentResponse( + candidates=[ + types.Candidate( + content=types.Content( + role="model", parts=[types.Part(text="sure: ")] + ) + ) + ] + ), + types.GenerateContentResponse( + candidates=[ + types.Candidate( + content=types.Content( + role="model", + parts=[ + types.Part( + inline_data=types.Blob( + data=PNG, mime_type="image/png" + ) + ) + ], + ) + ) + ] + ), + ] + acc: list = [] + for ch in chunks: + blocks = extract_gemini_content_from_chunk(ch) + if blocks: + acc.extend(blocks) + out = format_gemini_streaming_output(acc) + types_seen = [b["type"] for b in out[0]["content"]] + assert "text" in types_seen and "image" in types_seen + + def test_streaming_chunk_with_mixed_text_and_image_captures_both(self): + chunk = types.GenerateContentResponse( + candidates=[ + types.Candidate( + content=types.Content( + role="model", + parts=[ + types.Part(text="here: "), + types.Part( + inline_data=types.Blob(data=PNG, mime_type="image/png") + ), + ], + ) + ) + ] + ) + blocks = extract_gemini_content_from_chunk(chunk) + assert blocks == [ + {"type": "text", "text": "here: "}, + { + "type": "image", + "inline_data": { + "mime_type": "image/png", + "data": base64.b64encode(PNG).decode(), + }, + }, + ] + + out = format_gemini_streaming_output(blocks) + content = out[0]["content"] + types_seen = [b["type"] for b in content] + assert types_seen == ["text", "image"] + assert content[0] == {"type": "text", "text": "here: "} + assert content[1]["inline_data"]["data"] == base64.b64encode(PNG).decode() diff --git a/posthog/test/ai/openai/test_openai.py b/posthog/test/ai/openai/test_openai.py index c895e490..e82bb902 100644 --- a/posthog/test/ai/openai/test_openai.py +++ b/posthog/test/ai/openai/test_openai.py @@ -27,7 +27,6 @@ Response, ResponseOutputMessage, ResponseOutputText, - ResponseUsage, ResponseFunctionToolCall, ParsedResponse, ) @@ -39,7 +38,7 @@ from posthog.ai.openai import OpenAI from posthog.ai.openai.openai_async import AsyncOpenAI from posthog.ai.openai.wrapper_utils import reset_fallback_warnings - from posthog.test.ai.utils import RecordingAsyncStream + from posthog.test.ai.utils import RecordingAsyncStream, make_response_usage OPENAI_AVAILABLE = True except ImportError: @@ -114,12 +113,11 @@ def mock_openai_response_with_responses_api(): ], parallel_tool_calls=True, previous_response_id=None, - usage=ResponseUsage( + usage=make_response_usage( input_tokens=10, output_tokens=10, - input_tokens_details={"prompt_tokens": 10, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 15}, total_tokens=20, + reasoning_tokens=15, ), user=None, metadata={}, @@ -167,12 +165,11 @@ def mock_parsed_response(): }, parallel_tool_calls=True, previous_response_id=None, - usage=ResponseUsage( + usage=make_response_usage( input_tokens=15, output_tokens=20, - input_tokens_details={"prompt_tokens": 15, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 5}, total_tokens=35, + reasoning_tokens=5, ), user=None, metadata={}, @@ -480,11 +477,9 @@ def mock_responses_api_with_tool_calls(): status="completed", ), ], - usage=ResponseUsage( + usage=make_response_usage( input_tokens=30, output_tokens=20, - input_tokens_details={"prompt_tokens": 30, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 0}, total_tokens=50, ), previous_response_id=None, @@ -1240,7 +1235,6 @@ def test_fallback_logs_warning( def test_responses_api_streaming_with_tokens(mock_client): """Test that Responses API streaming properly captures token usage from response.usage.""" - from openai.types.responses import ResponseUsage from unittest.mock import MagicMock # Create mock response chunks with usage data in the correct location @@ -1262,12 +1256,10 @@ def test_responses_api_streaming_with_tokens(mock_client): chunk3 = MagicMock() chunk3.type = "response.completed" chunk3.response = MagicMock() - chunk3.response.usage = ResponseUsage( + chunk3.response.usage = make_response_usage( input_tokens=25, output_tokens=30, total_tokens=55, - input_tokens_details={"prompt_tokens": 25, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 0}, ) chunk3.response.output = ["Test response"] chunks.append(chunk3) @@ -1381,7 +1373,6 @@ async def chunk_iterable(): @pytest.mark.asyncio async def test_async_responses_streaming_with_tokens(mock_client): - from openai.types.responses import ResponseUsage from unittest.mock import MagicMock chunks = [] @@ -1399,12 +1390,10 @@ async def test_async_responses_streaming_with_tokens(mock_client): chunk3 = MagicMock() chunk3.type = "response.completed" chunk3.response = MagicMock() - chunk3.response.usage = ResponseUsage( + chunk3.response.usage = make_response_usage( input_tokens=25, output_tokens=30, total_tokens=55, - input_tokens_details={"prompt_tokens": 25, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 0}, ) chunk3.response.output = ["Test response"] chunks.append(chunk3) @@ -1991,7 +1980,6 @@ def test_streaming_chat_prefers_kwargs_model_over_chunk_model(mock_client): def test_streaming_responses_api_extracts_model_from_response_object(mock_client): """Test that Responses API streaming extracts model from chunk.response.model (stored prompts).""" from unittest.mock import MagicMock - from openai.types.responses import ResponseUsage chunks = [] @@ -2008,12 +1996,10 @@ def test_streaming_responses_api_extracts_model_from_response_object(mock_client chunk2.type = "response.completed" chunk2.response = MagicMock() chunk2.response.model = "gpt-4o-mini-stored" # Model from stored prompt - chunk2.response.usage = ResponseUsage( + chunk2.response.usage = make_response_usage( input_tokens=20, output_tokens=10, total_tokens=30, - input_tokens_details={"prompt_tokens": 20, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 0}, ) chunk2.response.output = ["Test response"] chunks.append(chunk2) @@ -2115,11 +2101,9 @@ def test_non_streaming_responses_api_extracts_model_from_response(mock_client): ], parallel_tool_calls=True, previous_response_id=None, - usage=ResponseUsage( + usage=make_response_usage( input_tokens=10, output_tokens=10, - input_tokens_details={"prompt_tokens": 10, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 0}, total_tokens=20, ), user=None, @@ -2287,7 +2271,6 @@ async def chunk_iterable(): async def test_async_streaming_responses_extracts_model_from_response(mock_client): """Test async Responses API streaming extracts model from chunk.response.model.""" from unittest.mock import MagicMock - from openai.types.responses import ResponseUsage chunks = [] @@ -2301,12 +2284,10 @@ async def test_async_streaming_responses_extracts_model_from_response(mock_clien chunk2.type = "response.completed" chunk2.response = MagicMock() chunk2.response.model = "gpt-4o-mini-async-stored" - chunk2.response.usage = ResponseUsage( + chunk2.response.usage = make_response_usage( input_tokens=20, output_tokens=10, total_tokens=30, - input_tokens_details={"prompt_tokens": 20, "cached_tokens": 0}, - output_tokens_details={"reasoning_tokens": 0}, ) chunk2.response.output = ["Test"] chunks.append(chunk2) diff --git a/posthog/test/ai/openai/test_openai_converter.py b/posthog/test/ai/openai/test_openai_converter.py new file mode 100644 index 00000000..1c3668f6 --- /dev/null +++ b/posthog/test/ai/openai/test_openai_converter.py @@ -0,0 +1,200 @@ +import pytest + +try: + from openai.types.chat import ChatCompletionMessage + + OPENAI_AVAILABLE = True +except ImportError: + OPENAI_AVAILABLE = False + +from posthog.ai.openai.openai_converter import format_openai_input +from posthog.test.ai.utils import make_response_usage + +pytestmark = pytest.mark.skipif(not OPENAI_AVAILABLE, reason="openai not available") + + +def _build_reasoning_response(): + from openai.types.responses import Response + from openai.types.responses.response_output_message import ResponseOutputMessage + from openai.types.responses.response_output_text import ResponseOutputText + from openai.types.responses.response_reasoning_item import ResponseReasoningItem + + return Response( + id="r", + created_at=1.0, + model="o4-mini", + object="response", + output=[ + ResponseReasoningItem(id="rs", summary=[], type="reasoning"), + ResponseOutputMessage( + id="m", + role="assistant", + status="completed", + type="message", + content=[ + ResponseOutputText(annotations=[], text="42", type="output_text") + ], + ), + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=make_response_usage( + input_tokens=1, + output_tokens=1, + total_tokens=2, + ), + ) + + +def test_tool_calls_preserved_in_input(): + out = format_openai_input( + [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "42"}, + ] + ) + assert out[0]["tool_calls"][0]["id"] == "c1" + assert out[0]["content"] is None + assert out[1]["tool_call_id"] == "c1" + + +def test_message_object_does_not_crash(): + msg = ChatCompletionMessage(role="assistant", content="hello", refusal=None) + out = format_openai_input([{"role": "user", "content": "hi"}, msg]) + assert out[1]["role"] == "assistant" + assert out[1]["content"] == "hello" + + +def test_responses_output_keeps_all_items(): + from posthog.ai.openai.openai_converter import format_openai_response + + resp = _build_reasoning_response() + out = format_openai_response(resp) + types_seen = [b["type"] for b in out[0]["content"]] + assert "reasoning" in types_seen + assert any(b.get("text") == "42" for b in out[0]["content"]) + + +def test_responses_output_dict_items_not_dropped(): + """Proxied/serialized Responses arrive dict-shaped; typed-only access used + to read type as None and drop every item, capturing [].""" + import types + + from posthog.ai.openai.openai_converter import format_openai_response + + resp = types.SimpleNamespace( + output=[ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "42"}], + }, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": "{}", + }, + ] + ) + + out = format_openai_response(resp) + content = out[0]["content"] + assert {"type": "text", "text": "42"} in content + assert { + "type": "function", + "id": "c1", + "function": {"name": "get_weather", "arguments": "{}"}, + } in content + + +def test_responses_streaming_matches_nonstreaming(): + from posthog.ai.openai.openai_converter import ( + extract_openai_content_from_chunk, + format_openai_response, + format_openai_streaming_output, + ) + + class Completed: + type = "response.completed" + + def __init__(self, response): + self.response = response + + resp = _build_reasoning_response() + acc = [extract_openai_content_from_chunk(Completed(resp), "responses")] + streaming_out = format_openai_streaming_output( + [a for a in acc if a is not None], "responses" + ) + assert streaming_out == format_openai_response(resp) + + +def test_chat_streaming_audio_and_refusal_deltas(): + """ + `ChoiceDelta` has no declared `audio` field (only the non-streaming + `ChatCompletionMessage` does), but the OpenAI SDK's pydantic models allow + extra fields (`model_config = ConfigDict(extra="allow")`), so an `audio` + delta from the gpt-4o-audio-preview streaming API round-trips as a plain + dict via `model_validate` — no stub object needed. + """ + from openai.types.chat.chat_completion_chunk import ( + ChatCompletionChunk, + Choice, + ChoiceDelta, + ) + + from posthog.ai.openai.openai_converter import ( + extract_openai_content_from_chunk, + format_openai_streaming_output, + ) + + def _chunk(delta_kwargs): + delta = ChoiceDelta.model_validate(delta_kwargs) + return ChatCompletionChunk( + id="c1", + object="chat.completion.chunk", + created=1, + model="gpt-4o-audio-preview", + choices=[Choice(index=0, delta=delta, finish_reason=None)], + ) + + chunks = [ + _chunk( + { + "role": "assistant", + "audio": {"id": "a1", "transcript": "hel", "data": "AA=="}, + } + ), + _chunk({"audio": {"transcript": "lo", "data": "BB=="}}), + _chunk({"refusal": "I can't help with that"}), + ] + + accumulated = [ + item + for item in ( + extract_openai_content_from_chunk(chunk, "chat") for chunk in chunks + ) + if item is not None + ] + + output = format_openai_streaming_output(accumulated, "chat") + content = output[0]["content"] + + audio_block = next(b for b in content if b["type"] == "audio") + assert audio_block["id"] == "a1" + assert audio_block["transcript"] == "hello" + assert audio_block["data"] == "AA==BB==" + + refusal_block = next(b for b in content if b["type"] == "refusal") + assert refusal_block["refusal"] == "I can't help with that" diff --git a/posthog/test/ai/openai_agents/test_processor.py b/posthog/test/ai/openai_agents/test_processor.py index b1e98c2a..3d4d5b8b 100644 --- a/posthog/test/ai/openai_agents/test_processor.py +++ b/posthog/test/ai/openai_agents/test_processor.py @@ -1,3 +1,4 @@ +import base64 import logging from unittest.mock import MagicMock, patch @@ -17,6 +18,7 @@ ) from posthog.ai.openai_agents import PostHogTracingProcessor, instrument + from posthog.ai.openai_agents.processor import _ensure_serializable OPENAI_AGENTS_AVAILABLE = True except ImportError: @@ -401,6 +403,91 @@ def test_privacy_mode_redacts_content(self, mock_client, mock_span): assert call_kwargs["properties"]["$ai_input_tokens"] == 10 assert call_kwargs["properties"]["$ai_output_tokens"] == 20 + def test_generation_span_image_input_is_redacted( + self, processor, mock_client, mock_span + ): + """Test that base64 image data URLs in generation span input are redacted.""" + png_b64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 400).decode() + span_data = GenerationSpanData( + input=[ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": f"data:image/png;base64,{png_b64}", + } + ], + } + ], + output=[{"role": "assistant", "content": "ok"}], + model="gpt-4o", + ) + mock_span.span_data = span_data + + processor.on_span_start(mock_span) + processor.on_span_end(mock_span) + + call_kwargs = mock_client.capture.call_args[1] + captured_input = call_kwargs["properties"]["$ai_input"] + assert captured_input[0]["content"][0]["image_url"] == "[base64 image redacted]" + + def test_generation_span_bytes_input_becomes_base64_in_passthrough_mode( + self, processor, mock_client, mock_span + ): + """Test that raw bytes content becomes a base64 string under multimodal passthrough.""" + mock_client._enable_multimodal_capture = True + raw = b"\x00\x01\x02\x03" + span_data = GenerationSpanData( + input=[{"role": "user", "content": raw}], + output=[], + model="gpt-4o", + ) + mock_span.span_data = span_data + + processor.on_span_start(mock_span) + processor.on_span_end(mock_span) + + call_kwargs = mock_client._capture_ai.call_args[1] + captured_input = call_kwargs["properties"]["$ai_input"] + assert captured_input[0]["content"] == base64.b64encode(raw).decode() + + def test_response_span_image_output_is_redacted( + self, processor, mock_client, mock_span + ): + """Test that base64 image data URLs in response span output are redacted.""" + png_b64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 400).decode() + mock_response = MagicMock() + mock_response.id = "resp_123" + mock_response.model = "gpt-4o" + mock_response.output = [ + { + "type": "message", + "content": [ + { + "type": "output_image", + "image_url": f"data:image/png;base64,{png_b64}", + } + ], + } + ] + mock_response.usage = MagicMock() + mock_response.usage.input_tokens = 1 + mock_response.usage.output_tokens = 1 + mock_response.usage.cost = None + + span_data = ResponseSpanData(response=mock_response, input="draw a cat") + mock_span.span_data = span_data + + processor.on_span_start(mock_span) + processor.on_span_end(mock_span) + + call_kwargs = mock_client.capture.call_args[1] + captured_output = call_kwargs["properties"]["$ai_output_choices"] + assert ( + captured_output[0]["content"][0]["image_url"] == "[base64 image redacted]" + ) + def test_error_handling_in_span(self, processor, mock_client, mock_span): """Test that span errors are captured correctly.""" span_data = GenerationSpanData(model="gpt-4o") @@ -643,6 +730,46 @@ def test_transcription_span_with_pass_through_properties( == "This is the transcribed text." ) + def test_transcription_span_audio_input_redacted( + self, processor, mock_client, mock_span + ): + """Transcription input is base64 audio, not text — a bare string the + structural redactor can't recognize, so it must be redacted here.""" + b64 = base64.b64encode(b"\x00" * 1000).decode() + span_data = TranscriptionSpanData( + input=b64, + input_format="pcm", + output="This is the transcribed text.", + model="whisper-1", + ) + mock_span.span_data = span_data + + processor.on_span_start(mock_span) + processor.on_span_end(mock_span) + + call_kwargs = mock_client.capture.call_args[1] + assert call_kwargs["properties"]["$ai_input"] == "[base64 audio redacted]" + + def test_transcription_span_audio_input_passthrough( + self, processor, mock_client, mock_span + ): + """Under multimodal passthrough the raw audio is kept intact.""" + mock_client._enable_multimodal_capture = True + b64 = base64.b64encode(b"\x00" * 1000).decode() + span_data = TranscriptionSpanData( + input=b64, + input_format="pcm", + output="This is the transcribed text.", + model="whisper-1", + ) + mock_span.span_data = span_data + + processor.on_span_start(mock_span) + processor.on_span_end(mock_span) + + call_kwargs = mock_client._capture_ai.call_args[1] + assert call_kwargs["properties"]["$ai_input"] == b64 + def test_latency_calculation(self, processor, mock_client, mock_span): """Test that latency is calculated correctly.""" span_data = GenerationSpanData(model="gpt-4o") @@ -797,6 +924,26 @@ def test_eviction_of_stale_entries(self, mock_client): assert len(processor._trace_metadata) <= 10 +class TestEnsureSerializableCycleGuard: + def test_self_referencing_dict_returns_circular_marker(self): + node = {"a": 1} + node["self"] = node + + result = _ensure_serializable(node) + + assert result["a"] == 1 + assert result["self"] == "" + + def test_self_referencing_list_returns_circular_marker(self): + node = [1, 2] + node.append(node) + + result = _ensure_serializable(node) + + assert result[:2] == [1, 2] + assert result[2] == "" + + class TestInstrumentHelper: """Tests for the instrument() convenience function.""" diff --git a/posthog/test/ai/test_capture_contract.py b/posthog/test/ai/test_capture_contract.py new file mode 100644 index 00000000..5b4951c7 --- /dev/null +++ b/posthog/test/ai/test_capture_contract.py @@ -0,0 +1,347 @@ +"""Capture invariant: no known provider content kind may capture as [], None, or a repr string.""" + +import base64 +from uuid import uuid4 + +import pytest + +from posthog.ai.sanitization import redact_media + +PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 40 +PNG_B64 = base64.b64encode(PNG).decode() + +# Long enough to clear the structural redactor's strong-context length floor +# (_STRONG_CONTEXT_MIN_LEN) so the healthy-path pins actually exercise redaction. +LONG_PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 400).decode() +PLACEHOLDER = "[base64 image redacted]" + + +def _flat_reprs(value): + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [r for v in value.values() for r in _flat_reprs(v)] + if isinstance(value, list): + return [r for v in value for r in _flat_reprs(v)] + return [] + + +def _assert_faithful(formatted_messages): + assert formatted_messages, "captured nothing" + for message in formatted_messages: + content = message.get("content") + assert content != [], f"empty content for {message}" + for leaf in _flat_reprs(message): + assert "object at 0x" not in leaf and not leaf.startswith( + ("TextBlock(", "ToolUseBlock(", "ResponseReasoningItem(") + ), f"repr leaked: {leaf[:80]}" + + +GEMINI_PART_KINDS = [ + {"text": "hi"}, + {"inline_data": {"mime_type": "image/png", "data": PNG_B64}}, + {"inline_data": {"mime_type": "video/mp4", "data": PNG_B64}}, + {"file_data": {"mime_type": "video/mp4", "file_uri": "https://f/1"}}, + {"function_call": {"name": "f", "args": {}}}, + {"function_response": {"name": "f", "response": {}}}, + {"executable_code": {"language": "PYTHON", "code": "1"}}, + {"code_execution_result": {"outcome": "OUTCOME_OK", "output": "1"}}, +] + + +@pytest.mark.parametrize("part", GEMINI_PART_KINDS) +def test_gemini_kind_coverage_dict(part): + from posthog.ai.gemini.gemini_converter import format_gemini_input + + out = format_gemini_input([{"role": "user", "parts": [part]}]) + _assert_faithful(out) + assert out[0]["content"], f"part dropped: {part}" + + +@pytest.mark.parametrize("part", GEMINI_PART_KINDS) +def test_gemini_kind_coverage_typed(part): + types = pytest.importorskip("google.genai").types + from posthog.ai.gemini.gemini_converter import format_gemini_input + + typed = types.Part.model_validate(part) + out = format_gemini_input([types.Content(role="user", parts=[typed])]) + _assert_faithful(out) + assert out[0]["content"], f"typed part dropped: {part}" + + +ANTHROPIC_BLOCK_KINDS = [ + {"type": "text", "text": "hi"}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}, + }, + {"type": "image", "source": {"type": "url", "url": "https://x/y.png"}}, + { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": PNG_B64}, + }, + {"type": "tool_use", "id": "t", "name": "f", "input": {}}, + {"type": "tool_result", "tool_use_id": "t", "content": "ok"}, + { + "type": "tool_result", + "tool_use_id": "t", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_B64, + }, + } + ], + }, + {"type": "thinking", "thinking": "hmm", "signature": "s"}, +] + + +@pytest.mark.parametrize("block", ANTHROPIC_BLOCK_KINDS) +def test_anthropic_input_kind_coverage(block): + from posthog.ai.anthropic.anthropic_converter import format_anthropic_input + + out = format_anthropic_input([{"role": "user", "content": [block]}]) + _assert_faithful(out) + assert out[0]["content"], f"block dropped: {block}" + + +OPENAI_CHAT_PART_KINDS = [ + {"type": "text", "text": "hi"}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{PNG_B64}"}}, + {"type": "input_audio", "input_audio": {"data": PNG_B64, "format": "wav"}}, + {"type": "file", "file": {"file_id": "f1"}}, +] + + +@pytest.mark.parametrize("part", OPENAI_CHAT_PART_KINDS) +def test_openai_chat_input_kind_coverage(part): + from posthog.ai.openai.openai_converter import format_openai_input + + out = format_openai_input([{"role": "user", "content": [part]}]) + _assert_faithful(out) + assert out[0]["content"], f"part dropped: {part}" + + +OPENAI_RESPONSES_INPUT_KINDS = [ + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "c1", "output": "72F"}, + { + "type": "reasoning", + "id": "r1", + "summary": [{"type": "summary_text", "text": "thinking"}], + }, +] + + +@pytest.mark.parametrize("item", OPENAI_RESPONSES_INPUT_KINDS) +def test_openai_responses_input_kind_coverage(item): + from posthog.ai.openai.openai_converter import format_openai_input + + out = format_openai_input(input_data=[item]) + _assert_faithful(out) + assert out[0] != {"role": "user", "content": ""}, f"item collapsed: {item}" + + +# Typed variants of the same kinds. The canonical Responses agent loop is +# `input_list += response.output`, which appends typed SDK objects — those must +# normalize like the dict shapes above instead of capturing as repr strings. +def _typed_responses_input_items(): + responses = pytest.importorskip("openai").types.responses + return [ + responses.response_function_tool_call.ResponseFunctionToolCall( + arguments="{}", call_id="c1", name="get_weather", type="function_call" + ), + responses.response_reasoning_item.ResponseReasoningItem( + id="r1", summary=[], type="reasoning" + ), + responses.response_output_message.ResponseOutputMessage( + id="m1", + role="assistant", + status="completed", + type="message", + content=[ + responses.response_output_text.ResponseOutputText( + annotations=[], text="hi", type="output_text" + ) + ], + ), + ] + + +def test_openai_responses_input_kind_coverage_typed(): + from posthog.ai.openai.openai_converter import format_openai_input + + for item in _typed_responses_input_items(): + out = format_openai_input(input_data=[item]) + _assert_faithful(out) + assert out[0] != {"role": "user", "content": ""}, f"item collapsed: {item!r}" + + +def test_openai_responses_streaming_empty_output_not_fabricated(): + from posthog.ai.openai.openai_converter import format_openai_streaming_output + + assert format_openai_streaming_output([], "responses") == [] + + +# --- Healthy-path pins ------------------------------------------------- +# +# The kind-coverage tests above only prove a block survives *formatting* +# faithfully; they don't touch redaction. These pin the structural redactor +# (posthog/ai/sanitization.py `redact_media`) actually strips base64 media +# for the shapes each provider/framework sends it in, at the layer each +# capture path calls it from. + + +def test_anthropic_top_level_image_redacted(): + out = redact_media( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": LONG_PNG_B64, + }, + } + ], + } + ] + ) + assert out[0]["content"][0]["source"]["data"] == PLACEHOLDER + + +def test_openai_chat_image_url_data_url_redacted(): + out = redact_media( + [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{LONG_PNG_B64}"}, + } + ], + } + ] + ) + assert out[0]["content"][0]["image_url"]["url"] == PLACEHOLDER + + +class FakePH: + privacy_mode = False + + def __init__(self): + self.events = [] + + def capture(self, *args, **kwargs): + self.events.append(kwargs) + + def flush(self): + pass + + +def _run_langchain_input(content): + """Run the CallbackHandler harness (on_chat_model_start + on_llm_end) and + return the captured $ai_input, mirroring how a real LangChain run drives + the handler.""" + from langchain_core.messages import AIMessage, HumanMessage + from langchain_core.outputs import ChatGeneration, LLMResult + from posthog.ai.langchain.callbacks import CallbackHandler + + fake = FakePH() + cb = CallbackHandler(client=fake) + run_id = uuid4() + cb.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content=content)]], + run_id=run_id, + ) + cb.on_llm_end( + LLMResult( + generations=[[ChatGeneration(message=AIMessage(content="ok"))]], + llm_output={}, + ), + run_id=run_id, + ) + return fake.events[-1]["properties"]["$ai_input"] + + +LANGCHAIN_IMAGE_SHAPES = [ + pytest.param( + [ + {"type": "text", "text": "what is this"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": LONG_PNG_B64, + }, + }, + ], + lambda content: content[1]["source"]["data"], + id="anthropic-style", + ), + pytest.param( + [ + {"type": "text", "text": "describe"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{LONG_PNG_B64}"}, + }, + ], + lambda content: content[1]["image_url"]["url"], + id="openai-style", + ), + pytest.param( + [ + {"type": "text", "text": "describe"}, + { + "type": "image", + "source_type": "base64", + "data": LONG_PNG_B64, + "mime_type": "image/png", + }, + ], + lambda content: content[1]["data"], + id="langchain-v0.3-image-block", + ), +] + + +@pytest.mark.parametrize("content, extract_data", LANGCHAIN_IMAGE_SHAPES) +def test_langchain_callback_input_image_redacted(content, extract_data): + pytest.importorskip("langchain_core") + + captured = _run_langchain_input(content) + assert extract_data(captured[0]["content"]) == PLACEHOLDER + + +def test_langchain_callback_v03_file_block_redacted(): + """The v0.3 standard file content block redacts its base64 `data` via the + `data` strong key with a `mime_type` sibling.""" + pytest.importorskip("langchain_core") + + content = [ + { + "type": "file", + "source_type": "base64", + "data": LONG_PNG_B64, + "mime_type": "application/pdf", + } + ] + captured = _run_langchain_input(content) + assert captured[0]["content"][0]["data"] == "[base64 file redacted]" diff --git a/posthog/test/ai/test_capture_pipeline.py b/posthog/test/ai/test_capture_pipeline.py new file mode 100644 index 00000000..3621f739 --- /dev/null +++ b/posthog/test/ai/test_capture_pipeline.py @@ -0,0 +1,339 @@ +import base64 + +import pytest + +PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 400).decode() +PLACEHOLDER = "[base64 image redacted]" + + +class FakePH: + privacy_mode = False + + def __init__(self): + self.events = [] + + def capture(self, *args, **kwargs): + self.events.append(kwargs) + + def flush(self): + pass + + +def props(fake): + assert fake.events + return fake.events[-1]["properties"] + + +@pytest.fixture +def fake_ph(): + return FakePH() + + +def test_gemini_dict_image_input_is_redacted(fake_ph, monkeypatch): + google_genai = pytest.importorskip("google.genai") + from google.genai import types + from posthog.ai.gemini import Client + + resp = types.GenerateContentResponse( + candidates=[ + types.Candidate( + content=types.Content(role="model", parts=[types.Part(text="ok")]), + finish_reason="STOP", + ) + ], + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=1, candidates_token_count=1, total_token_count=2 + ), + ) + monkeypatch.setattr( + google_genai.models.Models, "generate_content", lambda self, **kwargs: resp + ) + client = Client(posthog_client=fake_ph, api_key="k") + client.models.generate_content( + model="gemini-2.5-flash", + contents=[ + { + "role": "user", + "parts": [{"inline_data": {"mime_type": "image/png", "data": PNG_B64}}], + }, + ], + ) + block = props(fake_ph)["$ai_input"][0]["content"][0] + assert block["inline_data"]["data"] == PLACEHOLDER + + +def test_openai_audio_output_is_redacted(fake_ph, monkeypatch): + openai_mod = pytest.importorskip("openai") + from openai.types import CompletionUsage + from openai.types.chat import ChatCompletion, ChatCompletionMessage + from openai.types.chat.chat_completion import Choice + from openai.types.chat.chat_completion_audio import ChatCompletionAudio + from posthog.ai.openai import OpenAI + + completion = ChatCompletion( + id="c", + created=1, + model="gpt-4o-audio-preview", + object="chat.completion", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage( + role="assistant", + content=None, + audio=ChatCompletionAudio( + id="a", data=PNG_B64 * 3, expires_at=2, transcript="hi" + ), + ), + ) + ], + usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2), + ) + monkeypatch.setattr( + openai_mod.resources.chat.completions.Completions, + "create", + lambda self, **kwargs: completion, + ) + client = OpenAI(posthog_client=fake_ph, api_key="k") + client.chat.completions.create( + model="gpt-4o-audio-preview", messages=[{"role": "user", "content": "hi"}] + ) + audio_block = props(fake_ph)["$ai_output_choices"][0]["content"][0] + assert audio_block["data"] == "[base64 audio redacted]" + assert audio_block["transcript"] == "hi" + + +def test_openai_input_audio_is_redacted(fake_ph, monkeypatch): + openai_mod = pytest.importorskip("openai") + from openai.types import CompletionUsage + from openai.types.chat import ChatCompletion, ChatCompletionMessage + from openai.types.chat.chat_completion import Choice + from posthog.ai.openai import OpenAI + + completion = ChatCompletion( + id="c", + created=1, + model="gpt-4o-audio-preview", + object="chat.completion", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(role="assistant", content="ok"), + ) + ], + usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2), + ) + monkeypatch.setattr( + openai_mod.resources.chat.completions.Completions, + "create", + lambda self, **kwargs: completion, + ) + client = OpenAI(posthog_client=fake_ph, api_key="k") + client.chat.completions.create( + model="gpt-4o-audio-preview", + messages=[ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": {"data": PNG_B64 * 3, "format": "wav"}, + }, + ], + }, + ], + ) + part = props(fake_ph)["$ai_input"][0]["content"][0] + assert part["input_audio"]["data"] == "[base64 audio redacted]" + + +def test_anthropic_nested_tool_result_image_redacted(fake_ph, monkeypatch): + anthropic_mod = pytest.importorskip("anthropic") + from anthropic.types import Message, TextBlock, Usage + from posthog.ai.anthropic import Anthropic + + msg = Message( + id="m", + content=[TextBlock(text="ok", type="text")], + model="claude-sonnet-4-5", + role="assistant", + stop_reason="end_turn", + stop_sequence=None, + type="message", + usage=Usage(input_tokens=1, output_tokens=1), + ) + monkeypatch.setattr( + anthropic_mod.resources.messages.Messages, "create", lambda self, **kwargs: msg + ) + client = Anthropic(posthog_client=fake_ph, api_key="k") + client.messages.create( + model="claude-sonnet-4-5", + max_tokens=10, + messages=[ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_B64, + }, + }, + ], + }, + ], + }, + ], + ) + nested = props(fake_ph)["$ai_input"][0]["content"][0]["content"][0] + assert nested["source"]["data"] == PLACEHOLDER + + +@pytest.mark.asyncio +async def test_async_streaming_chat_message_object_is_formatted(fake_ph, monkeypatch): + openai_mod = pytest.importorskip("openai") + from openai.types.chat import ChatCompletionMessage + from posthog.ai.openai import AsyncOpenAI + + async def mock_create(self, **kwargs): + async def empty_stream(): + return + yield # pragma: no cover + + return empty_stream() + + monkeypatch.setattr( + openai_mod.resources.chat.completions.AsyncCompletions, "create", mock_create + ) + client = AsyncOpenAI(posthog_client=fake_ph, api_key="k") + stream = await client.chat.completions.create( + model="gpt-4o-mini", + messages=[ChatCompletionMessage(role="assistant", content="prior turn")], + stream=True, + ) + async for _chunk in stream: + pass + + assert props(fake_ph)["$ai_input"] == [ + {"role": "assistant", "content": "prior turn"} + ] + + +def test_openai_responses_image_generation_call_result_is_redacted( + fake_ph, monkeypatch +): + pytest.importorskip("openai") + from openai.resources.responses import Responses + from openai.types.responses import Response + from openai.types.responses.response_output_item import ImageGenerationCall + from posthog.ai.openai import OpenAI + + response = Response( + id="r1", + created_at=1, + model="gpt-4o", + object="response", + output=[ + ImageGenerationCall( + id="ig_1", + result=PNG_B64, + status="completed", + type="image_generation_call", + ) + ], + parallel_tool_calls=True, + tool_choice="auto", + tools=[], + ) + + monkeypatch.setattr(Responses, "create", lambda self, **kwargs: response) + client = OpenAI(posthog_client=fake_ph, api_key="k") + client.responses.create( + model="gpt-4o", input=[{"role": "user", "content": "draw a cat"}] + ) + + output_item = props(fake_ph)["$ai_output_choices"][0]["content"][0] + assert output_item["type"] == "image_generation_call" + assert output_item["result"] == PLACEHOLDER + + +def _langchain_image_response(): + from langchain_core.messages import AIMessage + from langchain_core.outputs import ChatGeneration, LLMResult + + return LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content=[ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_B64, + }, + }, + {"type": "text", "text": "here is the image"}, + ] + ) + ) + ] + ], + llm_output={}, + ) + + +def test_langchain_callback_output_choices_image_is_redacted(fake_ph): + pytest.importorskip("langchain_core") + from langchain_core.messages import HumanMessage + from uuid import uuid4 + + from posthog.ai.langchain.callbacks import CallbackHandler + + cb = CallbackHandler(client=fake_ph) + run_id = uuid4() + cb.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content="describe this image")]], + run_id=run_id, + ) + cb.on_llm_end(_langchain_image_response(), run_id=run_id) + + content = props(fake_ph)["$ai_output_choices"][0]["content"] + assert content[0]["source"]["data"] == PLACEHOLDER + assert content[1] == {"type": "text", "text": "here is the image"} + + +def test_langchain_callback_output_choices_image_passthrough_when_multimodal_enabled( + fake_ph, +): + pytest.importorskip("langchain_core") + from langchain_core.messages import HumanMessage + from uuid import uuid4 + + from posthog.ai.langchain.callbacks import CallbackHandler + + fake_ph._enable_multimodal_capture = True + + cb = CallbackHandler(client=fake_ph) + run_id = uuid4() + cb.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content="describe this image")]], + run_id=run_id, + ) + cb.on_llm_end(_langchain_image_response(), run_id=run_id) + + content = props(fake_ph)["$ai_output_choices"][0]["content"] + assert content[0]["source"]["data"] == PNG_B64 + assert content[1] == {"type": "text", "text": "here is the image"} diff --git a/posthog/test/ai/test_media.py b/posthog/test/ai/test_media.py new file mode 100644 index 00000000..6db92543 --- /dev/null +++ b/posthog/test/ai/test_media.py @@ -0,0 +1,80 @@ +import base64 +import json +from dataclasses import dataclass + +from posthog.ai.media import ( + bytes_to_base64, + ensure_serializable, + normalize_part_keys, + to_plain, +) +from posthog.ai.utils import finalize_ai_content + + +def test_to_plain_pydantic_strips_none(): + from google.genai import types + + part = types.Part(text="hi") + plain = to_plain(part) + assert plain == {"text": "hi"} + + +def test_to_plain_dict_strips_none(): + assert to_plain( + {"text": None, "inline_data": {"mime_type": "image/png", "data": "AA"}} + ) == {"inline_data": {"mime_type": "image/png", "data": "AA"}} + + +def test_to_plain_dataclass(): + @dataclass + class Block: + text: str + + assert to_plain(Block(text="x")) == {"text": "x"} + + +def test_to_plain_passthrough(): + assert to_plain("str") == "str" + assert to_plain(3) == 3 + + +def test_bytes_to_base64(): + assert bytes_to_base64(b"\x00\x01") == base64.b64encode(b"\x00\x01").decode() + + +def test_normalize_part_keys_camel_to_snake(): + out = normalize_part_keys({"inlineData": {"mimeType": "image/png", "data": "AA"}}) + assert out == {"inline_data": {"mime_type": "image/png", "data": "AA"}} + + +def test_normalize_part_keys_file_data(): + out = normalize_part_keys({"fileData": {"fileUri": "u", "mimeType": "video/mp4"}}) + assert out == {"file_data": {"file_uri": "u", "mime_type": "video/mp4"}} + + +def test_normalize_part_keys_untouched_snake(): + d = {"inline_data": {"mime_type": "image/png", "data": "AA"}} + assert normalize_part_keys(d) == d + + +def test_ensure_serializable_bytes_leaf_preserves_structure_for_redaction(): + # The claude-agent-sdk tool-span path is finalize_ai_content(ensure_serializable(...)). + # A single bytes leaf must not collapse the whole dict to a repr string, or + # the base64 under a strong key escapes finalize's redaction. + long_b64 = base64.b64encode(b"\xff" * 300).decode() + out = finalize_ai_content( + ensure_serializable({"data": long_b64, "blob": b"\x00\x01"}) + ) + assert isinstance(out, dict) + assert long_b64 not in repr(out) + assert out["data"] == "[base64 image redacted]" + + +def test_ensure_serializable_coerces_non_string_keys(): + out = ensure_serializable({1: "x"}) + assert out == {"1": "x"} + json.dumps(out) + + +def test_ensure_serializable_coerces_tuple_keys(): + json.dumps(ensure_serializable({("a", "b"): "x"})) diff --git a/posthog/test/ai/test_sanitization.py b/posthog/test/ai/test_sanitization.py index d67ec472..d538e181 100644 --- a/posthog/test/ai/test_sanitization.py +++ b/posthog/test/ai/test_sanitization.py @@ -1,38 +1,29 @@ +import base64 import types import unittest from unittest import mock from posthog.ai.sanitization import ( redact_base64_data_url, + redact_media, sanitize_openai, sanitize_openai_response, sanitize_anthropic, sanitize_gemini, sanitize_langchain, - is_base64_data_url, - is_raw_base64, REDACTED_IMAGE_PLACEHOLDER, ) +# Raw-base64 redaction requires a strong-context key and len >= 200, so +# fixtures standing in for "some base64" must be realistically long. +LONG_RAW_BASE64 = base64.b64encode(b"\x00" * 200).decode() + class TestSanitization(unittest.TestCase): def setUp(self): self.sample_base64_image = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." self.sample_base64_png = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..." self.regular_url = "https://example.com/image.jpg" - self.raw_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUl==" - - def test_is_base64_data_url(self): - self.assertTrue(is_base64_data_url(self.sample_base64_image)) - self.assertTrue(is_base64_data_url(self.sample_base64_png)) - self.assertFalse(is_base64_data_url(self.regular_url)) - self.assertFalse(is_base64_data_url("regular text")) - - def test_is_raw_base64(self): - self.assertTrue(is_raw_base64(self.raw_base64)) - self.assertFalse(is_raw_base64("short")) - self.assertFalse(is_raw_base64(self.regular_url)) - self.assertFalse(is_raw_base64("/path/to/file")) def test_redact_base64_data_url(self): self.assertEqual( @@ -134,7 +125,7 @@ def test_sanitize_anthropic(self): "source": { "type": "base64", "media_type": "image/jpeg", - "data": "base64data", + "data": LONG_RAW_BASE64, }, }, ], @@ -158,7 +149,7 @@ def test_sanitize_gemini(self): { "inline_data": { "mime_type": "image/jpeg", - "data": "base64data", + "data": LONG_RAW_BASE64, } }, ] @@ -200,7 +191,7 @@ def test_sanitize_langchain_anthropic_style(self): "content": [ { "type": "image", - "source": {"data": "base64data"}, + "source": {"data": LONG_RAW_BASE64}, } ], } @@ -256,8 +247,9 @@ def test_sanitize_with_data_url_format(self): self.assertEqual(result[0]["content"][0]["data"], REDACTED_IMAGE_PLACEHOLDER) def test_sanitize_with_raw_base64(self): - # Test that raw base64 strings (without data URL prefix) are detected - raw_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUl==" + # Raw base64 strings (no data URL prefix) are redacted under a strong + # key once they clear the len >= 200 floor. + raw_base64 = LONG_RAW_BASE64 # Test with Anthropic format anthropic_data = [ @@ -352,42 +344,9 @@ def test_sanitize_handles_single_message(self): ) -class TestAudioRedaction(unittest.TestCase): - def test_openai_audio_redacted(self): - input_data = [ - { - "role": "assistant", - "content": [ - {"type": "audio", "data": "base64audiodata", "id": "audio_123"} - ], - } - ] - - result = sanitize_openai(input_data) - self.assertEqual(result[0]["content"][0]["data"], REDACTED_IMAGE_PLACEHOLDER) - self.assertEqual(result[0]["content"][0]["id"], "audio_123") - - def test_gemini_audio_redacted(self): - input_data = [ - { - "parts": [ - { - "inline_data": { - "mime_type": "audio/L16;codec=pcm;rate=24000", - "data": "base64audiodata", - } - } - ] - } - ] - - result = sanitize_gemini(input_data) - self.assertEqual( - result[0]["parts"][0]["inline_data"]["data"], REDACTED_IMAGE_PLACEHOLDER - ) - - class TestClientMultimodalPassthrough(unittest.TestCase): + """Multimodal passthrough is gated on the client's _enable_multimodal_capture.""" + def setUp(self): self.image = "data:image/jpeg;base64," + "A" * 64 self.openai_input = [ @@ -468,5 +427,248 @@ def test_unspecced_mock_client_still_redacts(self): ) +class TestAudioRedaction(unittest.TestCase): + def _client(self, enabled): + return types.SimpleNamespace(_enable_multimodal_capture=enabled) + + def test_openai_audio_redacted_by_default(self): + input_data = [ + { + "role": "assistant", + "content": [ + {"type": "audio", "data": LONG_RAW_BASE64, "id": "audio_123"} + ], + } + ] + + result = sanitize_openai(input_data) + # The placeholder says "audio" (from the sibling "type" field) instead of + # the generic "image" default, since there's no mime_type/format sibling. + self.assertEqual(result[0]["content"][0]["data"], "[base64 audio redacted]") + self.assertEqual(result[0]["content"][0]["id"], "audio_123") + + def test_openai_audio_preserved_with_flag(self): + input_data = [ + { + "role": "assistant", + "content": [ + {"type": "audio", "data": "base64audiodata", "id": "audio_123"} + ], + } + ] + + result = sanitize_openai(input_data, ph_client=self._client(True)) + self.assertEqual(result[0]["content"][0]["data"], "base64audiodata") + + def test_gemini_audio_redacted_by_default(self): + input_data = [ + { + "parts": [ + { + "inline_data": { + "mime_type": "audio/L16;codec=pcm;rate=24000", + "data": LONG_RAW_BASE64, + } + } + ] + } + ] + + result = sanitize_gemini(input_data) + # Placeholder text reflects the sibling mime type instead of always "image". + self.assertEqual( + result[0]["parts"][0]["inline_data"]["data"], "[base64 audio redacted]" + ) + + def test_gemini_audio_preserved_with_flag(self): + input_data = [ + { + "parts": [ + { + "inline_data": { + "mime_type": "audio/L16;codec=pcm;rate=24000", + "data": "base64audiodata", + } + } + ] + } + ] + + result = sanitize_gemini(input_data, ph_client=self._client(True)) + self.assertEqual( + result[0]["parts"][0]["inline_data"]["data"], "base64audiodata" + ) + + +PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 400).decode() + + +class TestMediaRedactor: + def test_data_url_redacted_anywhere(self): + out = redact_media({"note": f"data:image/png;base64,{PNG_B64}"}) + assert out == {"note": REDACTED_IMAGE_PLACEHOLDER} + + def test_anthropic_image_source_redacted(self): + out = redact_media( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_B64, + }, + } + ], + } + ] + ) + assert out[0]["content"][0]["source"]["data"] == REDACTED_IMAGE_PLACEHOLDER + + def test_nested_tool_result_image_redacted(self): + out = redact_media( + [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_B64, + }, + } + ], + } + ], + } + ] + ) + assert ( + out[0]["content"][0]["content"][0]["source"]["data"] + == REDACTED_IMAGE_PLACEHOLDER + ) + + def test_document_pdf_redacted(self): + out = redact_media( + [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": PNG_B64, + }, + } + ] + ) + assert out[0]["source"]["data"] == "[base64 file redacted]" + + def test_input_audio_redacted(self): + out = redact_media( + [{"type": "input_audio", "input_audio": {"data": PNG_B64, "format": "wav"}}] + ) + assert out[0]["input_audio"]["data"] == "[base64 audio redacted]" + + def test_gemini_inline_data_redacted(self): + out = redact_media( + { + "type": "image", + "inline_data": {"mime_type": "image/png", "data": PNG_B64}, + } + ) + assert out["inline_data"]["data"] == REDACTED_IMAGE_PLACEHOLDER + + def test_bytes_redacted_by_default(self): + out = redact_media( + {"inline_data": {"mime_type": "video/mp4", "data": b"\x00" * 300}} + ) + assert out["inline_data"]["data"] == "[base64 video redacted]" + + def test_bytes_base64d_in_passthrough(self): + client = types.SimpleNamespace(_enable_multimodal_capture=True) + raw = b"\x00\x01\x02" + out = redact_media( + {"inline_data": {"mime_type": "video/mp4", "data": raw}}, ph_client=client + ) + assert out["inline_data"]["data"] == base64.b64encode(raw).decode() + + def test_passthrough_leaves_strings(self): + client = types.SimpleNamespace(_enable_multimodal_capture=True) + val = {"inline_data": {"mime_type": "image/png", "data": PNG_B64}} + assert redact_media(val, ph_client=client) == val + + def test_shared_reference_not_nulled(self): + img = { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}, + } + out = redact_media( + [{"role": "user", "content": [img]}, {"role": "user", "content": [img]}] + ) + assert out[1]["content"][0] is not None + assert out[1]["content"][0]["source"]["data"] == REDACTED_IMAGE_PLACEHOLDER + + def test_cycle_does_not_hang(self): + a: dict = {"x": 1} + a["self"] = a + out = redact_media(a) + assert out["x"] == 1 + + def test_short_token_in_result_key_not_redacted(self): + tok = "A1b2C3d4" * 10 # 80 chars, plausible hash/JWT-ish token + out = redact_media({"type": "tool_result", "result": tok}) + assert out["result"] == tok + + def test_long_raw_base64_in_data_key_redacted(self): + long_b64 = base64.b64encode(b"\xff" * 600).decode() # 800 chars + out = redact_media({"data": long_b64, "mime_type": "image/png"}) + assert out["data"] == REDACTED_IMAGE_PLACEHOLDER + + def test_long_string_weak_context_untouched(self): + s = "A" * 800 + assert redact_media({"note": s}) == {"note": s} + + def test_max_string_len_truncates_leaves(self): + out = redact_media({"content": "x" * 100}, max_string_len=10) + assert out["content"] == "x" * 10 + "... [truncated]" + + def test_placeholder_idempotent(self): + v = {"data": REDACTED_IMAGE_PLACEHOLDER, "mime_type": "image/png"} + assert redact_media(v) == v + + def test_raw_base64_under_image_url_url_redacted(self): + long_b64 = base64.b64encode(b"\xff" * 300).decode() + out = redact_media({"type": "image_url", "image_url": {"url": long_b64}}) + assert out["image_url"]["url"] == REDACTED_IMAGE_PLACEHOLDER + + def test_https_url_under_image_url_url_untouched(self): + url = "https://example.com/" + "a" * 300 + out = redact_media({"type": "image_url", "image_url": {"url": url}}) + assert out["image_url"]["url"] == url + + def test_data_url_under_image_url_url_still_redacted(self): + data_url = f"data:image/png;base64,{PNG_B64}" + out = redact_media({"image_url": {"url": data_url}}) + assert out["image_url"]["url"] == REDACTED_IMAGE_PLACEHOLDER + + +def test_sanitize_messages_importable_from_utils(): + """Back-compat: sanitize_messages was previously only in utils, now re-exported.""" + from posthog.ai.utils import sanitize_messages + + data_url = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." + result = sanitize_messages([{"role": "user", "content": data_url}]) + assert result[0]["content"] == REDACTED_IMAGE_PLACEHOLDER + + if __name__ == "__main__": unittest.main() diff --git a/posthog/test/ai/utils.py b/posthog/test/ai/utils.py index 99fc3f02..e1ad48f7 100644 --- a/posthog/test/ai/utils.py +++ b/posthog/test/ai/utils.py @@ -1,6 +1,46 @@ """Shared test helpers for the AI wrapper test suites.""" +def make_response_usage( + input_tokens: int, + output_tokens: int, + total_tokens: int, + cached_tokens: int = 0, + reasoning_tokens: int = 0, +): + """Build an ``openai.types.responses.ResponseUsage`` across SDK versions. + + openai has repeatedly added required fields to ``InputTokensDetails`` / + ``OutputTokensDetails`` (e.g. ``cache_write_tokens`` in 2.45). Rather than + hardcode a fixed field set that breaks on every such bump, this fills any + required field it doesn't recognize with 0. + """ + from openai.types.responses import ResponseUsage + from openai.types.responses.response_usage import ( + InputTokensDetails, + OutputTokensDetails, + ) + + def build(model_cls, known): + values = dict(known) + for name, field in model_cls.model_fields.items(): + if name not in values and field.is_required(): + values[name] = 0 + return model_cls(**values) + + return ResponseUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + input_tokens_details=build( + InputTokensDetails, {"cached_tokens": cached_tokens} + ), + output_tokens_details=build( + OutputTokensDetails, {"reasoning_tokens": reasoning_tokens} + ), + ) + + class RecordingAsyncStream: """Mock provider async stream that is iterable and records when closed. diff --git a/pyproject.toml b/pyproject.toml index fd45a767..e7434a15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ test = [ "pytest-timeout", "pytest-asyncio", "django>=5.2.15,<6.0", - "openai>=2.0", + "openai-agents>=0.18", "anthropic>=0.72", "langgraph>=1.0", "langgraph-checkpoint>=4.1.1", diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 848b3075..68e3dace 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -47,7 +47,6 @@ alias posthog.ai.anthropic.anthropic.handle_anthropic_content_block_start -> pos alias posthog.ai.anthropic.anthropic.handle_anthropic_text_delta -> posthog.ai.anthropic.anthropic_converter.handle_anthropic_text_delta alias posthog.ai.anthropic.anthropic.handle_anthropic_tool_delta -> posthog.ai.anthropic.anthropic_converter.handle_anthropic_tool_delta alias posthog.ai.anthropic.anthropic.merge_usage_stats -> posthog.ai.utils.merge_usage_stats -alias posthog.ai.anthropic.anthropic.sanitize_anthropic -> posthog.ai.sanitization.sanitize_anthropic alias posthog.ai.anthropic.anthropic.setup -> posthog.setup alias posthog.ai.anthropic.anthropic_async.AsyncStreamWrapper -> posthog.ai.stream.AsyncStreamWrapper alias posthog.ai.anthropic.anthropic_async.PostHogClient -> posthog.client.Client @@ -61,7 +60,6 @@ alias posthog.ai.anthropic.anthropic_async.handle_anthropic_content_block_start alias posthog.ai.anthropic.anthropic_async.handle_anthropic_text_delta -> posthog.ai.anthropic.anthropic_converter.handle_anthropic_text_delta alias posthog.ai.anthropic.anthropic_async.handle_anthropic_tool_delta -> posthog.ai.anthropic.anthropic_converter.handle_anthropic_tool_delta alias posthog.ai.anthropic.anthropic_async.merge_usage_stats -> posthog.ai.utils.merge_usage_stats -alias posthog.ai.anthropic.anthropic_async.sanitize_anthropic -> posthog.ai.sanitization.sanitize_anthropic alias posthog.ai.anthropic.anthropic_async.setup -> posthog.setup alias posthog.ai.anthropic.anthropic_converter.FormattedContentItem -> posthog.ai.types.FormattedContentItem alias posthog.ai.anthropic.anthropic_converter.FormattedFunctionCall -> posthog.ai.types.FormattedFunctionCall @@ -71,6 +69,7 @@ alias posthog.ai.anthropic.anthropic_converter.StreamingContentBlock -> posthog. alias posthog.ai.anthropic.anthropic_converter.TokenUsage -> posthog.ai.types.TokenUsage alias posthog.ai.anthropic.anthropic_converter.ToolInProgress -> posthog.ai.types.ToolInProgress alias posthog.ai.anthropic.anthropic_converter.serialize_raw_usage -> posthog.ai.utils.serialize_raw_usage +alias posthog.ai.anthropic.anthropic_converter.to_plain -> posthog.ai.media.to_plain alias posthog.ai.anthropic.anthropic_providers.AsyncWrappedMessages -> posthog.ai.anthropic.anthropic_async.AsyncWrappedMessages alias posthog.ai.anthropic.anthropic_providers.PostHogClient -> posthog.client.Client alias posthog.ai.anthropic.anthropic_providers.WrappedMessages -> posthog.ai.anthropic.anthropic.WrappedMessages @@ -83,7 +82,14 @@ alias posthog.ai.claude_agent_sdk.PostHogClaudeAgentProcessor -> posthog.ai.clau alias posthog.ai.claude_agent_sdk.PostHogClaudeSDKClient -> posthog.ai.claude_agent_sdk.client.PostHogClaudeSDKClient alias posthog.ai.claude_agent_sdk.client.Client -> posthog.client.Client alias posthog.ai.claude_agent_sdk.client.PostHogClaudeAgentProcessor -> posthog.ai.claude_agent_sdk.processor.PostHogClaudeAgentProcessor +alias posthog.ai.claude_agent_sdk.client.format_assistant_blocks -> posthog.ai.claude_agent_sdk.formatting.format_assistant_blocks +alias posthog.ai.claude_agent_sdk.client.format_tool_result_content -> posthog.ai.claude_agent_sdk.formatting.format_tool_result_content +alias posthog.ai.claude_agent_sdk.formatting.redact_media -> posthog.ai.sanitization.redact_media +alias posthog.ai.claude_agent_sdk.formatting.to_plain -> posthog.ai.media.to_plain alias posthog.ai.claude_agent_sdk.processor.Client -> posthog.client.Client +alias posthog.ai.claude_agent_sdk.processor.finalize_ai_content -> posthog.ai.utils.finalize_ai_content +alias posthog.ai.claude_agent_sdk.processor.format_assistant_blocks -> posthog.ai.claude_agent_sdk.formatting.format_assistant_blocks +alias posthog.ai.claude_agent_sdk.processor.format_tool_result_content -> posthog.ai.claude_agent_sdk.formatting.format_tool_result_content alias posthog.ai.claude_agent_sdk.processor.setup -> posthog.setup alias posthog.ai.gemini.AsyncClient -> posthog.ai.gemini.gemini_async.AsyncClient alias posthog.ai.gemini.Client -> posthog.ai.gemini.gemini.Client @@ -99,10 +105,10 @@ alias posthog.ai.gemini.gemini.extract_gemini_content_from_chunk -> posthog.ai.g alias posthog.ai.gemini.gemini.extract_gemini_embedding_token_count -> posthog.ai.gemini.gemini_converter.extract_gemini_embedding_token_count alias posthog.ai.gemini.gemini.extract_gemini_stop_reason_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason_from_chunk alias posthog.ai.gemini.gemini.extract_gemini_usage_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk +alias posthog.ai.gemini.gemini.finalize_ai_content -> posthog.ai.utils.finalize_ai_content alias posthog.ai.gemini.gemini.format_gemini_streaming_output -> posthog.ai.gemini.gemini_converter.format_gemini_streaming_output alias posthog.ai.gemini.gemini.merge_system_prompt -> posthog.ai.utils.merge_system_prompt alias posthog.ai.gemini.gemini.merge_usage_stats -> posthog.ai.utils.merge_usage_stats -alias posthog.ai.gemini.gemini.sanitize_gemini -> posthog.ai.sanitization.sanitize_gemini alias posthog.ai.gemini.gemini.setup -> posthog.setup alias posthog.ai.gemini.gemini.with_privacy_mode -> posthog.ai.utils.with_privacy_mode alias posthog.ai.gemini.gemini_async.AsyncStreamWrapper -> posthog.ai.stream.AsyncStreamWrapper @@ -115,20 +121,23 @@ alias posthog.ai.gemini.gemini_async.extract_gemini_content_from_chunk -> postho alias posthog.ai.gemini.gemini_async.extract_gemini_embedding_token_count -> posthog.ai.gemini.gemini_converter.extract_gemini_embedding_token_count alias posthog.ai.gemini.gemini_async.extract_gemini_stop_reason_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason_from_chunk alias posthog.ai.gemini.gemini_async.extract_gemini_usage_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk +alias posthog.ai.gemini.gemini_async.finalize_ai_content -> posthog.ai.utils.finalize_ai_content alias posthog.ai.gemini.gemini_async.format_gemini_streaming_output -> posthog.ai.gemini.gemini_converter.format_gemini_streaming_output alias posthog.ai.gemini.gemini_async.merge_system_prompt -> posthog.ai.utils.merge_system_prompt alias posthog.ai.gemini.gemini_async.merge_usage_stats -> posthog.ai.utils.merge_usage_stats -alias posthog.ai.gemini.gemini_async.sanitize_gemini -> posthog.ai.sanitization.sanitize_gemini alias posthog.ai.gemini.gemini_async.setup -> posthog.setup alias posthog.ai.gemini.gemini_async.with_privacy_mode -> posthog.ai.utils.with_privacy_mode alias posthog.ai.gemini.gemini_converter.FormattedContentItem -> posthog.ai.types.FormattedContentItem alias posthog.ai.gemini.gemini_converter.FormattedMessage -> posthog.ai.types.FormattedMessage alias posthog.ai.gemini.gemini_converter.TokenUsage -> posthog.ai.types.TokenUsage +alias posthog.ai.gemini.gemini_converter.bytes_to_base64 -> posthog.ai.media.bytes_to_base64 +alias posthog.ai.gemini.gemini_converter.normalize_part_keys -> posthog.ai.media.normalize_part_keys alias posthog.ai.gemini.gemini_converter.serialize_raw_usage -> posthog.ai.utils.serialize_raw_usage +alias posthog.ai.gemini.gemini_converter.to_plain -> posthog.ai.media.to_plain alias posthog.ai.langchain.CallbackHandler -> posthog.ai.langchain.callbacks.CallbackHandler alias posthog.ai.langchain.callbacks.Client -> posthog.client.Client +alias posthog.ai.langchain.callbacks.finalize_ai_content -> posthog.ai.utils.finalize_ai_content alias posthog.ai.langchain.callbacks.get_model_params -> posthog.ai.utils.get_model_params -alias posthog.ai.langchain.callbacks.sanitize_langchain -> posthog.ai.sanitization.sanitize_langchain alias posthog.ai.langchain.callbacks.setup -> posthog.setup alias posthog.ai.langchain.callbacks.warn_if_posthog_ai_gateway -> posthog.ai.gateway.warn_if_posthog_ai_gateway alias posthog.ai.langchain.callbacks.with_privacy_mode -> posthog.ai.utils.with_privacy_mode @@ -148,9 +157,8 @@ alias posthog.ai.openai.openai.extract_available_tool_calls -> posthog.ai.utils. alias posthog.ai.openai.openai.extract_openai_content_from_chunk -> posthog.ai.openai.openai_converter.extract_openai_content_from_chunk alias posthog.ai.openai.openai.extract_openai_tool_calls_from_chunk -> posthog.ai.openai.openai_converter.extract_openai_tool_calls_from_chunk alias posthog.ai.openai.openai.extract_openai_usage_from_chunk -> posthog.ai.openai.openai_converter.extract_openai_usage_from_chunk +alias posthog.ai.openai.openai.finalize_ai_content -> posthog.ai.utils.finalize_ai_content alias posthog.ai.openai.openai.merge_usage_stats -> posthog.ai.utils.merge_usage_stats -alias posthog.ai.openai.openai.sanitize_openai -> posthog.ai.sanitization.sanitize_openai -alias posthog.ai.openai.openai.sanitize_openai_response -> posthog.ai.sanitization.sanitize_openai_response alias posthog.ai.openai.openai.setup -> posthog.setup alias posthog.ai.openai.openai.with_privacy_mode -> posthog.ai.utils.with_privacy_mode alias posthog.ai.openai.openai_async.AsyncStreamWrapper -> posthog.ai.stream.AsyncStreamWrapper @@ -162,11 +170,11 @@ alias posthog.ai.openai.openai_async.extract_available_tool_calls -> posthog.ai. alias posthog.ai.openai.openai_async.extract_openai_content_from_chunk -> posthog.ai.openai.openai_converter.extract_openai_content_from_chunk alias posthog.ai.openai.openai_async.extract_openai_tool_calls_from_chunk -> posthog.ai.openai.openai_converter.extract_openai_tool_calls_from_chunk alias posthog.ai.openai.openai_async.extract_openai_usage_from_chunk -> posthog.ai.openai.openai_converter.extract_openai_usage_from_chunk +alias posthog.ai.openai.openai_async.finalize_ai_content -> posthog.ai.utils.finalize_ai_content +alias posthog.ai.openai.openai_async.format_openai_streaming_input -> posthog.ai.openai.openai_converter.format_openai_streaming_input alias posthog.ai.openai.openai_async.format_openai_streaming_output -> posthog.ai.openai.openai_converter.format_openai_streaming_output alias posthog.ai.openai.openai_async.get_model_params -> posthog.ai.utils.get_model_params alias posthog.ai.openai.openai_async.merge_usage_stats -> posthog.ai.utils.merge_usage_stats -alias posthog.ai.openai.openai_async.sanitize_openai -> posthog.ai.sanitization.sanitize_openai -alias posthog.ai.openai.openai_async.sanitize_openai_response -> posthog.ai.sanitization.sanitize_openai_response alias posthog.ai.openai.openai_async.setup -> posthog.setup alias posthog.ai.openai.openai_async.with_privacy_mode -> posthog.ai.utils.with_privacy_mode alias posthog.ai.openai.openai_converter.FormattedContentItem -> posthog.ai.types.FormattedContentItem @@ -176,6 +184,7 @@ alias posthog.ai.openai.openai_converter.FormattedMessage -> posthog.ai.types.Fo alias posthog.ai.openai.openai_converter.FormattedTextContent -> posthog.ai.types.FormattedTextContent alias posthog.ai.openai.openai_converter.TokenUsage -> posthog.ai.types.TokenUsage alias posthog.ai.openai.openai_converter.serialize_raw_usage -> posthog.ai.utils.serialize_raw_usage +alias posthog.ai.openai.openai_converter.to_plain -> posthog.ai.media.to_plain alias posthog.ai.openai.openai_providers.AsyncWrappedBeta -> posthog.ai.openai.openai_async.WrappedBeta alias posthog.ai.openai.openai_providers.AsyncWrappedChat -> posthog.ai.openai.openai_async.WrappedChat alias posthog.ai.openai.openai_providers.AsyncWrappedEmbeddings -> posthog.ai.openai.openai_async.WrappedEmbeddings @@ -188,6 +197,7 @@ alias posthog.ai.openai.openai_providers.WrappedResponses -> posthog.ai.openai.o alias posthog.ai.openai.openai_providers.setup -> posthog.setup alias posthog.ai.openai_agents.PostHogTracingProcessor -> posthog.ai.openai_agents.processor.PostHogTracingProcessor alias posthog.ai.openai_agents.processor.Client -> posthog.client.Client +alias posthog.ai.openai_agents.processor.finalize_ai_content -> posthog.ai.utils.finalize_ai_content alias posthog.ai.openai_agents.processor.setup -> posthog.setup alias posthog.ai.otel.PostHogSpanProcessor -> posthog.ai.otel.processor.PostHogSpanProcessor alias posthog.ai.otel.PostHogTraceExporter -> posthog.ai.otel.exporter.PostHogTraceExporter @@ -200,6 +210,7 @@ alias posthog.ai.otel.processor.is_ai_span -> posthog.ai.otel.spans.is_ai_span alias posthog.ai.otel.processor.warn_if_posthog_ai_gateway_otel_attributes -> posthog.ai.gateway.warn_if_posthog_ai_gateway_otel_attributes alias posthog.ai.prompts.USER_AGENT -> posthog.request.USER_AGENT alias posthog.ai.prompts.remove_trailing_slash -> posthog.utils.remove_trailing_slash +alias posthog.ai.sanitization.bytes_to_base64 -> posthog.ai.media.bytes_to_base64 alias posthog.ai.utils.FormattedMessage -> posthog.ai.types.FormattedMessage alias posthog.ai.utils.PostHogClient -> posthog.client.Client alias posthog.ai.utils.StreamingEventData -> posthog.ai.types.StreamingEventData @@ -208,10 +219,8 @@ alias posthog.ai.utils.contexts -> posthog.contexts alias posthog.ai.utils.get_tags -> posthog.get_tags alias posthog.ai.utils.identify_context -> posthog.identify_context alias posthog.ai.utils.new_context -> posthog.new_context -alias posthog.ai.utils.sanitize_anthropic -> posthog.ai.sanitization.sanitize_anthropic -alias posthog.ai.utils.sanitize_gemini -> posthog.ai.sanitization.sanitize_gemini -alias posthog.ai.utils.sanitize_langchain -> posthog.ai.sanitization.sanitize_langchain -alias posthog.ai.utils.sanitize_openai -> posthog.ai.sanitization.sanitize_openai +alias posthog.ai.utils.redact_media -> posthog.ai.sanitization.redact_media +alias posthog.ai.utils.sanitize_messages -> posthog.ai.sanitization.sanitize_messages alias posthog.ai.utils.tag -> posthog.tag alias posthog.ai.utils.warn_if_posthog_ai_gateway -> posthog.ai.gateway.warn_if_posthog_ai_gateway alias posthog.args.FeatureFlagEvaluations -> posthog.feature_flag_evaluations.FeatureFlagEvaluations @@ -425,16 +434,24 @@ attribute posthog.ai.types.FormattedFunctionCall.id: Optional[str] attribute posthog.ai.types.FormattedFunctionCall.type: str attribute posthog.ai.types.FormattedImageContent.image: str attribute posthog.ai.types.FormattedImageContent.type: str +attribute posthog.ai.types.FormattedMessage.audio: NotRequired[Dict[str, Any]] attribute posthog.ai.types.FormattedMessage.content: Union[str, List[FormattedContentItem], Any] +attribute posthog.ai.types.FormattedMessage.name: NotRequired[str] +attribute posthog.ai.types.FormattedMessage.refusal: NotRequired[str] attribute posthog.ai.types.FormattedMessage.role: str +attribute posthog.ai.types.FormattedMessage.tool_call_id: NotRequired[str] +attribute posthog.ai.types.FormattedMessage.tool_calls: NotRequired[List[Dict[str, Any]]] attribute posthog.ai.types.FormattedTextContent.text: str attribute posthog.ai.types.FormattedTextContent.type: str attribute posthog.ai.types.ProviderResponse.error: Optional[str] attribute posthog.ai.types.ProviderResponse.messages: List[FormattedMessage] attribute posthog.ai.types.ProviderResponse.usage: TokenUsage +attribute posthog.ai.types.StreamingContentBlock.data: Optional[str] attribute posthog.ai.types.StreamingContentBlock.function: Optional[Dict[str, Any]] attribute posthog.ai.types.StreamingContentBlock.id: Optional[str] +attribute posthog.ai.types.StreamingContentBlock.signature: Optional[str] attribute posthog.ai.types.StreamingContentBlock.text: Optional[str] +attribute posthog.ai.types.StreamingContentBlock.thinking: Optional[str] attribute posthog.ai.types.StreamingContentBlock.type: str attribute posthog.ai.types.StreamingEventData.base_url: str attribute posthog.ai.types.StreamingEventData.distinct_id: Optional[str] @@ -925,12 +942,14 @@ function posthog.ai.anthropic.anthropic_converter.format_anthropic_streaming_out function posthog.ai.anthropic.anthropic_converter.handle_anthropic_content_block_start(event: Any) -> Tuple[Optional[StreamingContentBlock], Optional[ToolInProgress]] function posthog.ai.anthropic.anthropic_converter.handle_anthropic_text_delta(event: Any, current_block: Optional[StreamingContentBlock]) -> Optional[str] function posthog.ai.anthropic.anthropic_converter.handle_anthropic_tool_delta(event: Any, content_blocks: List[StreamingContentBlock], tools_in_progress: Dict[str, ToolInProgress]) -> None +function posthog.ai.claude_agent_sdk.formatting.format_assistant_blocks(blocks: Any) -> List[Dict[str, Any]] +function posthog.ai.claude_agent_sdk.formatting.format_tool_result_content(block: Any, ph_client: Any = None) -> Any function posthog.ai.claude_agent_sdk.instrument(client: Optional[Client] = None, distinct_id: Optional[Union[str, Callable[[ResultMessage], Optional[str]]]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None, properties: Optional[Dict[str, Any]] = None) -> PostHogClaudeAgentProcessor function posthog.ai.claude_agent_sdk.query(*, prompt: Any, options: Optional[ClaudeAgentOptions] = None, transport: Any = None, posthog_client: Optional[Client] = None, posthog_distinct_id: Optional[Union[str, Callable[[ResultMessage], Optional[str]]]] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: bool = False, posthog_groups: Optional[Dict[str, Any]] = None) function posthog.ai.gateway.is_posthog_ai_gateway_url(base_url: Any) -> bool function posthog.ai.gateway.warn_if_posthog_ai_gateway(base_url: Any) -> None function posthog.ai.gateway.warn_if_posthog_ai_gateway_otel_attributes(attributes: Optional[Mapping[str, Any]]) -> None -function posthog.ai.gemini.gemini_converter.extract_gemini_content_from_chunk(chunk: Any) -> Optional[Dict[str, Any]] +function posthog.ai.gemini.gemini_converter.extract_gemini_content_from_chunk(chunk: Any) -> Optional[List[FormattedContentItem]] function posthog.ai.gemini.gemini_converter.extract_gemini_embedding_token_count(response) -> int function posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason(response: Any) -> Optional[str] function posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason_from_chunk(chunk: Any) -> Optional[str] @@ -943,8 +962,12 @@ function posthog.ai.gemini.gemini_converter.format_gemini_input(contents: Any) - function posthog.ai.gemini.gemini_converter.format_gemini_input_with_system(contents: Any, config: Any = None) -> List[FormattedMessage] function posthog.ai.gemini.gemini_converter.format_gemini_response(response: Any) -> List[FormattedMessage] function posthog.ai.gemini.gemini_converter.format_gemini_streaming_output(accumulated_content: Union[str, List[Any]]) -> List[FormattedMessage] +function posthog.ai.media.bytes_to_base64(data: bytes) -> str +function posthog.ai.media.ensure_serializable(obj: Any) -> Any +function posthog.ai.media.normalize_part_keys(d: dict) -> dict +function posthog.ai.media.to_plain(obj: Any) -> Any function posthog.ai.openai.openai_converter.accumulate_openai_tool_calls(accumulated_tool_calls: Dict[int, Dict[str, Any]], chunk_tool_calls: List[Dict[str, Any]]) -> None -function posthog.ai.openai.openai_converter.extract_openai_content_from_chunk(chunk: Any, provider_type: str = 'chat') -> Optional[str] +function posthog.ai.openai.openai_converter.extract_openai_content_from_chunk(chunk: Any, provider_type: str = 'chat') -> Optional[Any] function posthog.ai.openai.openai_converter.extract_openai_stop_reason(response: Any) -> Optional[str] function posthog.ai.openai.openai_converter.extract_openai_tool_calls_from_chunk(chunk: Any) -> Optional[List[Dict[str, Any]]] function posthog.ai.openai.openai_converter.extract_openai_tools(kwargs: Dict[str, Any]) -> Optional[Any] @@ -960,33 +983,25 @@ function posthog.ai.openai.wrapper_utils.reset_fallback_warnings() -> None function posthog.ai.openai.wrapper_utils.warn_on_fallback(wrapper_name: str, name: str) -> None function posthog.ai.openai_agents.instrument(client: Optional[Client] = None, distinct_id: Optional[Union[str, Callable[[Trace], Optional[str]]]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None, properties: Optional[Dict[str, Any]] = None) -> PostHogTracingProcessor function posthog.ai.otel.spans.is_ai_span(span: ReadableSpan) -> bool -function posthog.ai.sanitization.is_base64_data_url(text: str) -> bool -function posthog.ai.sanitization.is_raw_base64(text: str) -> bool -function posthog.ai.sanitization.is_valid_url(text: str) -> bool -function posthog.ai.sanitization.process_gemini_item(item: Any) -> Any -function posthog.ai.sanitization.process_messages(messages: Any, transform_content_func) -> Any function posthog.ai.sanitization.redact_base64_data_url(value: Any) -> Any +function posthog.ai.sanitization.redact_media(value: Any, max_string_len: Optional[int] = None, ph_client: Any = None) -> Any function posthog.ai.sanitization.sanitize_anthropic(data: Any, ph_client: Any = None) -> Any -function posthog.ai.sanitization.sanitize_anthropic_image(item: Any) -> Any function posthog.ai.sanitization.sanitize_gemini(data: Any, ph_client: Any = None) -> Any -function posthog.ai.sanitization.sanitize_gemini_part(part: Any) -> Any function posthog.ai.sanitization.sanitize_langchain(data: Any, ph_client: Any = None) -> Any -function posthog.ai.sanitization.sanitize_langchain_image(item: Any) -> Any +function posthog.ai.sanitization.sanitize_messages(data: Any, provider: Optional[str] = None, ph_client: Any = None) -> Any function posthog.ai.sanitization.sanitize_openai(data: Any, ph_client: Any = None) -> Any -function posthog.ai.sanitization.sanitize_openai_image(item: Any) -> Any function posthog.ai.sanitization.sanitize_openai_response(data: Any, ph_client: Any = None) -> Any -function posthog.ai.sanitization.sanitize_openai_response_image(item: Any) -> Any function posthog.ai.utils.call_llm_and_track_usage(posthog_distinct_id: Optional[str], ph_client: PostHogClient, provider: str, posthog_trace_id: Optional[str], posthog_properties: Optional[Dict[str, Any]], posthog_privacy_mode: bool, posthog_groups: Optional[Dict[str, Any]], base_url: str, call_method: Callable[..., Any], **kwargs: Any) -> Any function posthog.ai.utils.call_llm_and_track_usage_async(posthog_distinct_id: Optional[str], ph_client: PostHogClient, provider: str, posthog_trace_id: Optional[str], posthog_properties: Optional[Dict[str, Any]], posthog_privacy_mode: bool, posthog_groups: Optional[Dict[str, Any]], base_url: str, call_async_method: Callable[..., Any], **kwargs: Any) -> Any function posthog.ai.utils.capture_streaming_event(ph_client: PostHogClient, event_data: StreamingEventData) function posthog.ai.utils.extract_available_tool_calls(provider: str, kwargs: Dict[str, Any]) function posthog.ai.utils.extract_stop_reason(response: Any, provider: str) -> Optional[str] +function posthog.ai.utils.finalize_ai_content(value: Any, ph_client: Any = None) -> Any function posthog.ai.utils.format_response(response, provider: str) function posthog.ai.utils.get_model_params(kwargs: Dict[str, Any]) -> Dict[str, Any] function posthog.ai.utils.get_usage(response, provider: str) -> TokenUsage function posthog.ai.utils.merge_system_prompt(kwargs: Dict[str, Any], provider: str) -> List[FormattedMessage] function posthog.ai.utils.merge_usage_stats(target: TokenUsage, source: TokenUsage, mode: str = 'incremental') -> None -function posthog.ai.utils.sanitize_messages(data: Any, provider: str, ph_client: Any = None) -> Any function posthog.ai.utils.serialize_raw_usage(raw_usage: Any) -> Optional[Dict[str, Any]] function posthog.ai.utils.with_privacy_mode(ph_client: PostHogClient, privacy_mode: bool, value: Any) function posthog.alias(previous_id: str, distinct_id: str, timestamp: Optional[datetime.datetime] = None, uuid: Optional[str] = None, disable_geoip: Optional[bool] = None) -> Optional[str] @@ -1316,6 +1331,7 @@ module posthog.ai.anthropic.anthropic_converter module posthog.ai.anthropic.anthropic_providers module posthog.ai.claude_agent_sdk module posthog.ai.claude_agent_sdk.client +module posthog.ai.claude_agent_sdk.formatting module posthog.ai.claude_agent_sdk.processor module posthog.ai.gateway module posthog.ai.gemini @@ -1324,6 +1340,7 @@ module posthog.ai.gemini.gemini_async module posthog.ai.gemini.gemini_converter module posthog.ai.langchain module posthog.ai.langchain.callbacks +module posthog.ai.media module posthog.ai.openai module posthog.ai.openai.openai module posthog.ai.openai.openai_async diff --git a/uv.lock b/uv.lock index 113e453b..dfd3a809 100644 --- a/uv.lock +++ b/uv.lock @@ -181,7 +181,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "typing-extensions", marker = "python_full_version != '3.11.*'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6a/68/fb4fb78c9eac59d5e819108a57664737f855c5a8e9b76aec1738bb137f9e/asgiref-3.9.0.tar.gz", hash = "sha256:3dd2556d0f08c4fab8a010d9ab05ef8c34565f6bf32381d17505f7ca5b273767", size = 36772, upload-time = "2025-07-03T13:25:01.491Z" } wheels = [ @@ -2071,7 +2071,7 @@ wheels = [ [[package]] name = "openai" -version = "2.44.0" +version = "2.46.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2083,9 +2083,27 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/f5/7c7cb955305cb41f7f3c5fd7e0e38bf6bbf2658468863d4b7b868a5cb8df/openai-2.44.0.tar.gz", hash = "sha256:68a5a5ffad82b8ff7d451c437529fb64f7c3b8123aaf0c021966a882d9e3947d", size = 988753, upload-time = "2026-06-24T20:56:02.293Z" } +sdist = { url = "https://files.pythonhosted.org/packages/af/ac/f725c4efbda8657d02be684607e5a2e5ce362e4790fdbcbdfb7c15018647/openai-2.46.0.tar.gz", hash = "sha256:0421e0735ac41451cad894af4cddf0435bfbf8cbc538ac0e15b3c062f2ddc06a", size = 1114628, upload-time = "2026-07-17T02:48:06.05Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/f4/561ed79fd94876160018a5e75254cfcb9b0e62d4dded9dcb20072e86d623/openai-2.44.0-py3-none-any.whl", hash = "sha256:0a2a3ab2e29aeda368700f662ff9ba0f9df17ba4c54577a64e08b8115a3cc0ad", size = 1366216, upload-time = "2026-06-24T20:55:58.882Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7b/206238ebcb50b235942b1c66dba4974776f2057402a8d91c399be587d66a/openai-2.46.0-py3-none-any.whl", hash = "sha256:672381db55efb3a1e2610f29304c130cccdd0b319bace4d492b2443cb64c1e7c", size = 1637556, upload-time = "2026-07-17T02:48:03.695Z" }, +] + +[[package]] +name = "openai-agents" +version = "0.18.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffelib" }, + { name = "mcp" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/3f/b1162cad8720fafc9cf658d6896027385967f5006adfb92ae0dab2b54a70/openai_agents-0.18.3.tar.gz", hash = "sha256:e637f5f5a50692ccbedb0e4f7f2e4f8e2facfcddd41142f35faf90c89b700fc3", size = 5577652, upload-time = "2026-07-17T03:40:25.208Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/7c/08b9929a15131081898e38c5612d44ee293851d5c80f31ca1153efe82143/openai_agents-0.18.3-py3-none-any.whl", hash = "sha256:c6ed971fdeb34d39a9931787bd3960c1e84dc5d7345705794cc5cab8a1158d07", size = 880799, upload-time = "2026-07-17T03:40:23.163Z" }, ] [[package]] @@ -2535,7 +2553,7 @@ test = [ { name = "langgraph" }, { name = "langgraph-checkpoint" }, { name = "mcp" }, - { name = "openai" }, + { name = "openai-agents" }, { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, { name = "parameterized" }, @@ -2582,7 +2600,7 @@ requires-dist = [ { name = "mcp", marker = "extra == 'test'", specifier = ">=1.28.1" }, { name = "mypy", marker = "extra == 'dev'" }, { name = "mypy-baseline", marker = "extra == 'dev'" }, - { name = "openai", marker = "extra == 'test'", specifier = ">=2.0" }, + { name = "openai-agents", marker = "extra == 'test'", specifier = ">=0.18" }, { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.20.0" }, { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'test'", specifier = ">=1.20.0" }, { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.20.0" }, @@ -3443,8 +3461,8 @@ name = "secretstorage" version = "3.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "jeepney", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/a4/f48c9d79cb507ed1373477dbceaba7401fd8a23af63b837fa61f1dcd3691/SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77", size = 19739, upload-time = "2022-08-13T16:22:46.976Z" } wheels = [