From 8c89b82e5736040346a2b01493fe28556af77fd8 Mon Sep 17 00:00:00 2001 From: Arseny Boykov Date: Fri, 17 Jul 2026 16:16:25 +0400 Subject: [PATCH 1/2] chore: bump protocol to 1.50.1 for simulation metrics pooling stats --- livekit-protocol/protocol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/livekit-protocol/protocol b/livekit-protocol/protocol index 74296228..13270fa5 160000 --- a/livekit-protocol/protocol +++ b/livekit-protocol/protocol @@ -1 +1 @@ -Subproject commit 74296228107d852bafdc3e0ca9cc72643150be43 +Subproject commit 13270fa5980f931b8a3bc26bd86fce954b3c4b65 From e05c6df96e30c300add985464696284a7646107b Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:40:27 +0000 Subject: [PATCH 2/2] generated protobuf --- .../livekit/protocol/agent_simulation.py | 132 +++++++++--------- .../livekit/protocol/agent_simulation.pyi | 30 ++-- 2 files changed, 88 insertions(+), 74 deletions(-) diff --git a/livekit-protocol/livekit/protocol/agent_simulation.py b/livekit-protocol/livekit/protocol/agent_simulation.py index 2264bd84..96d1e236 100644 --- a/livekit-protocol/livekit/protocol/agent_simulation.py +++ b/livekit-protocol/livekit/protocol/agent_simulation.py @@ -28,7 +28,7 @@ from . import models as _models_ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1elivekit_agent_simulation.proto\x12\x07livekit\x1a!agent/livekit_agent_session.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19livekit_cloud_agent.proto\x1a\x14livekit_models.proto\"\xea\x02\n\x14SimulationRunSummary\x12\x0e\n\x06passed\x18\x01 \x01(\x05\x12\x0e\n\x06\x66\x61iled\x18\x02 \x01(\x05\x12\x12\n\ngoing_well\x18\x03 \x01(\t\x12\x12\n\nto_improve\x18\x04 \x01(\t\x12\x33\n\x06issues\x18\x05 \x03(\x0b\x32#.livekit.SimulationRunSummary.Issue\x12\x44\n\x0c\x63hat_history\x18\x06 \x03(\x0b\x32..livekit.SimulationRunSummary.ChatHistoryEntry\x1aN\n\x10\x43hatHistoryEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.livekit.agent.ChatContext:\x02\x38\x01\x1a?\n\x05Issue\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x12\n\nsuggestion\x18\x02 \x01(\t\x12\r\n\x05label\x18\x03 \x01(\t\"\xa8/\n\rSimulationRun\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\nproject_id\x18\x02 \x01(\t\x12-\n\x06status\x18\x03 \x01(\x0e\x32\x1d.livekit.SimulationRun.Status\x12\x19\n\x11\x61gent_description\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12.\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12(\n\x04jobs\x18\x07 \x03(\x0b\x32\x1a.livekit.SimulationRun.Job\x12\x12\n\nagent_name\x18\t \x01(\t\x12.\n\x0escenario_group\x18\n \x01(\x0b\x32\x16.livekit.ScenarioGroup\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\tjob_count\x18\x0c \x01(\x05\x12\x14\n\x0cpassed_count\x18\r \x01(\x05\x12\x14\n\x0c\x66\x61iled_count\x18\x0e \x01(\x05\x12\x17\n\x0fnum_simulations\x18\x0f \x01(\x05\x12+\n\x05usage\x18\x10 \x01(\x0b\x32\x1c.livekit.SimulationRun.Usage\x12\x13\n\x0b\x63oncurrency\x18\x11 \x01(\x05\x12%\n\x04mode\x18\x12 \x01(\x0e\x32\x17.livekit.SimulationMode\x12\x32\n\x07metrics\x18\x13 \x01(\x0b\x32!.livekit.SimulationRun.RunMetrics\x12\x14\n\x0csummary_zstd\x18\x14 \x01(\x0c\x1a\xbe\x04\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x31\n\x06status\x18\x02 \x01(\x0e\x32!.livekit.SimulationRun.Job.Status\x12\x14\n\x0cinstructions\x18\x03 \x01(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x05 \x01(\t\x12\r\n\x05label\x18\x06 \x01(\t\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x11\n\troom_name\x18\x08 \x01(\t\x12.\n\nstarted_at\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07room_id\x18\x0c \x01(\t\x12/\n\x05usage\x18\r \x01(\x0b\x32 .livekit.SimulationRun.Job.Usage\x12\x32\n\x07metrics\x18\x0e \x01(\x0b\x32!.livekit.SimulationRun.JobMetrics\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"o\n\x06Status\x12\x12\n\x0eSTATUS_PENDING\x10\x00\x12\x12\n\x0eSTATUS_RUNNING\x10\x01\x12\x14\n\x10STATUS_COMPLETED\x10\x02\x12\x11\n\rSTATUS_FAILED\x10\x03\x12\x14\n\x10STATUS_CANCELLED\x10\x04J\x04\x08\t\x10\n\x1a\xf5\x18\n\nJobMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1c\n\x0ftask_completion\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.Conversation\x12>\n\tsimulator\x18\x08 \x01(\x0b\x32+.livekit.SimulationRun.JobMetrics.Simulator\x12\x35\n\x05turns\x18\t \x03(\x0b\x32&.livekit.SimulationRun.JobMetrics.Turn\x12\x13\n\x0bjudge_model\x18\n \x01(\t\x12\x19\n\x11\x61udio_judge_model\x18\x0b \x01(\t\x12\x1a\n\x12has_remote_session\x18\x0c \x01(\x08\x12&\n\x02t0\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\xe3\x02\n\x03STT\x12\x10\n\x03wer\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x12\n\x05words\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x18\n\x0bword_errors\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x10\n\x03\x63\x65r\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1b\n\x0ekeyterm_recall\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x1d\n\x10keyterms_uttered\x18\x06 \x01(\rH\x05\x88\x01\x01\x12 \n\x13keyterms_recognized\x18\x07 \x01(\rH\x06\x88\x01\x01\x12%\n\x18transcription_latency_ms\x18\x08 \x01(\rH\x07\x88\x01\x01\x42\x06\n\x04_werB\x08\n\x06_wordsB\x0e\n\x0c_word_errorsB\x06\n\x04_cerB\x11\n\x0f_keyterm_recallB\x13\n\x11_keyterms_utteredB\x16\n\x14_keyterms_recognizedB\x1b\n\x19_transcription_latency_ms\x1a\xb5\x01\n\x03LLM\x12\x14\n\x07ttft_ms\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x14\n\x07ttfs_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x1e\n\x11tokens_per_second\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x63onciseness_score\x18\x04 \x01(\x02H\x03\x88\x01\x01\x42\n\n\x08_ttft_msB\n\n\x08_ttfs_msB\x14\n\x12_tokens_per_secondB\x14\n\x12_conciseness_score\x1a\x81\x02\n\x03TTS\x12\x14\n\x07ttfa_ms\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x14\n\x07ttfb_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x10\n\x03wer\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1c\n\x0fspeech_rate_wpm\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1e\n\x11naturalness_score\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x1e\n\x11\x65nunciation_score\x18\x06 \x01(\x02H\x05\x88\x01\x01\x42\n\n\x08_ttfa_msB\n\n\x08_ttfb_msB\x06\n\x04_werB\x12\n\x10_speech_rate_wpmB\x14\n\x12_naturalness_scoreB\x14\n\x12_enunciation_score\x1a\xc6\x07\n\x0c\x43onversation\x12\x1e\n\x11turn_taking_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12$\n\x17response_latency_p50_ms\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12$\n\x17response_latency_p95_ms\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12$\n\x17response_latency_p99_ms\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16\x61gent_yield_latency_ms\x18\x05 \x01(\rH\x04\x88\x01\x01\x12$\n\x17\x65ot_misprediction_count\x18\x06 \x01(\rH\x05\x88\x01\x01\x12\x1a\n\roverlap_ratio\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11overlap_speech_ms\x18\x08 \x01(\rH\x07\x88\x01\x01\x12\x1c\n\x0ftotal_speech_ms\x18\t \x01(\rH\x08\x88\x01\x01\x12\x1d\n\x10silence_total_ms\x18\n \x01(\rH\t\x88\x01\x01\x12\"\n\x15\x61wkward_silence_count\x18\x0b \x01(\rH\n\x88\x01\x01\x12%\n\x18unanswered_persona_turns\x18\x0c \x01(\rH\x0b\x88\x01\x01\x12%\n\x18\x66\x61lse_interruption_count\x18\r \x01(\rH\x0c\x88\x01\x01\x12\x31\n$false_interruption_unrecovered_count\x18\x0e \x01(\rH\r\x88\x01\x01\x12*\n\x1d\x61gent_reported_e2e_latency_ms\x18\x0f \x01(\rH\x0e\x88\x01\x01\x42\x14\n\x12_turn_taking_scoreB\x1a\n\x18_response_latency_p50_msB\x1a\n\x18_response_latency_p95_msB\x1a\n\x18_response_latency_p99_msB\x19\n\x17_agent_yield_latency_msB\x1a\n\x18_eot_misprediction_countB\x10\n\x0e_overlap_ratioB\x14\n\x12_overlap_speech_msB\x12\n\x10_total_speech_msB\x13\n\x11_silence_total_msB\x18\n\x16_awkward_silence_countB\x1b\n\x19_unanswered_persona_turnsB\x1b\n\x19_false_interruption_countB\'\n%_false_interruption_unrecovered_countB \n\x1e_agent_reported_e2e_latency_ms\x1au\n\tSimulator\x12\x1e\n\x11\x65\x61rly_termination\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x1d\n\x10late_termination\x18\x02 \x01(\x08H\x01\x88\x01\x01\x42\x14\n\x12_early_terminationB\x13\n\x11_late_termination\x1a\xa1\x05\n\x04Turn\x12\r\n\x05index\x18\x01 \x01(\r\x12%\n\x04role\x18\x02 \x01(\x0e\x32\x17.livekit.agent.ChatRole\x12\x15\n\x08start_ms\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x13\n\x06\x65nd_ms\x18\x04 \x01(\rH\x01\x88\x01\x01\x12 \n\x13response_latency_ms\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12#\n\x16transcription_delay_ms\x18\x06 \x01(\rH\x03\x88\x01\x01\x12\x18\n\x0bllm_ttft_ms\x18\x07 \x01(\rH\x04\x88\x01\x01\x12\x18\n\x0bllm_ttfs_ms\x18\x08 \x01(\rH\x05\x88\x01\x01\x12\x18\n\x0btts_ttfa_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x18\n\x0btts_ttfb_ms\x18\n \x01(\rH\x07\x88\x01\x01\x12*\n\x1d\x61gent_reported_e2e_latency_ms\x18\x0b \x01(\rH\x08\x88\x01\x01\x12\x1e\n\x11\x63onciseness_score\x18\x0c \x01(\x02H\t\x88\x01\x01\x12\x1e\n\x11naturalness_score\x18\r \x01(\x02H\n\x88\x01\x01\x12\x1e\n\x11\x65nunciation_score\x18\x0e \x01(\x02H\x0b\x88\x01\x01\x12\r\n\x05\x66lags\x18\x0f \x03(\tB\x0b\n\t_start_msB\t\n\x07_end_msB\x16\n\x14_response_latency_msB\x19\n\x17_transcription_delay_msB\x0e\n\x0c_llm_ttft_msB\x0e\n\x0c_llm_ttfs_msB\x0e\n\x0c_tts_ttfa_msB\x0e\n\x0c_tts_ttfb_msB \n\x1e_agent_reported_e2e_latency_msB\x14\n\x12_conciseness_scoreB\x14\n\x12_naturalness_scoreB\x14\n\x12_enunciation_scoreB\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x12\n\x10_task_completion\x1a\xd1\x03\n\nRunMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1f\n\x12scenario_pass_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.Conversation\x12\x12\n\njobs_total\x18\t \x01(\r\x12\x13\n\x0bjobs_failed\x18\n \x01(\r\x12\x1c\n\x14jobs_simulator_fault\x18\x0b \x01(\rB\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x15\n\x13_scenario_pass_rate\x1a\xfd\x02\n\x06\x43reate\x1a\x8c\x02\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\nagent_name\x18\x02 \x01(\t\x12\x17\n\x0fnum_simulations\x18\x04 \x01(\x05\x12\x0e\n\x06region\x18\x06 \x01(\t\x12\x33\n\x0escenario_group\x18\x07 \x01(\x0b\x32\x16.livekit.ScenarioGroupH\x00\x88\x01\x01\x12\x18\n\x0b\x63oncurrency\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12%\n\x04mode\x18\t \x01(\x0e\x32\x17.livekit.SimulationModeB\x11\n\x0f_scenario_groupB\x0e\n\x0c_concurrencyJ\x04\x08\x03\x10\x04R\x11\x61gent_description\x1a\x64\n\x08Response\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12=\n\x16presigned_post_request\x18\x02 \x01(\x0b\x32\x1d.livekit.PresignedPostRequest\x1at\n\x13\x43onfirmSourceUpload\x1aQ\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x12\x17\n\x0f\x63ode_entrypoint\x18\x03 \x01(\t\x1a\n\n\x08Response\x1ap\n\x03Get\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a/\n\x08Response\x12#\n\x03run\x18\x01 \x01(\x0b\x32\x16.livekit.SimulationRun\x1a\x8c\x02\n\x04List\x1a\x9e\x01\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x32\n\x06status\x18\x02 \x01(\x0e\x32\x1d.livekit.SimulationRun.StatusH\x00\x88\x01\x01\x12\x31\n\npage_token\x18\x03 \x01(\x0b\x32\x18.livekit.TokenPaginationH\x01\x88\x01\x01\x42\t\n\x07_statusB\r\n\x0b_page_token\x1a\x63\n\x08Response\x12$\n\x04runs\x18\x01 \x03(\x0b\x32\x16.livekit.SimulationRun\x12\x31\n\x0fnext_page_token\x18\x02 \x01(\x0b\x32\x18.livekit.TokenPagination\x1aN\n\x06\x43\x61ncel\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a\n\n\x08Response\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"\xa5\x01\n\x06Status\x12\x19\n\x15STATUS_PENDING_UPLOAD\x10\x00\x12\x15\n\x11STATUS_GENERATING\x10\x01\x12\x12\n\x0eSTATUS_RUNNING\x10\x02\x12\x16\n\x12STATUS_SUMMARIZING\x10\x03\x12\x14\n\x10STATUS_COMPLETED\x10\x04\x12\x11\n\rSTATUS_FAILED\x10\x05\x12\x14\n\x10STATUS_CANCELLED\x10\x06J\x04\x08\x08\x10\tR\x07summary\"\xbc\x02\n\x08Scenario\x12\r\n\x05label\x18\x01 \x01(\t\x12\x14\n\x0cinstructions\x18\x02 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x03 \x01(\t\x12)\n\x04tags\x18\x04 \x03(\x0b\x32\x1b.livekit.Scenario.TagsEntry\x12\x10\n\x08userdata\x18\x05 \x01(\t\x1a\x84\x01\n\x11\x43reateFromSession\x1a>\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x0f\n\x07room_id\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x1a/\n\x08Response\x12#\n\x08scenario\x18\x01 \x01(\x0b\x32\x11.livekit.Scenario\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"C\n\rScenarioGroup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\tscenarios\x18\x02 \x03(\x0b\x32\x11.livekit.Scenario\"\x8b\x01\n\x12SimulationDispatch\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12#\n\x08scenario\x18\x03 \x01(\x0b\x32\x11.livekit.Scenario\x12%\n\x04mode\x18\x04 \x01(\x0e\x32\x17.livekit.SimulationMode*f\n\x0eSimulationMode\x12\x1f\n\x1bSIMULATION_MODE_UNSPECIFIED\x10\x00\x12\x18\n\x14SIMULATION_MODE_TEXT\x10\x01\x12\x19\n\x15SIMULATION_MODE_AUDIO\x10\x02\x32\x9e\x05\n\x0f\x41gentSimulation\x12\x64\n\x13\x43reateSimulationRun\x12%.livekit.SimulationRun.Create.Request\x1a&.livekit.SimulationRun.Create.Response\x12\x88\x01\n\x1d\x43onfirmSimulationSourceUpload\x12\x32.livekit.SimulationRun.ConfirmSourceUpload.Request\x1a\x33.livekit.SimulationRun.ConfirmSourceUpload.Response\x12[\n\x10GetSimulationRun\x12\".livekit.SimulationRun.Get.Request\x1a#.livekit.SimulationRun.Get.Response\x12_\n\x12ListSimulationRuns\x12#.livekit.SimulationRun.List.Request\x1a$.livekit.SimulationRun.List.Response\x12\x64\n\x13\x43\x61ncelSimulationRun\x12%.livekit.SimulationRun.Cancel.Request\x1a&.livekit.SimulationRun.Cancel.Response\x12v\n\x19\x43reateScenarioFromSession\x12+.livekit.Scenario.CreateFromSession.Request\x1a,.livekit.Scenario.CreateFromSession.ResponseBFZ#github.com/livekit/protocol/livekit\xaa\x02\rLiveKit.Proto\xea\x02\x0eLiveKit::Protob\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1elivekit_agent_simulation.proto\x12\x07livekit\x1a!agent/livekit_agent_session.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19livekit_cloud_agent.proto\x1a\x14livekit_models.proto\"\xea\x02\n\x14SimulationRunSummary\x12\x0e\n\x06passed\x18\x01 \x01(\x05\x12\x0e\n\x06\x66\x61iled\x18\x02 \x01(\x05\x12\x12\n\ngoing_well\x18\x03 \x01(\t\x12\x12\n\nto_improve\x18\x04 \x01(\t\x12\x33\n\x06issues\x18\x05 \x03(\x0b\x32#.livekit.SimulationRunSummary.Issue\x12\x44\n\x0c\x63hat_history\x18\x06 \x03(\x0b\x32..livekit.SimulationRunSummary.ChatHistoryEntry\x1aN\n\x10\x43hatHistoryEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.livekit.agent.ChatContext:\x02\x38\x01\x1a?\n\x05Issue\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x12\n\nsuggestion\x18\x02 \x01(\t\x12\r\n\x05label\x18\x03 \x01(\t\"\x9c\x31\n\rSimulationRun\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\nproject_id\x18\x02 \x01(\t\x12-\n\x06status\x18\x03 \x01(\x0e\x32\x1d.livekit.SimulationRun.Status\x12\x19\n\x11\x61gent_description\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12.\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12(\n\x04jobs\x18\x07 \x03(\x0b\x32\x1a.livekit.SimulationRun.Job\x12\x12\n\nagent_name\x18\t \x01(\t\x12.\n\x0escenario_group\x18\n \x01(\x0b\x32\x16.livekit.ScenarioGroup\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\tjob_count\x18\x0c \x01(\x05\x12\x14\n\x0cpassed_count\x18\r \x01(\x05\x12\x14\n\x0c\x66\x61iled_count\x18\x0e \x01(\x05\x12\x17\n\x0fnum_simulations\x18\x0f \x01(\x05\x12+\n\x05usage\x18\x10 \x01(\x0b\x32\x1c.livekit.SimulationRun.Usage\x12\x13\n\x0b\x63oncurrency\x18\x11 \x01(\x05\x12%\n\x04mode\x18\x12 \x01(\x0e\x32\x17.livekit.SimulationMode\x12\x32\n\x07metrics\x18\x13 \x01(\x0b\x32!.livekit.SimulationRun.RunMetrics\x12\x14\n\x0csummary_zstd\x18\x14 \x01(\x0c\x1a\xbe\x04\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x31\n\x06status\x18\x02 \x01(\x0e\x32!.livekit.SimulationRun.Job.Status\x12\x14\n\x0cinstructions\x18\x03 \x01(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x05 \x01(\t\x12\r\n\x05label\x18\x06 \x01(\t\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x11\n\troom_name\x18\x08 \x01(\t\x12.\n\nstarted_at\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07room_id\x18\x0c \x01(\t\x12/\n\x05usage\x18\r \x01(\x0b\x32 .livekit.SimulationRun.Job.Usage\x12\x32\n\x07metrics\x18\x0e \x01(\x0b\x32!.livekit.SimulationRun.JobMetrics\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"o\n\x06Status\x12\x12\n\x0eSTATUS_PENDING\x10\x00\x12\x12\n\x0eSTATUS_RUNNING\x10\x01\x12\x14\n\x10STATUS_COMPLETED\x10\x02\x12\x11\n\rSTATUS_FAILED\x10\x03\x12\x14\n\x10STATUS_CANCELLED\x10\x04J\x04\x08\t\x10\n\x1a\xe7\x1a\n\nJobMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1c\n\x0ftask_completion\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.Conversation\x12>\n\tsimulator\x18\x08 \x01(\x0b\x32+.livekit.SimulationRun.JobMetrics.Simulator\x12\x35\n\x05turns\x18\t \x03(\x0b\x32&.livekit.SimulationRun.JobMetrics.Turn\x12\x13\n\x0bjudge_model\x18\n \x01(\t\x12\x19\n\x11\x61udio_judge_model\x18\x0b \x01(\t\x12\x1a\n\x12has_remote_session\x18\x0c \x01(\x08\x12&\n\x02t0\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\xab\x03\n\x03STT\x12\x10\n\x03wer\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x12\n\x05words\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x18\n\x0bword_errors\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x10\n\x03\x63\x65r\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x12\n\x05\x63hars\x18\t \x01(\rH\x04\x88\x01\x01\x12\x18\n\x0b\x63har_errors\x18\n \x01(\rH\x05\x88\x01\x01\x12\x1b\n\x0ekeyterm_recall\x18\x05 \x01(\x02H\x06\x88\x01\x01\x12\x1d\n\x10keyterms_uttered\x18\x06 \x01(\rH\x07\x88\x01\x01\x12 \n\x13keyterms_recognized\x18\x07 \x01(\rH\x08\x88\x01\x01\x12%\n\x18transcription_latency_ms\x18\x08 \x01(\rH\t\x88\x01\x01\x42\x06\n\x04_werB\x08\n\x06_wordsB\x0e\n\x0c_word_errorsB\x06\n\x04_cerB\x08\n\x06_charsB\x0e\n\x0c_char_errorsB\x11\n\x0f_keyterm_recallB\x13\n\x11_keyterms_utteredB\x16\n\x14_keyterms_recognizedB\x1b\n\x19_transcription_latency_ms\x1a\xb5\x01\n\x03LLM\x12\x14\n\x07ttft_ms\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x14\n\x07ttfs_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x1e\n\x11tokens_per_second\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x63onciseness_score\x18\x04 \x01(\x02H\x03\x88\x01\x01\x42\n\n\x08_ttft_msB\n\n\x08_ttfs_msB\x14\n\x12_tokens_per_secondB\x14\n\x12_conciseness_score\x1a\xab\x03\n\x03TTS\x12\x14\n\x07ttfa_ms\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x14\n\x07ttfb_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x10\n\x03wer\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x12\n\x05words\x18\x07 \x01(\rH\x03\x88\x01\x01\x12\x18\n\x0bword_errors\x18\x08 \x01(\rH\x04\x88\x01\x01\x12\x10\n\x03\x63\x65r\x18\t \x01(\x02H\x05\x88\x01\x01\x12\x12\n\x05\x63hars\x18\n \x01(\rH\x06\x88\x01\x01\x12\x18\n\x0b\x63har_errors\x18\x0b \x01(\rH\x07\x88\x01\x01\x12\x1c\n\x0fspeech_rate_wpm\x18\x04 \x01(\x02H\x08\x88\x01\x01\x12\x1e\n\x11naturalness_score\x18\x05 \x01(\x02H\t\x88\x01\x01\x12\x1e\n\x11\x65nunciation_score\x18\x06 \x01(\x02H\n\x88\x01\x01\x42\n\n\x08_ttfa_msB\n\n\x08_ttfb_msB\x06\n\x04_werB\x08\n\x06_wordsB\x0e\n\x0c_word_errorsB\x06\n\x04_cerB\x08\n\x06_charsB\x0e\n\x0c_char_errorsB\x12\n\x10_speech_rate_wpmB\x14\n\x12_naturalness_scoreB\x14\n\x12_enunciation_score\x1a\xc6\x07\n\x0c\x43onversation\x12\x1e\n\x11turn_taking_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12$\n\x17response_latency_p50_ms\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12$\n\x17response_latency_p95_ms\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12$\n\x17response_latency_p99_ms\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16\x61gent_yield_latency_ms\x18\x05 \x01(\rH\x04\x88\x01\x01\x12$\n\x17\x65ot_misprediction_count\x18\x06 \x01(\rH\x05\x88\x01\x01\x12\x1a\n\roverlap_ratio\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11overlap_speech_ms\x18\x08 \x01(\rH\x07\x88\x01\x01\x12\x1c\n\x0ftotal_speech_ms\x18\t \x01(\rH\x08\x88\x01\x01\x12\x1d\n\x10silence_total_ms\x18\n \x01(\rH\t\x88\x01\x01\x12\"\n\x15\x61wkward_silence_count\x18\x0b \x01(\rH\n\x88\x01\x01\x12%\n\x18unanswered_persona_turns\x18\x0c \x01(\rH\x0b\x88\x01\x01\x12%\n\x18\x66\x61lse_interruption_count\x18\r \x01(\rH\x0c\x88\x01\x01\x12\x31\n$false_interruption_unrecovered_count\x18\x0e \x01(\rH\r\x88\x01\x01\x12*\n\x1d\x61gent_reported_e2e_latency_ms\x18\x0f \x01(\rH\x0e\x88\x01\x01\x42\x14\n\x12_turn_taking_scoreB\x1a\n\x18_response_latency_p50_msB\x1a\n\x18_response_latency_p95_msB\x1a\n\x18_response_latency_p99_msB\x19\n\x17_agent_yield_latency_msB\x1a\n\x18_eot_misprediction_countB\x10\n\x0e_overlap_ratioB\x14\n\x12_overlap_speech_msB\x12\n\x10_total_speech_msB\x13\n\x11_silence_total_msB\x18\n\x16_awkward_silence_countB\x1b\n\x19_unanswered_persona_turnsB\x1b\n\x19_false_interruption_countB\'\n%_false_interruption_unrecovered_countB \n\x1e_agent_reported_e2e_latency_ms\x1au\n\tSimulator\x12\x1e\n\x11\x65\x61rly_termination\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x1d\n\x10late_termination\x18\x02 \x01(\x08H\x01\x88\x01\x01\x42\x14\n\x12_early_terminationB\x13\n\x11_late_termination\x1a\xa1\x05\n\x04Turn\x12\r\n\x05index\x18\x01 \x01(\r\x12%\n\x04role\x18\x02 \x01(\x0e\x32\x17.livekit.agent.ChatRole\x12\x15\n\x08start_ms\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x13\n\x06\x65nd_ms\x18\x04 \x01(\rH\x01\x88\x01\x01\x12 \n\x13response_latency_ms\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12#\n\x16transcription_delay_ms\x18\x06 \x01(\rH\x03\x88\x01\x01\x12\x18\n\x0bllm_ttft_ms\x18\x07 \x01(\rH\x04\x88\x01\x01\x12\x18\n\x0bllm_ttfs_ms\x18\x08 \x01(\rH\x05\x88\x01\x01\x12\x18\n\x0btts_ttfa_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x18\n\x0btts_ttfb_ms\x18\n \x01(\rH\x07\x88\x01\x01\x12*\n\x1d\x61gent_reported_e2e_latency_ms\x18\x0b \x01(\rH\x08\x88\x01\x01\x12\x1e\n\x11\x63onciseness_score\x18\x0c \x01(\x02H\t\x88\x01\x01\x12\x1e\n\x11naturalness_score\x18\r \x01(\x02H\n\x88\x01\x01\x12\x1e\n\x11\x65nunciation_score\x18\x0e \x01(\x02H\x0b\x88\x01\x01\x12\r\n\x05\x66lags\x18\x0f \x03(\tB\x0b\n\t_start_msB\t\n\x07_end_msB\x16\n\x14_response_latency_msB\x19\n\x17_transcription_delay_msB\x0e\n\x0c_llm_ttft_msB\x0e\n\x0c_llm_ttfs_msB\x0e\n\x0c_tts_ttfa_msB\x0e\n\x0c_tts_ttfb_msB \n\x1e_agent_reported_e2e_latency_msB\x14\n\x12_conciseness_scoreB\x14\n\x12_naturalness_scoreB\x14\n\x12_enunciation_scoreB\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x12\n\x10_task_completion\x1a\xd3\x03\n\nRunMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1f\n\x12scenario_pass_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.Conversation\x12\x12\n\njobs_total\x18\t \x01(\r\x12\x15\n\rjobs_measured\x18\n \x01(\r\x12\x1c\n\x14jobs_simulator_fault\x18\x0b \x01(\rB\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x15\n\x13_scenario_pass_rate\x1a\xfd\x02\n\x06\x43reate\x1a\x8c\x02\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\nagent_name\x18\x02 \x01(\t\x12\x17\n\x0fnum_simulations\x18\x04 \x01(\x05\x12\x0e\n\x06region\x18\x06 \x01(\t\x12\x33\n\x0escenario_group\x18\x07 \x01(\x0b\x32\x16.livekit.ScenarioGroupH\x00\x88\x01\x01\x12\x18\n\x0b\x63oncurrency\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12%\n\x04mode\x18\t \x01(\x0e\x32\x17.livekit.SimulationModeB\x11\n\x0f_scenario_groupB\x0e\n\x0c_concurrencyJ\x04\x08\x03\x10\x04R\x11\x61gent_description\x1a\x64\n\x08Response\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12=\n\x16presigned_post_request\x18\x02 \x01(\x0b\x32\x1d.livekit.PresignedPostRequest\x1at\n\x13\x43onfirmSourceUpload\x1aQ\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x12\x17\n\x0f\x63ode_entrypoint\x18\x03 \x01(\t\x1a\n\n\x08Response\x1ap\n\x03Get\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a/\n\x08Response\x12#\n\x03run\x18\x01 \x01(\x0b\x32\x16.livekit.SimulationRun\x1a\x8c\x02\n\x04List\x1a\x9e\x01\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x32\n\x06status\x18\x02 \x01(\x0e\x32\x1d.livekit.SimulationRun.StatusH\x00\x88\x01\x01\x12\x31\n\npage_token\x18\x03 \x01(\x0b\x32\x18.livekit.TokenPaginationH\x01\x88\x01\x01\x42\t\n\x07_statusB\r\n\x0b_page_token\x1a\x63\n\x08Response\x12$\n\x04runs\x18\x01 \x03(\x0b\x32\x16.livekit.SimulationRun\x12\x31\n\x0fnext_page_token\x18\x02 \x01(\x0b\x32\x18.livekit.TokenPagination\x1aN\n\x06\x43\x61ncel\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a\n\n\x08Response\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"\xa5\x01\n\x06Status\x12\x19\n\x15STATUS_PENDING_UPLOAD\x10\x00\x12\x15\n\x11STATUS_GENERATING\x10\x01\x12\x12\n\x0eSTATUS_RUNNING\x10\x02\x12\x16\n\x12STATUS_SUMMARIZING\x10\x03\x12\x14\n\x10STATUS_COMPLETED\x10\x04\x12\x11\n\rSTATUS_FAILED\x10\x05\x12\x14\n\x10STATUS_CANCELLED\x10\x06J\x04\x08\x08\x10\tR\x07summary\"\xbc\x02\n\x08Scenario\x12\r\n\x05label\x18\x01 \x01(\t\x12\x14\n\x0cinstructions\x18\x02 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x03 \x01(\t\x12)\n\x04tags\x18\x04 \x03(\x0b\x32\x1b.livekit.Scenario.TagsEntry\x12\x10\n\x08userdata\x18\x05 \x01(\t\x1a\x84\x01\n\x11\x43reateFromSession\x1a>\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x0f\n\x07room_id\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x1a/\n\x08Response\x12#\n\x08scenario\x18\x01 \x01(\x0b\x32\x11.livekit.Scenario\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"C\n\rScenarioGroup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\tscenarios\x18\x02 \x03(\x0b\x32\x11.livekit.Scenario\"\x8b\x01\n\x12SimulationDispatch\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12#\n\x08scenario\x18\x03 \x01(\x0b\x32\x11.livekit.Scenario\x12%\n\x04mode\x18\x04 \x01(\x0e\x32\x17.livekit.SimulationMode*f\n\x0eSimulationMode\x12\x1f\n\x1bSIMULATION_MODE_UNSPECIFIED\x10\x00\x12\x18\n\x14SIMULATION_MODE_TEXT\x10\x01\x12\x19\n\x15SIMULATION_MODE_AUDIO\x10\x02\x32\x9e\x05\n\x0f\x41gentSimulation\x12\x64\n\x13\x43reateSimulationRun\x12%.livekit.SimulationRun.Create.Request\x1a&.livekit.SimulationRun.Create.Response\x12\x88\x01\n\x1d\x43onfirmSimulationSourceUpload\x12\x32.livekit.SimulationRun.ConfirmSourceUpload.Request\x1a\x33.livekit.SimulationRun.ConfirmSourceUpload.Response\x12[\n\x10GetSimulationRun\x12\".livekit.SimulationRun.Get.Request\x1a#.livekit.SimulationRun.Get.Response\x12_\n\x12ListSimulationRuns\x12#.livekit.SimulationRun.List.Request\x1a$.livekit.SimulationRun.List.Response\x12\x64\n\x13\x43\x61ncelSimulationRun\x12%.livekit.SimulationRun.Cancel.Request\x1a&.livekit.SimulationRun.Cancel.Response\x12v\n\x19\x43reateScenarioFromSession\x12+.livekit.Scenario.CreateFromSession.Request\x1a,.livekit.Scenario.CreateFromSession.ResponseBFZ#github.com/livekit/protocol/livekit\xaa\x02\rLiveKit.Proto\xea\x02\x0eLiveKit::Protob\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -40,8 +40,8 @@ _globals['_SIMULATIONRUNSUMMARY_CHATHISTORYENTRY']._serialized_options = b'8\001' _globals['_SCENARIO_TAGSENTRY']._loaded_options = None _globals['_SCENARIO_TAGSENTRY']._serialized_options = b'8\001' - _globals['_SIMULATIONMODE']._serialized_start=7114 - _globals['_SIMULATIONMODE']._serialized_end=7216 + _globals['_SIMULATIONMODE']._serialized_start=7358 + _globals['_SIMULATIONMODE']._serialized_end=7460 _globals['_SIMULATIONRUNSUMMARY']._serialized_start=161 _globals['_SIMULATIONRUNSUMMARY']._serialized_end=523 _globals['_SIMULATIONRUNSUMMARY_CHATHISTORYENTRY']._serialized_start=380 @@ -49,7 +49,7 @@ _globals['_SIMULATIONRUNSUMMARY_ISSUE']._serialized_start=460 _globals['_SIMULATIONRUNSUMMARY_ISSUE']._serialized_end=523 _globals['_SIMULATIONRUN']._serialized_start=526 - _globals['_SIMULATIONRUN']._serialized_end=6582 + _globals['_SIMULATIONRUN']._serialized_end=6826 _globals['_SIMULATIONRUN_JOB']._serialized_start=1136 _globals['_SIMULATIONRUN_JOB']._serialized_end=1710 _globals['_SIMULATIONRUN_JOB_USAGE']._serialized_start=1531 @@ -57,69 +57,69 @@ _globals['_SIMULATIONRUN_JOB_STATUS']._serialized_start=1593 _globals['_SIMULATIONRUN_JOB_STATUS']._serialized_end=1704 _globals['_SIMULATIONRUN_JOBMETRICS']._serialized_start=1713 - _globals['_SIMULATIONRUN_JOBMETRICS']._serialized_end=4902 + _globals['_SIMULATIONRUN_JOBMETRICS']._serialized_end=5144 _globals['_SIMULATIONRUN_JOBMETRICS_STT']._serialized_start=2279 - _globals['_SIMULATIONRUN_JOBMETRICS_STT']._serialized_end=2634 - _globals['_SIMULATIONRUN_JOBMETRICS_LLM']._serialized_start=2637 - _globals['_SIMULATIONRUN_JOBMETRICS_LLM']._serialized_end=2818 - _globals['_SIMULATIONRUN_JOBMETRICS_TTS']._serialized_start=2821 - _globals['_SIMULATIONRUN_JOBMETRICS_TTS']._serialized_end=3078 - _globals['_SIMULATIONRUN_JOBMETRICS_CONVERSATION']._serialized_start=3081 - _globals['_SIMULATIONRUN_JOBMETRICS_CONVERSATION']._serialized_end=4047 - _globals['_SIMULATIONRUN_JOBMETRICS_SIMULATOR']._serialized_start=4049 - _globals['_SIMULATIONRUN_JOBMETRICS_SIMULATOR']._serialized_end=4166 - _globals['_SIMULATIONRUN_JOBMETRICS_TURN']._serialized_start=4169 - _globals['_SIMULATIONRUN_JOBMETRICS_TURN']._serialized_end=4842 - _globals['_SIMULATIONRUN_RUNMETRICS']._serialized_start=4905 - _globals['_SIMULATIONRUN_RUNMETRICS']._serialized_end=5370 - _globals['_SIMULATIONRUN_CREATE']._serialized_start=5373 - _globals['_SIMULATIONRUN_CREATE']._serialized_end=5754 - _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_start=5384 - _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_end=5652 - _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_start=5654 - _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_end=5754 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_start=5756 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_end=5872 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_start=5779 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_end=5860 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_start=5654 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_end=5664 - _globals['_SIMULATIONRUN_GET']._serialized_start=5874 - _globals['_SIMULATIONRUN_GET']._serialized_end=5986 - _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_start=5779 - _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_end=5835 - _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_start=5939 - _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_end=5986 - _globals['_SIMULATIONRUN_LIST']._serialized_start=5989 - _globals['_SIMULATIONRUN_LIST']._serialized_end=6257 - _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_start=5998 - _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_end=6156 - _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_start=6158 - _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_end=6257 - _globals['_SIMULATIONRUN_CANCEL']._serialized_start=6259 - _globals['_SIMULATIONRUN_CANCEL']._serialized_end=6337 - _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_start=5779 - _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_end=5835 - _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_start=5654 - _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_end=5664 + _globals['_SIMULATIONRUN_JOBMETRICS_STT']._serialized_end=2706 + _globals['_SIMULATIONRUN_JOBMETRICS_LLM']._serialized_start=2709 + _globals['_SIMULATIONRUN_JOBMETRICS_LLM']._serialized_end=2890 + _globals['_SIMULATIONRUN_JOBMETRICS_TTS']._serialized_start=2893 + _globals['_SIMULATIONRUN_JOBMETRICS_TTS']._serialized_end=3320 + _globals['_SIMULATIONRUN_JOBMETRICS_CONVERSATION']._serialized_start=3323 + _globals['_SIMULATIONRUN_JOBMETRICS_CONVERSATION']._serialized_end=4289 + _globals['_SIMULATIONRUN_JOBMETRICS_SIMULATOR']._serialized_start=4291 + _globals['_SIMULATIONRUN_JOBMETRICS_SIMULATOR']._serialized_end=4408 + _globals['_SIMULATIONRUN_JOBMETRICS_TURN']._serialized_start=4411 + _globals['_SIMULATIONRUN_JOBMETRICS_TURN']._serialized_end=5084 + _globals['_SIMULATIONRUN_RUNMETRICS']._serialized_start=5147 + _globals['_SIMULATIONRUN_RUNMETRICS']._serialized_end=5614 + _globals['_SIMULATIONRUN_CREATE']._serialized_start=5617 + _globals['_SIMULATIONRUN_CREATE']._serialized_end=5998 + _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_start=5628 + _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_end=5896 + _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_start=5898 + _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_end=5998 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_start=6000 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_end=6116 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_start=6023 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_end=6104 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_start=5898 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_end=5908 + _globals['_SIMULATIONRUN_GET']._serialized_start=6118 + _globals['_SIMULATIONRUN_GET']._serialized_end=6230 + _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_start=6023 + _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_end=6079 + _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_start=6183 + _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_end=6230 + _globals['_SIMULATIONRUN_LIST']._serialized_start=6233 + _globals['_SIMULATIONRUN_LIST']._serialized_end=6501 + _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_start=6242 + _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_end=6400 + _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_start=6402 + _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_end=6501 + _globals['_SIMULATIONRUN_CANCEL']._serialized_start=6503 + _globals['_SIMULATIONRUN_CANCEL']._serialized_end=6581 + _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_start=6023 + _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_end=6079 + _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_start=5898 + _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_end=5908 _globals['_SIMULATIONRUN_USAGE']._serialized_start=1531 _globals['_SIMULATIONRUN_USAGE']._serialized_end=1591 - _globals['_SIMULATIONRUN_STATUS']._serialized_start=6402 - _globals['_SIMULATIONRUN_STATUS']._serialized_end=6567 - _globals['_SCENARIO']._serialized_start=6585 - _globals['_SCENARIO']._serialized_end=6901 - _globals['_SCENARIO_CREATEFROMSESSION']._serialized_start=6724 - _globals['_SCENARIO_CREATEFROMSESSION']._serialized_end=6856 - _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_start=6745 - _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_end=6807 - _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_start=6809 - _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_end=6856 - _globals['_SCENARIO_TAGSENTRY']._serialized_start=6858 - _globals['_SCENARIO_TAGSENTRY']._serialized_end=6901 - _globals['_SCENARIOGROUP']._serialized_start=6903 - _globals['_SCENARIOGROUP']._serialized_end=6970 - _globals['_SIMULATIONDISPATCH']._serialized_start=6973 - _globals['_SIMULATIONDISPATCH']._serialized_end=7112 - _globals['_AGENTSIMULATION']._serialized_start=7219 - _globals['_AGENTSIMULATION']._serialized_end=7889 + _globals['_SIMULATIONRUN_STATUS']._serialized_start=6646 + _globals['_SIMULATIONRUN_STATUS']._serialized_end=6811 + _globals['_SCENARIO']._serialized_start=6829 + _globals['_SCENARIO']._serialized_end=7145 + _globals['_SCENARIO_CREATEFROMSESSION']._serialized_start=6968 + _globals['_SCENARIO_CREATEFROMSESSION']._serialized_end=7100 + _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_start=6989 + _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_end=7051 + _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_start=7053 + _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_end=7100 + _globals['_SCENARIO_TAGSENTRY']._serialized_start=7102 + _globals['_SCENARIO_TAGSENTRY']._serialized_end=7145 + _globals['_SCENARIOGROUP']._serialized_start=7147 + _globals['_SCENARIOGROUP']._serialized_end=7214 + _globals['_SIMULATIONDISPATCH']._serialized_start=7217 + _globals['_SIMULATIONDISPATCH']._serialized_end=7356 + _globals['_AGENTSIMULATION']._serialized_start=7463 + _globals['_AGENTSIMULATION']._serialized_end=8133 # @@protoc_insertion_point(module_scope) diff --git a/livekit-protocol/livekit/protocol/agent_simulation.pyi b/livekit-protocol/livekit/protocol/agent_simulation.pyi index fe908a75..d54dd2c6 100644 --- a/livekit-protocol/livekit/protocol/agent_simulation.pyi +++ b/livekit-protocol/livekit/protocol/agent_simulation.pyi @@ -120,11 +120,13 @@ class SimulationRun(_message.Message): class JobMetrics(_message.Message): __slots__ = ("accuracy_score", "experience_score", "task_completion", "stt", "llm", "tts", "conversation", "simulator", "turns", "judge_model", "audio_judge_model", "has_remote_session", "t0") class STT(_message.Message): - __slots__ = ("wer", "words", "word_errors", "cer", "keyterm_recall", "keyterms_uttered", "keyterms_recognized", "transcription_latency_ms") + __slots__ = ("wer", "words", "word_errors", "cer", "chars", "char_errors", "keyterm_recall", "keyterms_uttered", "keyterms_recognized", "transcription_latency_ms") WER_FIELD_NUMBER: _ClassVar[int] WORDS_FIELD_NUMBER: _ClassVar[int] WORD_ERRORS_FIELD_NUMBER: _ClassVar[int] CER_FIELD_NUMBER: _ClassVar[int] + CHARS_FIELD_NUMBER: _ClassVar[int] + CHAR_ERRORS_FIELD_NUMBER: _ClassVar[int] KEYTERM_RECALL_FIELD_NUMBER: _ClassVar[int] KEYTERMS_UTTERED_FIELD_NUMBER: _ClassVar[int] KEYTERMS_RECOGNIZED_FIELD_NUMBER: _ClassVar[int] @@ -133,11 +135,13 @@ class SimulationRun(_message.Message): words: int word_errors: int cer: float + chars: int + char_errors: int keyterm_recall: float keyterms_uttered: int keyterms_recognized: int transcription_latency_ms: int - def __init__(self, wer: _Optional[float] = ..., words: _Optional[int] = ..., word_errors: _Optional[int] = ..., cer: _Optional[float] = ..., keyterm_recall: _Optional[float] = ..., keyterms_uttered: _Optional[int] = ..., keyterms_recognized: _Optional[int] = ..., transcription_latency_ms: _Optional[int] = ...) -> None: ... + def __init__(self, wer: _Optional[float] = ..., words: _Optional[int] = ..., word_errors: _Optional[int] = ..., cer: _Optional[float] = ..., chars: _Optional[int] = ..., char_errors: _Optional[int] = ..., keyterm_recall: _Optional[float] = ..., keyterms_uttered: _Optional[int] = ..., keyterms_recognized: _Optional[int] = ..., transcription_latency_ms: _Optional[int] = ...) -> None: ... class LLM(_message.Message): __slots__ = ("ttft_ms", "ttfs_ms", "tokens_per_second", "conciseness_score") TTFT_MS_FIELD_NUMBER: _ClassVar[int] @@ -150,20 +154,30 @@ class SimulationRun(_message.Message): conciseness_score: float def __init__(self, ttft_ms: _Optional[int] = ..., ttfs_ms: _Optional[int] = ..., tokens_per_second: _Optional[float] = ..., conciseness_score: _Optional[float] = ...) -> None: ... class TTS(_message.Message): - __slots__ = ("ttfa_ms", "ttfb_ms", "wer", "speech_rate_wpm", "naturalness_score", "enunciation_score") + __slots__ = ("ttfa_ms", "ttfb_ms", "wer", "words", "word_errors", "cer", "chars", "char_errors", "speech_rate_wpm", "naturalness_score", "enunciation_score") TTFA_MS_FIELD_NUMBER: _ClassVar[int] TTFB_MS_FIELD_NUMBER: _ClassVar[int] WER_FIELD_NUMBER: _ClassVar[int] + WORDS_FIELD_NUMBER: _ClassVar[int] + WORD_ERRORS_FIELD_NUMBER: _ClassVar[int] + CER_FIELD_NUMBER: _ClassVar[int] + CHARS_FIELD_NUMBER: _ClassVar[int] + CHAR_ERRORS_FIELD_NUMBER: _ClassVar[int] SPEECH_RATE_WPM_FIELD_NUMBER: _ClassVar[int] NATURALNESS_SCORE_FIELD_NUMBER: _ClassVar[int] ENUNCIATION_SCORE_FIELD_NUMBER: _ClassVar[int] ttfa_ms: int ttfb_ms: int wer: float + words: int + word_errors: int + cer: float + chars: int + char_errors: int speech_rate_wpm: float naturalness_score: float enunciation_score: float - def __init__(self, ttfa_ms: _Optional[int] = ..., ttfb_ms: _Optional[int] = ..., wer: _Optional[float] = ..., speech_rate_wpm: _Optional[float] = ..., naturalness_score: _Optional[float] = ..., enunciation_score: _Optional[float] = ...) -> None: ... + def __init__(self, ttfa_ms: _Optional[int] = ..., ttfb_ms: _Optional[int] = ..., wer: _Optional[float] = ..., words: _Optional[int] = ..., word_errors: _Optional[int] = ..., cer: _Optional[float] = ..., chars: _Optional[int] = ..., char_errors: _Optional[int] = ..., speech_rate_wpm: _Optional[float] = ..., naturalness_score: _Optional[float] = ..., enunciation_score: _Optional[float] = ...) -> None: ... class Conversation(_message.Message): __slots__ = ("turn_taking_score", "response_latency_p50_ms", "response_latency_p95_ms", "response_latency_p99_ms", "agent_yield_latency_ms", "eot_misprediction_count", "overlap_ratio", "overlap_speech_ms", "total_speech_ms", "silence_total_ms", "awkward_silence_count", "unanswered_persona_turns", "false_interruption_count", "false_interruption_unrecovered_count", "agent_reported_e2e_latency_ms") TURN_TAKING_SCORE_FIELD_NUMBER: _ClassVar[int] @@ -265,7 +279,7 @@ class SimulationRun(_message.Message): t0: _timestamp_pb2.Timestamp def __init__(self, accuracy_score: _Optional[float] = ..., experience_score: _Optional[float] = ..., task_completion: _Optional[float] = ..., stt: _Optional[_Union[SimulationRun.JobMetrics.STT, _Mapping]] = ..., llm: _Optional[_Union[SimulationRun.JobMetrics.LLM, _Mapping]] = ..., tts: _Optional[_Union[SimulationRun.JobMetrics.TTS, _Mapping]] = ..., conversation: _Optional[_Union[SimulationRun.JobMetrics.Conversation, _Mapping]] = ..., simulator: _Optional[_Union[SimulationRun.JobMetrics.Simulator, _Mapping]] = ..., turns: _Optional[_Iterable[_Union[SimulationRun.JobMetrics.Turn, _Mapping]]] = ..., judge_model: _Optional[str] = ..., audio_judge_model: _Optional[str] = ..., has_remote_session: bool = ..., t0: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ... class RunMetrics(_message.Message): - __slots__ = ("accuracy_score", "experience_score", "scenario_pass_rate", "stt", "llm", "tts", "conversation", "jobs_total", "jobs_failed", "jobs_simulator_fault") + __slots__ = ("accuracy_score", "experience_score", "scenario_pass_rate", "stt", "llm", "tts", "conversation", "jobs_total", "jobs_measured", "jobs_simulator_fault") ACCURACY_SCORE_FIELD_NUMBER: _ClassVar[int] EXPERIENCE_SCORE_FIELD_NUMBER: _ClassVar[int] SCENARIO_PASS_RATE_FIELD_NUMBER: _ClassVar[int] @@ -274,7 +288,7 @@ class SimulationRun(_message.Message): TTS_FIELD_NUMBER: _ClassVar[int] CONVERSATION_FIELD_NUMBER: _ClassVar[int] JOBS_TOTAL_FIELD_NUMBER: _ClassVar[int] - JOBS_FAILED_FIELD_NUMBER: _ClassVar[int] + JOBS_MEASURED_FIELD_NUMBER: _ClassVar[int] JOBS_SIMULATOR_FAULT_FIELD_NUMBER: _ClassVar[int] accuracy_score: float experience_score: float @@ -284,9 +298,9 @@ class SimulationRun(_message.Message): tts: SimulationRun.JobMetrics.TTS conversation: SimulationRun.JobMetrics.Conversation jobs_total: int - jobs_failed: int + jobs_measured: int jobs_simulator_fault: int - def __init__(self, accuracy_score: _Optional[float] = ..., experience_score: _Optional[float] = ..., scenario_pass_rate: _Optional[float] = ..., stt: _Optional[_Union[SimulationRun.JobMetrics.STT, _Mapping]] = ..., llm: _Optional[_Union[SimulationRun.JobMetrics.LLM, _Mapping]] = ..., tts: _Optional[_Union[SimulationRun.JobMetrics.TTS, _Mapping]] = ..., conversation: _Optional[_Union[SimulationRun.JobMetrics.Conversation, _Mapping]] = ..., jobs_total: _Optional[int] = ..., jobs_failed: _Optional[int] = ..., jobs_simulator_fault: _Optional[int] = ...) -> None: ... + def __init__(self, accuracy_score: _Optional[float] = ..., experience_score: _Optional[float] = ..., scenario_pass_rate: _Optional[float] = ..., stt: _Optional[_Union[SimulationRun.JobMetrics.STT, _Mapping]] = ..., llm: _Optional[_Union[SimulationRun.JobMetrics.LLM, _Mapping]] = ..., tts: _Optional[_Union[SimulationRun.JobMetrics.TTS, _Mapping]] = ..., conversation: _Optional[_Union[SimulationRun.JobMetrics.Conversation, _Mapping]] = ..., jobs_total: _Optional[int] = ..., jobs_measured: _Optional[int] = ..., jobs_simulator_fault: _Optional[int] = ...) -> None: ... class Create(_message.Message): __slots__ = () class Request(_message.Message):