Skip to content

Tag evals with the tracer version and ml app #12184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ddog-thibault-nadin wants to merge 1 commit into
base: thibault/llmobs-submit-feedback
Choose a base branch
from
Draft

Tag evals with the tracer version and ml app #12184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import com.squareup.moshi.Moshi;
import datadog.trace.api.DDTraceApiInfo;
import datadog.trace.llmobs.LLMObsIntakeWorker;
import java.io.IOException;
import java.util.List;
Expand Down Expand Up @@ -45,7 +46,10 @@ public LLMObsEval(
this.ml_app = mlApp;
this.metric_type = metricType;
this.label = label;
this.tags = tags == null ? null : IntakeTags.flatten(tags);
// Every submission carries the tracer version and the ML app, as dd-trace-py and dd-trace-js
// do, so that evals can be filtered by tag and not only by the top level ml_app field.
this.tags =
IntakeTags.flatten(tags, "ddtrace.version:" + DDTraceApiInfo.VERSION, "ml_app:" + mlApp);
}

/**
Expand Down
19 changes: 13 additions & 6 deletions dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/LLMObsEvalTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
import com.squareup.moshi.Types;
import datadog.trace.api.DDTraceApiInfo;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -21,8 +22,9 @@
* an adapter held by the worker. These tests exist to prove that indirection changed nothing on the
* wire: the same key set as before, no {@code submitter}, no feedback-only target.
*
* <p>The single intentional addition is {@code event_kind:"evaluation"}, which discriminates
* evaluations from feedback the way dd-trace-py and dd-trace-js do.
* <p>The intentional additions are {@code event_kind:"evaluation"}, which discriminates evaluations
* from feedback, and the {@code ddtrace.version} and {@code ml_app} tags every submission carries.
* Both align the payload with dd-trace-py and dd-trace-js.
*/
class LLMObsEvalTest {

Expand Down Expand Up @@ -67,9 +69,12 @@ void testScoreEvalCarriesTheV1KeySetAndNothingElse() throws IOException {
assertEquals("score", metric.get("metric_type"));
assertEquals("sentiment", metric.get("label"));
assertEquals(0.75, metric.get("score_value"));
assertEquals(Collections.singletonList("source:web-ui"), metric.get("tags"));
assertEquals(
Arrays.asList(
"ddtrace.version:" + DDTraceApiInfo.VERSION, "ml_app:my-app", "source:web-ui"),
metric.get("tags"));

// The only addition to the v1 payload.
// The additions to the v1 payload.
assertEquals("evaluation", metric.get("event_kind"));

// Feedback-only keys must never leak into the v1 payload.
Expand All @@ -92,8 +97,10 @@ void testCategoricalEvalCarriesTheV1KeySet() throws IOException {
assertEquals("categorical", metric.get("metric_type"));
assertEquals("positive", metric.get("categorical_value"));
assertFalse(metric.containsKey("score_value"), metric.toString());
// A null tag map is omitted rather than serialized as an empty list.
assertFalse(metric.containsKey("tags"), metric.toString());
// A null user tag map still yields the tags every submission carries.
assertEquals(
Arrays.asList("ddtrace.version:" + DDTraceApiInfo.VERSION, "ml_app:my-app"),
metric.get("tags"));
assertEquals("evaluation", metric.get("event_kind"));
}

Expand Down
Loading