diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 3ce78b88c22..daf35c2c351 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -24,6 +24,10 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni } public @Nullable BackendApi createBackendApi(Intake intake) { + return createBackendApi(intake, true); + } + + public @Nullable BackendApi createBackendApi(Intake intake, boolean responseCompression) { HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true); if (intake.isAgentlessEnabled(config)) { @@ -46,23 +50,28 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); featuresDiscovery.discoverIfOutdated(); - if (featuresDiscovery.supportsEvpProxy()) { - String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); - String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); - HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); - String subdomain = intake.getUrlPrefix(); - return new EvpProxyApi( - traceId, - evpProxyUrl, - subdomain, - retryPolicyFactory, - sharedCommunicationObjects.agentHttpClient, - true); + if (!featuresDiscovery.supportsEvpProxy()) { + log.warn( + "Cannot create backend API client since agentless mode is disabled, " + + "and agent does not support EVP proxy"); + return null; } + String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); - log.warn( - "Cannot create backend API client since agentless mode is disabled, " - + "and agent does not support EVP proxy"); - return null; + String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); + log.debug( + "Creating EVP proxy client for {} using endpoint {} with responseCompression={}", + intake, + evpProxyEndpoint, + responseCompression); + HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); + String subdomain = intake.getUrlPrefix(); + return new EvpProxyApi( + traceId, + evpProxyUrl, + subdomain, + retryPolicyFactory, + sharedCommunicationObjects.agentHttpClient, + responseCompression); } } diff --git a/communication/src/main/java/datadog/communication/EvpProxy.java b/communication/src/main/java/datadog/communication/EvpProxy.java new file mode 100644 index 00000000000..c2453bccb25 --- /dev/null +++ b/communication/src/main/java/datadog/communication/EvpProxy.java @@ -0,0 +1,15 @@ +package datadog.communication; + +/** Shared EVP proxy constants. */ +public final class EvpProxy { + + public static final String SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; + + /** + * Default SDK-side target for uncompressed EVP request bodies. Writers may split batches at or + * below this size to keep Agent proxy requests comfortably bounded. + */ + public static final int PAYLOAD_SIZE_LIMIT_BYTES = 5 * 1024 * 1024; + + private EvpProxy() {} +} diff --git a/communication/src/main/java/datadog/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java index 83037ab9663..267f4952fe2 100644 --- a/communication/src/main/java/datadog/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -20,12 +20,12 @@ public class EvpProxyApi implements BackendApi { private static final Logger log = LoggerFactory.getLogger(EvpProxyApi.class); private static final String API_VERSION = "v2"; - private static final String X_DATADOG_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id"; private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id"; private static final String ACCEPT_ENCODING_HEADER = "Accept-Encoding"; private static final String CONTENT_ENCODING_HEADER = "Content-Encoding"; private static final String GZIP_ENCODING = "gzip"; + private static final String IDENTITY_ENCODING = "identity"; private final String traceId; private final HttpRetryPolicy.Factory retryPolicyFactory; @@ -62,7 +62,7 @@ public T post( Request.Builder requestBuilder = new Request.Builder() .url(url) - .addHeader(X_DATADOG_EVP_SUBDOMAIN_HEADER, subdomain) + .addHeader(EvpProxy.SUBDOMAIN_HEADER, subdomain) .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); @@ -74,11 +74,21 @@ public T post( requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING); } + // OkHttp's BridgeInterceptor adds a transparent Accept-Encoding: gzip when the caller does not + // set one. Set the header explicitly on both paths so responseCompression=false actually + // suppresses gzip negotiation on the wire. if (responseCompression) { requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING); + } else { + requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, IDENTITY_ENCODING); } final Request request = requestBuilder.post(requestBody).build(); + log.debug( + "Posting EVP request to {} with responseCompression={} requestCompression={}", + url, + responseCompression, + requestCompression); try (okhttp3.Response response = OkHttpUtils.sendWithRetries(httpClient, retryPolicyFactory, request)) { diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java new file mode 100644 index 00000000000..7aa33de7742 --- /dev/null +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -0,0 +1,112 @@ +package datadog.communication; + +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.metrics.api.Monitoring; +import datadog.trace.api.Config; +import datadog.trace.api.ProtocolVersion; +import datadog.trace.api.intake.Intake; +import java.nio.charset.StandardCharsets; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.Test; + +class BackendApiFactoryTest { + + private static final MediaType JSON = MediaType.parse("application/json"); + + @Test + void noBackendApiWhenAgentDoesNotAdvertiseEvpProxy() { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null); + final BackendApiFactory factory = + new BackendApiFactory(Config.get(), sharedCommunicationObjects(discovery, null)); + + assertNull(factory.createBackendApi(Intake.EVENT_PLATFORM, false)); + } + + @Test + void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(V4_EVP_PROXY_ENDPOINT); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); + } finally { + agent.shutdown(); + } + } + + private static SharedCommunicationObjects sharedCommunicationObjects( + final DDAgentFeaturesDiscovery discovery, final HttpUrl agentUrl) { + final TestSharedCommunicationObjects sco = new TestSharedCommunicationObjects(discovery); + sco.agentUrl = agentUrl != null ? agentUrl : HttpUrl.get("http://localhost:8126/"); + sco.agentHttpClient = new OkHttpClient(); + return sco; + } + + private static final class TestSharedCommunicationObjects extends SharedCommunicationObjects { + private final DDAgentFeaturesDiscovery discovery; + + private TestSharedCommunicationObjects(final DDAgentFeaturesDiscovery discovery) { + this.discovery = discovery; + } + + @Override + public DDAgentFeaturesDiscovery featuresDiscovery(final Config config) { + return discovery; + } + } + + private static final class FakeFeaturesDiscovery extends DDAgentFeaturesDiscovery { + private final String evpProxyEndpoint; + + private FakeFeaturesDiscovery(final String evpProxyEndpoint) { + super( + new OkHttpClient(), + Monitoring.DISABLED, + HttpUrl.get("http://localhost:8126/"), + ProtocolVersion.V0_5, + true, + false); + this.evpProxyEndpoint = evpProxyEndpoint; + } + + @Override + public void discoverIfOutdated() {} + + @Override + public String getEvpProxyEndpoint() { + return evpProxyEndpoint; + } + + @Override + public boolean supportsEvpProxy() { + return evpProxyEndpoint != null; + } + } +} diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index a611a422d7b..cd0b33ebe6b 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -524,15 +524,17 @@ public static void shutdown(final boolean sync) { if (profilingEnabled) { shutdownProfilingAgent(sync); } + // Before telemetry: the feature flagging writers queue drop/degradation metrics during their + // final flush, and only a still-running telemetry worker can drain and transmit them. + if (featureFlaggingEnabled) { + shutdownFeatureFlagging(AGENT_CLASSLOADER); + } if (telemetryEnabled) { stopTelemetry(); } if (flareEnabled) { stopFlarePoller(); } - if (featureFlaggingEnabled) { - shutdownFeatureFlagging(AGENT_CLASSLOADER); - } if (agentlessLogSubmissionEnabled) { shutdownLogsIntake(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java index e911a980c31..9ec4eca5f4e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java @@ -3,6 +3,7 @@ import static datadog.trace.common.writer.DDIntakeWriter.DEFAULT_INTAKE_TIMEOUT; import static datadog.trace.common.writer.DDIntakeWriter.DEFAULT_INTAKE_VERSION; +import datadog.communication.EvpProxy; import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.trace.api.civisibility.InstrumentationBridge; @@ -26,7 +27,6 @@ public class DDEvpProxyApi extends RemoteApi { private static final Logger log = LoggerFactory.getLogger(DDEvpProxyApi.class); - private static final String DD_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; private static final String CONTENT_ENCODING_HEADER = "Content-Encoding"; private static final String GZIP_CONTENT_TYPE = "gzip"; @@ -131,7 +131,7 @@ public Response sendSerializedTraces(Payload payload) { Request.Builder builder = new Request.Builder() .url(proxiedApiUrl) - .addHeader(DD_EVP_SUBDOMAIN_HEADER, subdomain) + .addHeader(EvpProxy.SUBDOMAIN_HEADER, subdomain) .tag(OkHttpUtils.CustomListener.class, telemetryListener); if (isCompressionEnabled()) { diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index d55935e9835..2346aa20218 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -2,6 +2,7 @@ import static datadog.communication.http.OkHttpUtils.gzippedMsgpackRequestBodyOf; +import datadog.communication.EvpProxy; import datadog.communication.serialization.GrowableBuffer; import datadog.communication.serialization.Writable; import datadog.communication.serialization.msgpack.MsgPackWriter; @@ -99,7 +100,7 @@ public class LLMObsSpanMapper implements RemoteMapper { private int spansWritten; public LLMObsSpanMapper() { - this(5 << 20); + this(EvpProxy.PAYLOAD_SIZE_LIMIT_BYTES); } private LLMObsSpanMapper(int size) { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy index fd797695d99..52ad90a9514 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.common.writer.ddintake import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper +import datadog.communication.EvpProxy import datadog.communication.serialization.ByteBufferConsumer import datadog.communication.serialization.FlushingBuffer import datadog.communication.serialization.msgpack.MsgPackWriter @@ -64,7 +65,7 @@ class DDEvpProxyApiTest extends DDCoreSpecification { clientResponse.status().present clientResponse.status().asInt == 200 agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(DDEvpProxyApi.DD_EVP_SUBDOMAIN_HEADER) == intakeSubdomain + agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain cleanup: agentEvpProxy.close() @@ -100,7 +101,7 @@ class DDEvpProxyApiTest extends DDCoreSpecification { clientResponse.status().present clientResponse.status().asInt == 200 agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(DDEvpProxyApi.DD_EVP_SUBDOMAIN_HEADER) == intakeSubdomain + agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain cleanup: agentEvpProxy.close() diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java b/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java index d33fcf1529d..d09dc2f2ac0 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java @@ -29,6 +29,14 @@ private CoreMetricCollector() { this.metricsQueue = new ArrayBlockingQueue<>(RAW_QUEUE_SIZE); } + public void count(String metricName, long value, String tag) { + if (value <= 0) { + return; + } + this.metricsQueue.offer( + new CoreMetric(METRIC_NAMESPACE, true, metricName, "count", value, tag)); + } + @Override public void prepareMetrics() { // Collect span metrics diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index e9ef282cc7d..79774b348b1 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -67,6 +67,7 @@ public enum AgentThread { LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"), FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"), + FEATURE_FLAG_EVALUATION_PROCESSOR("dd-ffe-evaluation-processor"), FEATURE_FLAG_CONFIGURATION_POLLER("dd-feature-flagging-http-poller"); public final String threadName; diff --git a/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy index 205da2bd0f5..5d0b920bfb9 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy @@ -53,4 +53,24 @@ class CoreMetricCollectorTest extends DDSpecification { collector.prepareMetrics() collector.drain().size() == limit } + + def "direct count core metric"() { + setup: + def collector = CoreMetricCollector.getInstance() + collector.drain() + + when: + collector.count('flagevaluation.rows.dropped', 3, 'reason:queue_overflow') + def metrics = collector.drain() + + then: + metrics.size() == 1 + + def metric = metrics[0] + metric.type == 'count' + metric.value == 3 + metric.namespace == 'tracers' + metric.metricName == 'flagevaluation.rows.dropped' + metric.tags == ['reason:queue_overflow'] + } } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index aaad660aae9..90e734dd1c4 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1585,6 +1585,14 @@ "aliases": [] } ], + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": [] + } + ], "DD_FORCE_CLEAR_TEXT_HTTP_FOR_INTAKE_CLIENT": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 91b32ee1d64..4cc841e737a 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -6,6 +6,8 @@ import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,6 +18,7 @@ public class FeatureFlaggingSystem { private static volatile ConfigurationSourceService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; + private static volatile FlagEvaluationWriter FLAG_EVAL_WRITER; private static volatile SpanEnrichmentWriter SPAN_ENRICHMENT_WRITER; private static volatile FeatureFlaggingGateway.ActivationListener ACTIVATION_LISTENER; private static volatile boolean STARTED; @@ -49,12 +52,7 @@ public static synchronized void start(final SharedCommunicationObjects sco) { return; } - try { - initializeSystem(sco, config); - } catch (final RuntimeException | Error e) { - STARTED = false; - throw e; - } + initializeOrRollBack(sco, config); } private static synchronized void activateAgentless( @@ -65,10 +63,17 @@ private static synchronized void activateAgentless( } ACTIVATION_LISTENER = null; FeatureFlaggingGateway.removeActivationListener(activationListener); + initializeOrRollBack(sco, config); + } + + // Any failure leaves the subsystem fully stopped: stop() releases whatever initializeSystem + // managed to publish before it threw, so a later start() begins from a clean state. + private static void initializeOrRollBack( + final SharedCommunicationObjects sco, final Config config) { try { initializeSystem(sco, config); } catch (final RuntimeException | Error e) { - STARTED = false; + stop(); throw e; } } @@ -82,6 +87,24 @@ private static void initializeSystem(final SharedCommunicationObjects sco, final final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); initialize(configService, exposureWriter); + final boolean evalCountsEnabled = + config + .configProvider() + .getBoolean(FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, true); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(evalCountsEnabled); + if (evalCountsEnabled) { + final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); + // Publish before start() so a failed start is still reachable by the rollback in stop(). + FLAG_EVAL_WRITER = evalWriter; + evalWriter.start(); + LOGGER.debug("Flag evaluation EVP writer started"); + } else { + FeatureFlaggingGateway.setFlagEvalWriter(null); + LOGGER.debug( + "Flag evaluation EVP writer disabled ({}=false)", + FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED); + } + // APM span enrichment: agent-side listener for flag-evaluation seam events. Uses the process- // wide singleton so a subsystem restart reuses the one already-registered trace interceptor // (which the tracer cannot remove) instead of registering a second, rejected one. Cheap: it @@ -134,37 +157,39 @@ static ConfigurationSourceService createConfigurationSourceService( justification = "Agent-internal class; Class object does not escape to app code and lock only guards the subsystem lifecycle.") public static synchronized void stop() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + FeatureFlaggingGateway.setFlagEvalWriter(null); final FeatureFlaggingGateway.ActivationListener activationListener = ACTIVATION_LISTENER; + final FlagEvaluationWriter flagEvalWriter = FLAG_EVAL_WRITER; final SpanEnrichmentWriter spanEnrichmentWriter = SPAN_ENRICHMENT_WRITER; final ExposureWriter exposureWriter = EXPOSURE_WRITER; final ConfigurationSourceService configService = CONFIG_SERVICE; STARTED = false; ACTIVATION_LISTENER = null; + FLAG_EVAL_WRITER = null; SPAN_ENRICHMENT_WRITER = null; EXPOSURE_WRITER = null; CONFIG_SERVICE = null; if (activationListener != null) { FeatureFlaggingGateway.removeActivationListener(activationListener); } - try { - if (spanEnrichmentWriter != null) { - spanEnrichmentWriter.close(); - } - } finally { - try { - if (exposureWriter != null) { - exposureWriter.close(); - } - } finally { - if (configService != null) { - configService.close(); - } - } - } + closeQuietly(flagEvalWriter); + closeQuietly(spanEnrichmentWriter); + closeQuietly(exposureWriter); + closeQuietly(configService); LOGGER.debug("Feature Flagging system stopped"); } static boolean isAwaitingApplicationActivation() { return ACTIVATION_LISTENER != null; } + + private static void closeQuietly(final AutoCloseable resource) { + if (resource != null) { + try { + resource.close(); + } catch (Exception ignored) { + } + } + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index d408d91da4e..cce19d736ad 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,13 +27,23 @@ import datadog.remoteconfig.Product; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.test.junit.utils.config.WithConfig; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; class FeatureFlaggingSystemTest { + @AfterEach + void resetFlagEvaluationGateway() { + FeatureFlaggingSystem.stop(); + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") @WithConfig( @@ -82,6 +93,7 @@ void agentlessStopRemovesPendingApplicationProviderActivation() { } @Test + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "true") @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { @@ -94,6 +106,7 @@ void testFeatureFlagSystemInitialization() { when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); FeatureFlaggingSystem.start(sharedCommunicationObjects); FeatureFlaggingSystem.start(sharedCommunicationObjects); @@ -101,8 +114,13 @@ void testFeatureFlagSystemInitialization() { verify(poller).addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).addListener(eq(Product.FFE_FLAGS), any(ConfigurationDeserializer.class), any()); verify(poller).start(); + assertTrue(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertNotNull(FeatureFlaggingGateway.getFlagEvalWriter()); FeatureFlaggingSystem.stop(); + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + // stop() is idempotent: a second call must be a safe no-op. FeatureFlaggingSystem.stop(); verify(poller).removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); @@ -110,6 +128,58 @@ void testFeatureFlagSystemInitialization() { verify(poller).stop(); } + @Test + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "false") + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig( + key = FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, + value = "http://localhost:1/config") + void testFlagEvaluationWriterCanBeDisabled() { + SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + FeatureFlaggingGateway.setFlagEvalWriter(mock(FlagEvaluationWriter.class)); + + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects); + // Agentless defers initialization until the application provider activates. + FeatureFlaggingGateway.activate(); + + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } finally { + FeatureFlaggingSystem.stop(); + } + } + + @Test + void testFeatureFlagSystemShutdownClearsGatewayState() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + FeatureFlaggingGateway.setFlagEvalWriter(mock(FlagEvaluationWriter.class)); + + FeatureFlaggingSystem.stop(); + + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") + void failedStartRollsBackPartiallyInitializedState() { + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + + assertThrows( + IllegalStateException.class, () -> FeatureFlaggingSystem.start(sharedCommunicationObjects)); + + // A failed start must leave nothing behind: no listener awaiting activation, no gateway + // writer, and STARTED cleared so a later start() is not swallowed as "already started". + assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertThrows( + IllegalStateException.class, () -> FeatureFlaggingSystem.start(sharedCommunicationObjects)); + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") @@ -127,6 +197,9 @@ void testThatRemoteConfigIsRequired() { @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig( + key = FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, + value = "http://localhost:1/config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void agentlessConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { assertInstanceOf( @@ -135,6 +208,26 @@ void agentlessConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { sharedCommunicationObjects(), Config.get())); } + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig( + key = FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, + value = "http://localhost:1/config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "true") + void agentlessConfigurationSourceStartsTelemetryWritersWithoutRemoteConfig() { + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects()); + // Agentless defers initialization until the application provider activates. + FeatureFlaggingGateway.activate(); + + assertTrue(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertNotNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } finally { + FeatureFlaggingSystem.stop(); + } + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") @@ -149,6 +242,14 @@ void explicitRemoteConfigUsesRemoteConfigService() { sharedCommunicationObjects, Config.get())); } + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") + void offlineConfigurationSourceDoesNotStartNetworkSource() { + assertNull( + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") void invalidConfigurationSourceDoesNotStartNetworkSource() { @@ -167,14 +268,6 @@ void unsupportedNormalizedConfigurationSourceDoesNotStartNetworkSource() { sharedCommunicationObjects(), config)); } - @Test - @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") - void offlineConfigurationSourceDoesNotStartNetworkSource() { - assertNull( - FeatureFlaggingSystem.createConfigurationSourceService( - sharedCommunicationObjects(), Config.get())); - } - @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") void startWithOfflineConfigurationSourceDisablesSystem() { diff --git a/products/feature-flagging/feature-flagging-api/build.gradle.kts b/products/feature-flagging/feature-flagging-api/build.gradle.kts index 88531ceaeed..5de303ffe0c 100644 --- a/products/feature-flagging/feature-flagging-api/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-api/build.gradle.kts @@ -5,6 +5,7 @@ plugins { `java-library` idea `maven-publish` + id("me.champeau.jmh") } apply(from = "$rootDir/gradle/java.gradle") @@ -55,6 +56,23 @@ dependencies { testImplementation(libs.bundles.mockito) testImplementation(libs.moshi) testImplementation("org.awaitility:awaitility:4.3.0") + + // The main source set gets the bootstrap/config types as compileOnly, so the JMH source set + // needs them on its own compile and runtime classpath to drive the hook end to end. + jmhImplementation(project(":products:feature-flagging:feature-flagging-bootstrap")) + jmhImplementation(project(":products:feature-flagging:feature-flagging-config")) + jmhImplementation(project(":utils:config-utils")) +} + +jmh { + jmhVersion = libs.versions.jmh.get() + duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE + if (project.hasProperty("jmhIncludes")) { + includes = listOf(project.property("jmhIncludes").toString()) + } + if (project.hasProperty("jmhProf")) { + profilers = listOf(project.property("jmhProf").toString()) + } } fun AbstractCompile.configureCompiler( diff --git a/products/feature-flagging/feature-flagging-api/src/jmh/java/datadog/trace/api/openfeature/FlagEvalHookHotPathBenchmark.java b/products/feature-flagging/feature-flagging-api/src/jmh/java/datadog/trace/api/openfeature/FlagEvalHookHotPathBenchmark.java new file mode 100644 index 00000000000..e292424e53e --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/jmh/java/datadog/trace/api/openfeature/FlagEvalHookHotPathBenchmark.java @@ -0,0 +1,203 @@ +package datadog.trace.api.openfeature; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.FlagValueType; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.ImmutableStructure; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.Reason; +import dev.openfeature.sdk.Value; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Evaluation-thread benchmark for the OpenFeature hook: the inline cost a caller actually pays + * inside FlagEvalLoggingHook.finallyAfter, across context shapes. + * + *

This is the counterpart to FlagEvaluationHotPathBenchmark in feature-flagging-lib, which + * measures the writer queue and the worker-thread aggregation. The hook and the OpenFeature context + * types live in this module, so the inline cost has to be measured here. + * + *

The dominant inline cost under consent-on is DDEvaluator.copyPrunedContext: one bounded walk + * of the caller-owned EvaluationContext. contextCopy isolates that walk so its share of + * hookFinallyAfter is directly readable. Under consent-off the hook skips the copy entirely, which + * hookFinallyAfterConsentOff measures as the protected-path floor. + * + *

The writer used here is a no-op that discards events, so no queue or worker cost is included. + * + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-api:jmh + * -PjmhIncludes=FlagEvalHookHotPathBenchmark}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class FlagEvalHookHotPathBenchmark { + + /** + * Context shapes. The three 100-leaf shapes (flat/100attrs, nested/10structs_10fields, + * list/10lists_10items) carry the same leaf count under different structure, so the spread + * between them isolates shape cost from leaf count. + */ + @Param({ + "flat/0attrs", + "flat/10attrs", + "flat/100attrs", + "nested/10structs_10fields", + "list/10lists_10items" + }) + public String shape; + + private HookContext hookContext; + private FlagEvaluationDetails consentOnDetails; + private FlagEvaluationDetails consentOffDetails; + private FlagEvalLoggingHook hook; + + @Setup(Level.Trial) + public void setUp() { + final MutableContext ctx = buildContext(shape); + hookContext = + HookContext.builder() + .flagKey("bench-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .build(); + + consentOnDetails = details(true); + consentOffDetails = details(false); + + // Discarding writer: isolates hook-inline cost from queue mechanics. + hook = new FlagEvalLoggingHook<>(new NoOpWriter()); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + + @TearDown(Level.Trial) + public void tearDown() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + } + + /** Total inline cost under consent-on: scalar extraction, bounded context copy, and enqueue. */ + @Benchmark + public void hookFinallyAfter() { + hook.finallyAfter(hookContext, consentOnDetails, Collections.emptyMap()); + } + + /** Protected-path floor: consent-off skips the context copy, leaving scalar work plus enqueue. */ + @Benchmark + public void hookFinallyAfterConsentOff() { + hook.finallyAfter(hookContext, consentOffDetails, Collections.emptyMap()); + } + + /** The bounded context copy alone - the component that scales with context shape. */ + @Benchmark + public void contextCopy(final Blackhole blackhole) { + blackhole.consume(DDEvaluator.copyPrunedContext(hookContext.getCtx())); + } + + private static FlagEvaluationDetails details(final boolean observeFullEvaluationData) { + return FlagEvaluationDetails.builder() + .flagKey("bench-flag") + .value("on-value") + .variant("on") + .reason(Reason.TARGETING_MATCH.name()) + .flagMetadata( + ImmutableMetadata.builder() + .addString("allocationKey", "alloc-1") + .addLong("__dd_eval_timestamp_ms", 1_700_000_000_000L) + .addBoolean( + DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA, observeFullEvaluationData) + .build()) + .build(); + } + + private static MutableContext buildContext(final String shape) { + final MutableContext ctx = new MutableContext("bench-user"); + if ("flat/0attrs".equals(shape)) { + return ctx; + } + if ("flat/10attrs".equals(shape)) { + return addFlat(ctx, 10); + } + if ("flat/100attrs".equals(shape)) { + return addFlat(ctx, 100); + } + if ("nested/10structs_10fields".equals(shape)) { + for (int i = 0; i < 10; i++) { + final Map inner = new HashMap<>(); + for (int j = 0; j < 10; j++) { + inner.put("field" + j, new Value("value" + j)); + } + ctx.add("struct" + i, new ImmutableStructure(inner)); + } + return ctx; + } + if ("list/10lists_10items".equals(shape)) { + for (int i = 0; i < 10; i++) { + final List items = new ArrayList<>(10); + for (int j = 0; j < 10; j++) { + items.add(new Value("value" + j)); + } + ctx.add("list" + i, items); + } + return ctx; + } + throw new IllegalArgumentException("unknown benchmark shape: " + shape); + } + + private static MutableContext addFlat(final MutableContext ctx, final int count) { + for (int i = 0; i < count; i++) { + ctx.add("field" + i, "value" + i); + } + return ctx; + } + + /** Discards events so only hook-inline work is measured. */ + private static final class NoOpWriter implements FlagEvaluationWriter { + @Override + public void enqueue(final FlagEvalEvent event) {} + + @Override + public boolean hasCapacityForEnqueue() { + return true; + } + + @Override + public void countPreQueueOverflow() {} + + @Override + public void countContextTruncated(final String reason) {} + + @Override + public void start() {} + + @Override + public void close() {} + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index fe47bbab16f..fa394417c8a 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -19,6 +19,7 @@ import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.ImmutableStructure; import dev.openfeature.sdk.ProviderEvaluation; import dev.openfeature.sdk.Reason; import dev.openfeature.sdk.Structure; @@ -28,11 +29,15 @@ import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -46,11 +51,59 @@ class DDEvaluator implements Evaluator, FeatureFlaggingGateway.ConfigListener { private static final Set> SUPPORTED_RESOLUTION_TYPES = new HashSet<>(asList(String.class, Boolean.class, Integer.class, Double.class, Value.class)); + /** + * Maximum evaluation-context nesting depth captured on the hot path. Recursion runs on the + * caller's evaluation thread over a caller-owned Value tree, so an arbitrarily deep + * list/structure would overflow that thread's stack - and a StackOverflowError is not caught by + * the LinkageError | Exception guards that keep telemetry from breaking an evaluation. Values + * below the limit are truncated to null, the same way the cycle guard truncates. Kept aligned + * with the cross-SDK RFC target (4). + */ + static final int MAX_SNAPSHOT_DEPTH = 4; + + /** + * Maximum number of top-level context fields retained by copyPrunedContext. Bounds the width of + * the caller-supplied context and, transitively, the size of every FlagEvalEvent sitting in the + * async hand-off queue. Kept aligned with the cross-SDK RFC. + */ + static final int MAX_CONTEXT_FIELDS = 256; + + /** + * Maximum character length for a single context KEY retained by copyPrunedContext. Keys are + * stored verbatim in every full-tier bucket, so an unbounded key size would let a single caller + * inflate steady-state heap use. Longer keys cause the field to be skipped. + */ + static final int MAX_KEY_LENGTH = 256; + + /** + * Maximum character length for a single context string VALUE retained by copyPrunedContext. + * Longer values cause the field to be skipped (matches previous pruneContext behavior). + * Non-string scalars are not length-bounded. + */ + static final int MAX_VALUE_LENGTH = 256; + + /** + * Maximum number of elements walked per list encountered during copyPrunedContext. Bounds the + * fan-out of a single wide list at capture time so one caller cannot inflate the hot path with a + * huge but shallow structure. Elements past the limit are skipped. + */ + static final int MAX_LIST_ELEMENTS = 256; + + /** + * Maximum number of properties walked per structure encountered during copyPrunedContext. Same + * intent as MAX_LIST_ELEMENTS for structures. Properties past the limit are skipped. + */ + static final int MAX_STRUCTURE_PROPERTIES = 256; + // Evaluation-metadata keys consumed by the span-enrichment capture hook (see // SpanEnrichmentHook). Emitted only when the span-enrichment gate is on. static final String METADATA_SPLIT_SERIAL_ID = "__dd_split_serial_id"; static final String METADATA_DO_LOG = "__dd_do_log"; + // Stamped on every DD-produced evaluation (including PROVIDER_NOT_READY, with false). Missing + // key = non-DD provider; the hook falls back to false (fail-closed). + static final String METADATA_OBSERVE_FULL_EVALUATION_DATA = "observe_full_evaluation_data"; + // Read once: when off, the __dd_* span-enrichment metadata is not attached to evaluations, so an // enabled provider pays nothing extra unless span enrichment is also enabled. The gate does not // change at runtime, and this class is loaded lazily (well after startup) so config is ready. @@ -99,33 +152,46 @@ public ProviderEvaluation evaluate( final String key, final T defaultValue, final EvaluationContext context) { + // Snapshot the config once and thread observeFullEvaluationData through every + // ProviderEvaluation returned, so the hook's consent decision is pinned to this evaluation's + // config and cannot drift on a concurrent Remote Config swap. + final ServerConfiguration config = configuration.get(); + // Boolean.TRUE.equals covers both null (privacy-preserving default) and Boolean.FALSE without + // an NPE — the field is boxed so a malformed UFC message doesn't abort the whole parse. + final boolean observeFullEvaluationData = + config != null && Boolean.TRUE.equals(config.observeFullEvaluationData); try { - final ServerConfiguration config = configuration.get(); if (config == null) { - return error(defaultValue, ErrorCode.PROVIDER_NOT_READY); + return error(defaultValue, ErrorCode.PROVIDER_NOT_READY, null, observeFullEvaluationData); } if (context == null) { - return error(defaultValue, ErrorCode.INVALID_CONTEXT); + return error(defaultValue, ErrorCode.INVALID_CONTEXT, null, observeFullEvaluationData); } final Flag flag = config.flags.get(key); if (flag == null) { - return error(defaultValue, ErrorCode.FLAG_NOT_FOUND); + return error(defaultValue, ErrorCode.FLAG_NOT_FOUND, null, observeFullEvaluationData); } if (!flag.enabled) { return ProviderEvaluation.builder() .value(defaultValue) .reason(Reason.DISABLED.name()) + .flagMetadata(consentMetadata(observeFullEvaluationData)) .build(); } if (flag.allocations == null) { - return error(defaultValue, ErrorCode.GENERAL, "Missing allocations for flag " + key); + return error( + defaultValue, + ErrorCode.GENERAL, + "Missing allocations for flag " + key, + observeFullEvaluationData); } final Instant now = Instant.now(); + final long evalTimestampMs = now.toEpochMilli(); final String targetingKey = context.getTargetingKey(); for (final Allocation allocation : flag.allocations) { @@ -143,10 +209,20 @@ public ProviderEvaluation evaluate( for (final Split split : allocation.splits) { if (isEmpty(split.shards)) { return resolveVariant( - target, key, defaultValue, flag, split.variationKey, allocation, split, context); + target, + key, + defaultValue, + flag, + split.variationKey, + allocation, + split, + context, + evalTimestampMs, + observeFullEvaluationData); } else { if (targetingKey == null) { - return error(defaultValue, ErrorCode.TARGETING_KEY_MISSING); + return error( + defaultValue, ErrorCode.TARGETING_KEY_MISSING, null, observeFullEvaluationData); } // To match a split, subject must match ALL underlying shards boolean allShardsMatch = true; @@ -165,7 +241,9 @@ public ProviderEvaluation evaluate( split.variationKey, allocation, split, - context); + context, + evalTimestampMs, + observeFullEvaluationData); } } } @@ -175,32 +253,40 @@ public ProviderEvaluation evaluate( return ProviderEvaluation.builder() .value(defaultValue) .reason(Reason.DEFAULT.name()) + .flagMetadata(consentMetadata(observeFullEvaluationData)) .build(); } catch (final PatternSyntaxException e) { - return error(defaultValue, ErrorCode.PARSE_ERROR, e); + return error(defaultValue, ErrorCode.PARSE_ERROR, e.getMessage(), observeFullEvaluationData); } catch (final NumberFormatException e) { - return error(defaultValue, ErrorCode.TYPE_MISMATCH, e); + return error( + defaultValue, ErrorCode.TYPE_MISMATCH, e.getMessage(), observeFullEvaluationData); } catch (final Exception e) { - return error(defaultValue, ErrorCode.GENERAL, e); + return error(defaultValue, ErrorCode.GENERAL, e.getMessage(), observeFullEvaluationData); } } - private static ProviderEvaluation error(final T defaultValue, final ErrorCode code) { - return error(defaultValue, code, (String) null); - } - - private static ProviderEvaluation error( - final T defaultValue, final ErrorCode code, final Throwable cause) { - return error(defaultValue, code, cause == null ? null : cause.getMessage()); + private static ImmutableMetadata consentMetadata(final boolean observeFullEvaluationData) { + return ImmutableMetadata.builder() + .addBoolean(METADATA_OBSERVE_FULL_EVALUATION_DATA, observeFullEvaluationData) + .build(); } private static ProviderEvaluation error( - final T defaultValue, final ErrorCode code, final String errorMessage) { + final T defaultValue, + final ErrorCode code, + final String errorMessage, + final boolean observeFullEvaluationData) { + // Under consent-off the errorMessage is dropped: exception messages from the outer catch blocks + // (NumberFormatException, generic Exception) can echo raw evaluation-context values, so they + // must never reach any consumer of ProviderEvaluation.getErrorMessage() — not just our own + // wire hook. Downstream (FlagEvalLoggingHook) falls back to ErrorCode.name(), so operators + // still get a stable signal like "TYPE_MISMATCH". return ProviderEvaluation.builder() .value(defaultValue) .reason(Reason.ERROR.name()) .errorCode(code) - .errorMessage(errorMessage) + .errorMessage(observeFullEvaluationData ? errorMessage : null) + .flagMetadata(consentMetadata(observeFullEvaluationData)) .build(); } @@ -362,15 +448,16 @@ private static ProviderEvaluation resolveVariant( final String variationKey, final Allocation allocation, final Split split, - final EvaluationContext context) { + final EvaluationContext context, + final long evalTimestampMs, + final boolean observeFullEvaluationData) { final Variant variant = flag.variations.get(variationKey); if (variant == null) { - return ProviderEvaluation.builder() - .value(defaultValue) - .reason(Reason.ERROR.name()) - .errorCode(ErrorCode.GENERAL) - .errorMessage("Variant not found for: " + variationKey) - .build(); + return error( + defaultValue, + ErrorCode.GENERAL, + "Variant not found for: " + variationKey, + observeFullEvaluationData); } if (!isTypeCompatible(target, flag.variationType)) { @@ -380,7 +467,8 @@ private static ProviderEvaluation resolveVariant( "Requested type " + target.getSimpleName() + " does not match flag variationType " - + flag.variationType.name()); + + flag.variationType.name(), + observeFullEvaluationData); } final T mappedValue; @@ -395,14 +483,19 @@ private static ProviderEvaluation resolveVariant( + "' value does not match declared type " + flag.variationType.name() + ": " - + e.getMessage()); + + e.getMessage(), + observeFullEvaluationData); } + // Stamp eval-time at the resolution point so first/last_evaluation reflect evaluation time, + // not hook-fire time. Passed to the hook via provider metadata "__dd_eval_timestamp_ms". final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder = ImmutableMetadata.builder() .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) - .addString("allocationKey", allocation.key); + .addString("allocationKey", allocation.key) + .addLong("__dd_eval_timestamp_ms", evalTimestampMs) + .addBoolean(METADATA_OBSERVE_FULL_EVALUATION_DATA, observeFullEvaluationData); // Surface the UFC split's serial id and the allocation's doLog flag for APM span enrichment — // only when span enrichment is on, so a provider without enrichment pays nothing extra. // __dd_split_serial_id is omitted when the split carries no serial id; __dd_do_log is always @@ -522,40 +615,297 @@ private static String allocationKey(final ProviderEvaluation resolution) } static AbstractMap flattenContext(final EvaluationContext context) { - final Set keys = context.keySet(); + return flattenValues(snapshotValues(context)); + } + + static Map snapshotValues(final EvaluationContext context) { + final HashMap values = new HashMap<>(); + final Set seenContainers = Collections.newSetFromMap(new IdentityHashMap<>()); + for (final String key : context.keySet()) { + values.put(key, snapshotValue(context.getValue(key), seenContainers, 0)); + } + return values; + } + + private static Value snapshotValue( + final Value value, final Set seenContainers, final int depth) { + if (value == null) { + return null; + } else if (value.isNull()) { + return new Value(); + } else if (value.isBoolean()) { + return new Value(value.asBoolean()); + } else if (value.isNumber()) { + final Object number = value.asObject(); + return number instanceof Integer + ? new Value((Integer) number) + : new Value(((Number) number).doubleValue()); + } else if (value.isString()) { + return new Value(value.asString()); + } else if (value.isInstant()) { + return new Value(value.asInstant()); + } else if (value.isList()) { + final List list = value.asList(); + if (depth >= MAX_SNAPSHOT_DEPTH || !seenContainers.add(list)) { + return new Value(); + } + final List snapshot = new ArrayList<>(list.size()); + for (final Value item : list) { + snapshot.add(snapshotValue(item, seenContainers, depth + 1)); + } + seenContainers.remove(list); + return new Value(Collections.unmodifiableList(snapshot)); + } else if (value.isStructure()) { + final Structure structure = value.asStructure(); + if (depth >= MAX_SNAPSHOT_DEPTH || !seenContainers.add(structure)) { + return new Value(); + } + final Map snapshot = new HashMap<>(); + for (final String key : structure.keySet()) { + snapshot.put(key, snapshotValue(structure.getValue(key), seenContainers, depth + 1)); + } + seenContainers.remove(structure); + return new Value(new ImmutableStructure(snapshot)); + } + throw new IllegalArgumentException("Unsupported OpenFeature value type: " + value); + } + + static AbstractMap flattenValues(final Map values) { final HashMap result = new HashMap<>(); - final Set seen = new HashSet<>(); - for (final String key : keys) { + final Set seenContainers = Collections.newSetFromMap(new IdentityHashMap<>()); + for (final Map.Entry root : values.entrySet()) { final Deque deque = new LinkedList<>(); - deque.push(new FlattenEntry(key, context.getValue(key))); + deque.push(new FlattenEntry(root.getKey(), root.getValue())); while (!deque.isEmpty()) { final FlattenEntry entry = deque.pop(); final Value value = entry.value; - if (value == null || seen.add(value)) { - if (value == null) { - result.put(entry.key, null); - } else if (value.isList()) { - final List list = value.asList(); + if (value == null) { + result.put(entry.key, null); + } else if (value.isList()) { + final List list = value.asList(); + if (seenContainers.add(list)) { for (int i = 0; i < list.size(); i++) { deque.push(new FlattenEntry(entry.key + "[" + i + "]", list.get(i))); } - } else if (value.isStructure()) { - final Structure structure = value.asStructure(); + } + } else if (value.isStructure()) { + final Structure structure = value.asStructure(); + if (seenContainers.add(structure)) { for (final String property : structure.keySet()) { deque.push( new FlattenEntry(entry.key + "." + property, structure.getValue(property))); } - } else { - result.put( - entry.key, - value.isInstant() ? value.asInstant().toString() : context.convertValue(value)); } + } else { + result.put(entry.key, convertValue(value)); } } } return result; } + private static Object convertValue(final Value value) { + if (value == null || value.isNull()) { + return null; + } else if (value.isBoolean()) { + return value.asBoolean(); + } else if (value.isNumber()) { + return value.asObject(); + } else if (value.isString()) { + return value.asString(); + } else if (value.isInstant()) { + return value.asInstant().toString(); + } + throw new IllegalArgumentException("Unsupported OpenFeature value type: " + value); + } + + // Reason-code bitmask constants used by copyPrunedContext to track which caps fired. + static final int REASON_MAX_CONTEXT_FIELDS = 1; + static final int REASON_MAX_KEY_LENGTH = 1 << 1; + static final int REASON_MAX_VALUE_LENGTH = 1 << 2; + static final int REASON_MAX_LIST_ELEMENTS = 1 << 3; + static final int REASON_MAX_STRUCTURE_PROPERTIES = 1 << 4; + static final int REASON_MAX_SNAPSHOT_DEPTH = 1 << 5; + static final int REASON_CYCLE = 1 << 6; + + /** Sorted reason-code strings, indexed by their bit position in the bitmask. */ + private static final String[] REASON_NAMES = { + "max_context_fields", + "max_key_length", + "max_value_length", + "max_list_elements", + "max_structure_properties", + "max_snapshot_depth", + "cycle", + }; + + /** + * Builds the sorted, comma-separated reason tag string from a bitmask of fired reason codes. + * Returns null when no reason bit is set (no truncation occurred). The returned string is ready + * to use directly as the "reason:..." tag value. + */ + static String truncationReasonTag(final int reasonMask) { + if (reasonMask == 0) { + return null; + } + final StringBuilder sb = new StringBuilder(); + for (int bit = 0; bit < REASON_NAMES.length; bit++) { + if ((reasonMask & (1 << bit)) != 0) { + if (sb.length() > 0) { + sb.append(','); + } + sb.append(REASON_NAMES[bit]); + } + } + return sb.toString(); + } + + /** + * Result of copyPrunedContext: the pruned attribute map plus an optional reason tag describing + * which caps fired during the walk. truncatedReason is null when no truncation occurred, so + * callers can skip the telemetry path with a single null check. + */ + static final class CopyResult { + final Map attrs; + + /** Non-null when at least one cap fired; ready to use as the "reason:..." tag value. */ + final String truncatedReason; + + CopyResult(final Map attrs, final String truncatedReason) { + this.attrs = attrs; + this.truncatedReason = truncatedReason; + } + } + + /** + * Single-pass bounded copy of a caller-owned EvaluationContext into the flattened, pruned map + * stored on a FlagEvalEvent and later canonicalized by the aggregator. + * + *

Every retained-size dimension is capped inline so the hot path performs work proportional to + * what is kept, never to what the caller supplied: MAX_CONTEXT_FIELDS - stop iterating the + * top-level context past this many retained fields MAX_KEY_LENGTH - skip fields whose flattened + * key exceeds this length (also enforced on every path segment produced by descending into + * lists/structures) MAX_VALUE_LENGTH - skip string values exceeding this length MAX_LIST_ELEMENTS + * - stop iterating a list past this many elements MAX_STRUCTURE_PROPERTIES - stop iterating a + * structure past this many properties MAX_SNAPSHOT_DEPTH - stop descending into lists/structures + * past this depth Cycle guard - identity-tracked containers currently on the recursion stack are + * treated as leaves + * + *

All numeric limits are named constants so they can be tuned independently. + * + *

Returns a CopyResult whose attrs is an empty map for null/empty input and whose + * truncatedReason is non-null when at least one cap fired. The returned map is a plain HashMap; + * canonical-key sorting happens once in the aggregator, off the hot path. + */ + static CopyResult copyPrunedContext(final EvaluationContext context) { + if (context == null) { + return new CopyResult(Collections.emptyMap(), null); + } + final Set keys = context.keySet(); + if (keys.isEmpty()) { + return new CopyResult(Collections.emptyMap(), null); + } + final HashMap out = new HashMap<>(); + final Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + final int[] reasonMask = {0}; + for (final String key : keys) { + if (out.size() >= MAX_CONTEXT_FIELDS) { + reasonMask[0] |= REASON_MAX_CONTEXT_FIELDS; + break; + } + if (EvaluationContext.TARGETING_KEY.equals(key)) { + continue; + } + copyPrunedValue(out, key, context.getValue(key), seen, 0, reasonMask); + } + final Map attrs = out.isEmpty() ? Collections.emptyMap() : out; + return new CopyResult(attrs, truncationReasonTag(reasonMask[0])); + } + + private static void copyPrunedValue( + final Map out, + final String key, + final Value value, + final Set seen, + final int depth, + final int[] reasonMask) { + if (out.size() >= MAX_CONTEXT_FIELDS) { + reasonMask[0] |= REASON_MAX_CONTEXT_FIELDS; + return; + } + if (key.length() > MAX_KEY_LENGTH) { + reasonMask[0] |= REASON_MAX_KEY_LENGTH; + return; + } + if (value == null || value.isNull()) { + out.put(key, null); + return; + } + if (value.isString()) { + final String s = value.asString(); + if (s.length() > MAX_VALUE_LENGTH) { + reasonMask[0] |= REASON_MAX_VALUE_LENGTH; + return; + } + out.put(key, s); + return; + } + if (value.isBoolean() || value.isNumber() || value.isInstant()) { + out.put(key, convertValue(value)); + return; + } + if (value.isList()) { + final List list = value.asList(); + if (depth >= MAX_SNAPSHOT_DEPTH) { + reasonMask[0] |= REASON_MAX_SNAPSHOT_DEPTH; + return; + } + if (!seen.add(list)) { + reasonMask[0] |= REASON_CYCLE; + return; + } + if (list.size() > MAX_LIST_ELEMENTS) { + reasonMask[0] |= REASON_MAX_LIST_ELEMENTS; + } + final int limit = Math.min(list.size(), MAX_LIST_ELEMENTS); + for (int i = 0; i < limit; i++) { + if (out.size() >= MAX_CONTEXT_FIELDS) { + reasonMask[0] |= REASON_MAX_CONTEXT_FIELDS; + break; + } + copyPrunedValue(out, key + "[" + i + "]", list.get(i), seen, depth + 1, reasonMask); + } + seen.remove(list); + return; + } + if (value.isStructure()) { + final Structure structure = value.asStructure(); + if (depth >= MAX_SNAPSHOT_DEPTH) { + reasonMask[0] |= REASON_MAX_SNAPSHOT_DEPTH; + return; + } + if (!seen.add(structure)) { + reasonMask[0] |= REASON_CYCLE; + return; + } + int walked = 0; + for (final String property : structure.keySet()) { + if (walked >= MAX_STRUCTURE_PROPERTIES) { + reasonMask[0] |= REASON_MAX_STRUCTURE_PROPERTIES; + break; + } + if (out.size() >= MAX_CONTEXT_FIELDS) { + reasonMask[0] |= REASON_MAX_CONTEXT_FIELDS; + break; + } + walked++; + copyPrunedValue( + out, key + "." + property, structure.getValue(property), seen, depth + 1, reasonMask); + } + seen.remove(structure); + } + } + @FunctionalInterface private interface NumberComparator { boolean compare(double a, double b); diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java new file mode 100644 index 00000000000..322f11ac9e1 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java @@ -0,0 +1,168 @@ +package datadog.trace.api.openfeature; + +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +/** + * OpenFeature Hook that captures flag evaluation events for EVP flagevaluation emission. + * + *

Contract: finallyAfter does scalar metadata extraction, a single bounded copy of the + * evaluation context, and a non-blocking offer to the writer's bounded queue. Aggregation and + * posting are deferred to the writer's worker thread. + * + *

Hot-path cost: under consent-on (observeFullEvaluationData=true) DDEvaluator.copyPrunedContext + * performs one bounded walk of the caller-owned EvaluationContext, applying every retained-size cap + * inline so work is proportional to what is kept, never to what the caller supplied. The returned + * map is capped by field count, key length, value length, list width, structure width, and depth. + * Under consent-off the context is dropped on emit, so the copy is skipped entirely and the hot + * path is scalar-only. + * + *

This hook is registered alongside the existing OTel FlagEvalMetricsHook - it does NOT replace + * it. + * + *

The writer is resolved lazily on each call, so the hook is always safe to register - if the + * writer is absent (killswitch off or not yet started) it is a no-op. + */ +class FlagEvalLoggingHook implements Hook { + + /** + * Singleton instance: always registered when the provider is created; harmless when writer=null + * (killswitch off or not yet started). + */ + static final FlagEvalLoggingHook INSTANCE = new FlagEvalLoggingHook<>(); + + /** + * Writer resolver. Production instances resolve through FeatureFlaggingGateway; tests can inject + * a direct writer or a resolver that simulates old-bootstrap linkage failures. + */ + private final Supplier writerSupplier; + + /** Production constructor - resolves writer from gateway. */ + FlagEvalLoggingHook() { + this(FeatureFlaggingGateway::getFlagEvalWriter); + } + + /** Test-only constructor - injects a writer directly, bypassing the gateway. */ + FlagEvalLoggingHook(final FlagEvaluationWriter writer) { + this(() -> writer); + } + + /** Test-only constructor - injects a writer resolver directly, bypassing the gateway. */ + FlagEvalLoggingHook(final Supplier writerSupplier) { + this.writerSupplier = writerSupplier; + } + + /** + * Capture + non-blocking enqueue only; no flattening, aggregation, or I/O. Runs at the finally + * stage so it covers success, error, and default-value paths. + * + *

The context snapshot taken here scales with context size/nesting - see the class javadoc for + * why it cannot be deferred. + */ + @Override + public void finallyAfter( + final HookContext ctx, + final FlagEvaluationDetails details, + final Map hints) { + try { + if (details == null) { + return; + } + if (!FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()) { + return; + } + + final FlagEvaluationWriter w = writerSupplier.get(); + if (w == null) { + return; + } + + // Pre-queue guard: if the queue is already saturated, avoid the context-copy work whose + // result would be discarded on offer(). Best-effort — the queue can still be full at + // offer() time — but flips a full snapshot into an O(1) check on the drop path. + if (!w.hasCapacityForEnqueue()) { + w.countPreQueueOverflow(); + return; + } + + // Scalar extraction - individual typed metadata reads, no JSON, no flattening + final String flagKey = details.getFlagKey(); + final ImmutableMetadata metadata = details.getFlagMetadata(); + + // allocationKey: "allocationKey" (camelCase) - consistent with FlagEvalMetricsHook.java + final String allocationKey = metadata != null ? metadata.getString("allocationKey") : null; + + // eval-time: from flag metadata "__dd_eval_timestamp_ms" (Long), fallback to hook-fire time. + // ImmutableMetadata.getLong available since sdk 1.4+. + final Long evalTimeObj = metadata != null ? metadata.getLong("__dd_eval_timestamp_ms") : null; + final long evalTimeMs = evalTimeObj != null ? evalTimeObj : System.currentTimeMillis(); + + // variant: the OpenFeature variant key (same source as the OTel FlagEvalMetricsHook), NOT the + // evaluated value. A null variant means no variant was selected (runtime default). + final String variant = details.getVariant(); + + // targetingKey from evaluation context + final String targetingKey = + ctx != null && ctx.getCtx() != null ? ctx.getCtx().getTargetingKey() : null; + + // Consent is read from metadata stamped by DDEvaluator (pinned to its ServerConfiguration). + // Missing key = non-DD provider → false, the privacy-preserving default. + final Boolean consentFromMetadata = + metadata != null + ? metadata.getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA) + : null; + final boolean observeFullEvaluationData = consentFromMetadata != null && consentFromMetadata; + + // Error message: prefer the human-readable message under consent-on; under consent-off the + // provider's raw message can echo evaluation-context values (e.g. NumberFormatException: + // "For input string: \"jane.doe@...\""), so replace it with the ErrorCode name — a stable, + // PII-free signal. Same substitution path is used when the message is absent regardless of + // consent (some providers populate only the code). Null on success. + String errorMessage = observeFullEvaluationData ? details.getErrorMessage() : null; + if ((errorMessage == null || errorMessage.isEmpty()) && details.getErrorCode() != null) { + errorMessage = details.getErrorCode().name(); + } + if (errorMessage != null && errorMessage.isEmpty()) { + errorMessage = null; + } + + // On the protected path (consent-off) the evaluation context is dropped on emit and never + // consulted by the aggregator, so skip the bounded copy entirely — the copy cost only + // applies under consent-on. + final Map attrs; + if (observeFullEvaluationData && ctx != null && ctx.getCtx() != null) { + // Bounded copy of the caller's mutable context (see DDEvaluator.copyPrunedContext for + // every retained-size cap). Runs inline because the event is consumed asynchronously + // and the source context is caller-owned; work is proportional to what is retained. + final DDEvaluator.CopyResult copy = DDEvaluator.copyPrunedContext(ctx.getCtx()); + if (copy.truncatedReason != null) { + w.countContextTruncated(copy.truncatedReason); + } + attrs = copy.attrs; + } else { + attrs = Collections.emptyMap(); + } + + w.enqueue( + new FlagEvalEvent( + flagKey, + variant, + allocationKey, + targetingKey, + errorMessage, + evalTimeMs, + observeFullEvaluationData, + attrs)); + } catch (LinkageError e) { + // Never let EVP recording break flag evaluation + } + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java similarity index 91% rename from products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java rename to products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java index 1132602a53f..a3e0dbfc83c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java @@ -7,11 +7,11 @@ import dev.openfeature.sdk.ImmutableMetadata; import java.util.Map; -class FlagEvalHook implements Hook { +class FlagEvalMetricsHook implements Hook { private final FlagEvalMetrics metrics; - FlagEvalHook(FlagEvalMetrics metrics) { + FlagEvalMetricsHook(FlagEvalMetrics metrics) { this.metrics = metrics; } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 38fa735d4e9..17009d9933a 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -36,7 +36,7 @@ public class Provider extends EventProvider implements Metadata { private final AtomicReference initializationState = new AtomicReference<>(InitializationState.NOT_STARTED); private final FlagEvalMetrics flagEvalMetrics; - private final FlagEvalHook flagEvalHook; + private final FlagEvalMetricsHook flagEvalMetricsHook; // Span enrichment: null unless the gate is on, so the feature has no idle overhead when off. private final SpanEnrichmentHook spanEnrichmentHook; // Precomputed hook list returned by getProviderHooks() on every evaluation. Immutable and built @@ -66,16 +66,16 @@ public Provider(final Options options) { this.options = options; this.evaluator = evaluator; FlagEvalMetrics metrics = null; - FlagEvalHook hook = null; + FlagEvalMetricsHook hook = null; try { metrics = new FlagEvalMetrics(); - hook = new FlagEvalHook(metrics); + hook = new FlagEvalMetricsHook(metrics); } catch (LinkageError | Exception e) { // This outer catch fires when the metrics helper itself can't load (OTel API absent). log.warn("Evaluation metrics unavailable — OTel API classes not on classpath", e); } this.flagEvalMetrics = metrics; - this.flagEvalHook = hook; + this.flagEvalMetricsHook = hook; // Span enrichment is wired ONLY when the gate is on — off means no capture hook and no idle // per-evaluation overhead. @@ -87,9 +87,19 @@ public Provider(final Options options) { // Precompute the immutable hook list once so getProviderHooks() (called on every evaluation) // allocates nothing, including when the gate is off. - final List hooks = new ArrayList<>(2); - if (flagEvalHook != null) { - hooks.add(flagEvalHook); + final List hooks = new ArrayList<>(3); + if (flagEvalMetricsHook != null) { + hooks.add(flagEvalMetricsHook); + } + // EVP flagevaluation hook: always registered; no-op when writer is absent (killswitch off). + // Writer is resolved lazily from FeatureFlaggingGateway.getFlagEvalWriter() on each call. + try { + final Hook flagEvalLoggingHook = buildFlagEvalLoggingHook(); + if (flagEvalLoggingHook != null) { + hooks.add(flagEvalLoggingHook); + } + } catch (LinkageError | Exception e) { + // Keep older bootstrap/API combinations working: EVP recording is best-effort. } if (spanEnrichmentHook != null) { hooks.add(spanEnrichmentHook); @@ -216,6 +226,10 @@ public List getProviderHooks() { return providerHooks; } + Hook buildFlagEvalLoggingHook() { + return FlagEvalLoggingHook.INSTANCE; + } + @Override public void shutdown() { if (flagEvalMetrics != null) { diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index 155c5eb1ea8..06d3d42af24 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -12,6 +12,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -25,8 +27,14 @@ import com.squareup.moshi.Types; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.Allocation; +import datadog.trace.api.featureflag.ufc.v1.ConditionConfiguration; +import datadog.trace.api.featureflag.ufc.v1.ConditionOperator; import datadog.trace.api.featureflag.ufc.v1.Flag; +import datadog.trace.api.featureflag.ufc.v1.Rule; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import datadog.trace.api.featureflag.ufc.v1.Split; +import datadog.trace.api.featureflag.ufc.v1.ValueType; +import datadog.trace.api.featureflag.ufc.v1.Variant; import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.MutableContext; @@ -199,7 +207,7 @@ public void testNoAllocations() { flags.put("null-allocation", new Flag("target", true, null, null, null)); flags.put("empty-allocation", new Flag("target", true, null, null, emptyList())); final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); - evaluator.accept(new ServerConfiguration("", "", null, flags)); + evaluator.accept(new ServerConfiguration("", "", false, null, flags)); final EvaluationContext ctx = new MutableContext("target").setTargetingKey("allocation"); @@ -214,6 +222,241 @@ public void testNoAllocations() { assertThat(details.getErrorCode(), nullValue()); } + // ---- observeFullEvaluationData metadata is stamped from the evaluator's ServerConfiguration + // ---- + // + // Every code path that returns a ProviderEvaluation must stamp the consent boolean so downstream + // hooks can honour it. These tests exercise each stamp site with both consent values (on/off) so + // a mutation to any stamp — deleting the line, hardcoding the value — flips at least one + // assertion. + + // -- success path: resolveVariant (variant metadata builder) -- + + @Test + public void observeFullEvaluationDataStampedTrueOnResolvedVariant() { + final ProviderEvaluation details = evaluateMatchingFlag(true); + + assertThat(details.getReason(), equalTo("STATIC")); + assertThat(details.getVariant(), equalTo("on")); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(true)); + } + + @Test + public void observeFullEvaluationDataStampedFalseOnResolvedVariant() { + // Symmetric consent-off assertion. Paired with the consent-on test above this pins the + // resolveVariant metadata line (DDEvaluator.java: METADATA_OBSERVE_FULL_EVALUATION_DATA) so + // deleting it or hardcoding either value would fail at least one assertion. + final ProviderEvaluation details = evaluateMatchingFlag(false); + + assertThat(details.getReason(), equalTo("STATIC")); + assertThat(details.getVariant(), equalTo("on")); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(false)); + } + + // -- DISABLED path: flag.enabled=false -- + + @Test + public void observeFullEvaluationDataStampedTrueOnDisabledFlag() { + final ProviderEvaluation details = evaluateDisabledFlag(true); + + assertThat(details.getReason(), equalTo("DISABLED")); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(true)); + } + + @Test + public void observeFullEvaluationDataStampedFalseOnDisabledFlag() { + final ProviderEvaluation details = evaluateDisabledFlag(false); + + assertThat(details.getReason(), equalTo("DISABLED")); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(false)); + } + + // -- DEFAULT path: no allocation matches -- + + @Test + public void observeFullEvaluationDataStampedTrueOnDefault() { + // Allocation exists but has empty splits, so the loop finishes without returning and we fall + // through to the DEFAULT branch. + final ProviderEvaluation details = evaluateWithEmptySplits(true); + + assertThat(details.getReason(), equalTo("DEFAULT")); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(true)); + } + + @Test + public void observeFullEvaluationDataStampedFalseOnDefault() { + final ProviderEvaluation details = evaluateWithEmptySplits(false); + + assertThat(details.getReason(), equalTo("DEFAULT")); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(false)); + } + + // -- error paths: FLAG_NOT_FOUND / PROVIDER_NOT_READY (via consentMetadata in error()) -- + + @Test + public void observeFullEvaluationDataStampedOnFlagNotFoundError() { + // Was previously named "…OnSuccess" but actually exercises the error() helper's stamp via + // FLAG_NOT_FOUND — kept for that stamp site, correctly named. + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); + evaluator.accept(new ServerConfiguration("", "", true, null, new HashMap<>())); + + final EvaluationContext ctx = new MutableContext("target").setTargetingKey("k"); + final ProviderEvaluation details = + evaluator.evaluate(Integer.class, "unknown-flag", 23, ctx); + + assertThat(details.getErrorCode(), equalTo(ErrorCode.FLAG_NOT_FOUND)); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(true)); + } + + @Test + public void observeFullEvaluationDataDefaultsToFalseWhenEvaluatorHasNoConfig() { + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); + final ProviderEvaluation details = + evaluator.evaluate(Integer.class, "test", 23, mock(EvaluationContext.class)); + assertThat(details.getErrorCode(), equalTo(ErrorCode.PROVIDER_NOT_READY)); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(false)); + } + + @Test + public void observeFullEvaluationDataNullConfigFieldTreatedAsFalse() { + // The field is boxed so Moshi tolerates a malformed consent value in the UFC JSON without + // aborting the whole parse. The evaluator must then interpret null as the privacy-preserving + // default. An auto-unbox at the read site (config.observeFullEvaluationData) would NPE here. + final Map flags = new HashMap<>(); + flags.put("target", new Flag("target", true, ValueType.INTEGER, emptyMap(), emptyList())); + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); + evaluator.accept(new ServerConfiguration("", "", null, null, flags)); + + final EvaluationContext ctx = new MutableContext("target").setTargetingKey("k"); + final ProviderEvaluation details = evaluator.evaluate(Integer.class, "target", 23, ctx); + + // Flags still evaluate — availability preserved despite the malformed consent field. + assertThat(details.getReason(), equalTo("DEFAULT")); + assertThat( + details.getFlagMetadata().getBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA), + equalTo(false)); + } + + // Builds a flag that reaches resolveVariant: enabled, one allocation with no rules, one split + // with empty shards (so the shard-match branch is skipped and the split is picked immediately), + // and a single "on" variant whose value maps to the requested Integer type. + private static ProviderEvaluation evaluateMatchingFlag( + final boolean observeFullEvaluationData) { + final Map variations = new HashMap<>(); + variations.put("on", new Variant("on", 1)); + final Split split = new Split(emptyList(), "on", emptyMap(), null); + final Allocation allocation = + new Allocation("alloc-1", null, null, null, singletonList(split), Boolean.FALSE); + return evaluateFlag( + new Flag("target", true, ValueType.INTEGER, variations, singletonList(allocation)), + observeFullEvaluationData); + } + + private static ProviderEvaluation evaluateDisabledFlag( + final boolean observeFullEvaluationData) { + return evaluateFlag( + new Flag("target", false, ValueType.INTEGER, emptyMap(), null), observeFullEvaluationData); + } + + private static ProviderEvaluation evaluateWithEmptySplits( + final boolean observeFullEvaluationData) { + // Enabled, allocations present, allocation active, no rules, empty splits → falls through the + // for-loop to the DEFAULT return. + final Allocation allocation = + new Allocation("alloc-1", null, null, null, emptyList(), Boolean.FALSE); + return evaluateFlag( + new Flag("target", true, ValueType.INTEGER, emptyMap(), singletonList(allocation)), + observeFullEvaluationData); + } + + private static ProviderEvaluation evaluateFlag( + final Flag flag, final boolean observeFullEvaluationData) { + final Map flags = new HashMap<>(); + flags.put("target", flag); + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); + evaluator.accept(new ServerConfiguration("", "", observeFullEvaluationData, null, flags)); + + final EvaluationContext ctx = new MutableContext("target").setTargetingKey("user-1"); + return evaluator.evaluate(Integer.class, "target", 23, ctx); + } + + // ---- error message redaction respects observeFullEvaluationData ---- + + @Test + public void numericConditionOnTargetingKeyDropsExceptionMessageUnderConsentOff() { + // Rule {attribute:"id", operator:GT, value:0} + "id" not in context → + // DDEvaluator.resolveAttribute + // falls back to the targeting key, so Double.parseDouble("jane.doe@datadoghq.com") throws + // NumberFormatException. The exception message echoes the raw context value verbatim, so it + // must be dropped when observeFullEvaluationData=false. + final ProviderEvaluation details = + evaluateWithNumericRuleOnId("jane.doe@datadoghq.com", false); + + assertThat(details.getErrorCode(), equalTo(ErrorCode.TYPE_MISMATCH)); + assertNull(details.getErrorMessage(), "consent-off must not surface the raw exception message"); + } + + @Test + public void numericConditionOnTargetingKeyPreservesExceptionMessageUnderConsentOn() { + // Symmetric case: with consent on, the raw exception message flows through unchanged so + // operators keep the diagnostic detail they opted in to. + final ProviderEvaluation details = + evaluateWithNumericRuleOnId("jane.doe@datadoghq.com", true); + + assertThat(details.getErrorCode(), equalTo(ErrorCode.TYPE_MISMATCH)); + assertThat(details.getErrorMessage(), equalTo("For input string: \"jane.doe@datadoghq.com\"")); + } + + @Test + public void numericConditionOnTargetingKeyErrorMessageNeverContainsPiiUnderConsentOff() { + // Belt-and-suspenders: independent of the exact null/empty form, the raw PII value must never + // appear in the message under consent-off. Guards against future changes that might replace + // null with a redacted string or a code-name suffix. + final ProviderEvaluation details = + evaluateWithNumericRuleOnId("jane.doe@datadoghq.com", false); + + final String message = details.getErrorMessage(); + assertFalse( + message != null && message.contains("jane.doe@datadoghq.com"), + "consent-off errorMessage must not contain raw context values"); + } + + private static ProviderEvaluation evaluateWithNumericRuleOnId( + final String targetingKey, final boolean observeFullEvaluationData) { + final Map flags = new HashMap<>(); + final List rules = + singletonList( + new Rule(singletonList(new ConditionConfiguration(ConditionOperator.GT, "id", 0)))); + // Split must be non-empty so the allocation is considered a match target; its contents don't + // matter because the rule throws before a split is picked. + final Allocation allocation = + new Allocation("alloc", rules, null, null, emptyList(), Boolean.FALSE); + flags.put( + "num-rule", + new Flag("num-rule", true, ValueType.INTEGER, emptyMap(), singletonList(allocation))); + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); + evaluator.accept(new ServerConfiguration("", "", observeFullEvaluationData, null, flags)); + + final EvaluationContext ctx = new MutableContext(targetingKey); + return evaluator.evaluate(Integer.class, "num-rule", 23, ctx); + } + @Test public void testAllocationDateAbiAndInstantAccessors() throws Exception { final Date startAt = Date.from(Instant.parse("2024-01-01T00:00:00Z")); @@ -256,6 +499,9 @@ private static Arguments[] flatteningTestCases() { Arguments.of( mapOf("map", mapOf("key1", 1, "key2", 2, "key3", mapOf("key4", 4))), mapOf("map.key1", 1, "map.key2", 2, "map.key3.key4", 4))); + arguments.add( + Arguments.of( + mapOf("plan", "gold", "cohort", "gold"), mapOf("plan", "gold", "cohort", "gold"))); final Instant instant = Instant.parse("2026-07-10T12:34:56Z"); arguments.add(Arguments.of(mapOf("instant", instant), mapOf("instant", instant.toString()))); return arguments.toArray(new Arguments[0]); @@ -275,6 +521,162 @@ public void testFlattening( } } + @Test + public void testDeeplyNestedContextIsTruncatedRatherThanOverflowingTheStack() { + Value nested = new Value("leaf"); + for (int i = 0; i < 10_000; i++) { + nested = new Value(singletonList(nested)); + } + final EvaluationContext context = new MutableContext().add("deep", singletonList(nested)); + + final Map result = DDEvaluator.flattenContext(context); + + final StringBuilder truncatedKey = new StringBuilder("deep"); + for (int i = 0; i < DDEvaluator.MAX_SNAPSHOT_DEPTH; i++) { + truncatedKey.append("[0]"); + } + assertThat(result.size(), equalTo(1)); + assertThat(result, hasEntry(truncatedKey.toString(), null)); + } + + @Test + public void testCopyPrunedContextCapsTopLevelFieldCount() { + final MutableContext context = new MutableContext(); + for (int i = 0; i < DDEvaluator.MAX_CONTEXT_FIELDS + 100; i++) { + context.add(String.format("k%04d", i), "v"); + } + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + assertThat(result.attrs.size(), equalTo(DDEvaluator.MAX_CONTEXT_FIELDS)); + assertThat(result.truncatedReason, equalTo("max_context_fields")); + } + + @Test + public void testCopyPrunedContextSkipsOversizedStringValues() { + final char[] longChars = new char[DDEvaluator.MAX_VALUE_LENGTH + 1]; + java.util.Arrays.fill(longChars, 'x'); + final MutableContext context = new MutableContext(); + context.add("keep", "ok"); + context.add("drop", new String(longChars)); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + assertThat(result.attrs, hasEntry("keep", "ok")); + assertThat(result.attrs.containsKey("drop"), equalTo(false)); + assertThat(result.truncatedReason, equalTo("max_value_length")); + } + + @Test + public void testCopyPrunedContextSkipsOversizedKeys() { + final char[] longKeyChars = new char[DDEvaluator.MAX_KEY_LENGTH + 1]; + java.util.Arrays.fill(longKeyChars, 'k'); + final MutableContext context = new MutableContext(); + context.add("keep", "ok"); + context.add(new String(longKeyChars), "drop"); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + assertThat(result.attrs, hasEntry("keep", "ok")); + assertThat(result.attrs.size(), equalTo(1)); + assertThat(result.truncatedReason, equalTo("max_key_length")); + } + + @Test + public void testCopyPrunedContextCapsListWidth() { + final List wide = new java.util.ArrayList<>(); + for (int i = 0; i < DDEvaluator.MAX_LIST_ELEMENTS + 50; i++) { + wide.add(Value.objectToValue("v" + i)); + } + final EvaluationContext context = new MutableContext().add("list", wide); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + assertThat(result.attrs.containsKey("list[0]"), equalTo(true)); + assertThat( + result.attrs.containsKey("list[" + (DDEvaluator.MAX_LIST_ELEMENTS - 1) + "]"), + equalTo(true)); + assertThat( + result.attrs.containsKey("list[" + DDEvaluator.MAX_LIST_ELEMENTS + "]"), equalTo(false)); + assertThat(result.truncatedReason, equalTo("max_list_elements")); + } + + @Test + public void testCopyPrunedContextCapsStructureWidth() { + final dev.openfeature.sdk.MutableStructure wide = new dev.openfeature.sdk.MutableStructure(); + for (int i = 0; i < DDEvaluator.MAX_STRUCTURE_PROPERTIES + 50; i++) { + wide.add(String.format("p%04d", i), "v"); + } + final EvaluationContext context = new MutableContext().add("struct", wide); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + long structKeys = result.attrs.keySet().stream().filter(k -> k.startsWith("struct.")).count(); + assertThat(structKeys, equalTo((long) DDEvaluator.MAX_STRUCTURE_PROPERTIES)); + assertThat(result.truncatedReason, equalTo("max_structure_properties")); + } + + @Test + public void testCopyPrunedContextTruncatesDeepNesting() { + Value nested = new Value("leaf"); + for (int i = 0; i < 10_000; i++) { + nested = new Value(singletonList(nested)); + } + final EvaluationContext context = new MutableContext().add("deep", singletonList(nested)); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + final StringBuilder truncatedKey = new StringBuilder("deep"); + for (int i = 0; i < DDEvaluator.MAX_SNAPSHOT_DEPTH; i++) { + truncatedKey.append("[0]"); + } + // The recursion stops on the first list element at MAX_SNAPSHOT_DEPTH; deeper elements are + // never walked and no entry is emitted for them. + assertThat(result.attrs.containsKey(truncatedKey.toString()), equalTo(false)); + assertThat(result.attrs.size(), equalTo(0)); + assertThat(result.truncatedReason, equalTo("max_snapshot_depth")); + } + + @Test + public void testCopyPrunedContextExcludesTargetingKey() { + final MutableContext context = new MutableContext("user-42").add("region", "us-east-1"); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + assertThat(result.attrs, hasEntry("region", "us-east-1")); + assertThat(result.attrs.containsKey("targetingKey"), equalTo(false)); + assertThat(result.truncatedReason, equalTo(null)); + } + + @Test + public void testCopyPrunedContextNoTruncationReturnsNullReason() { + final MutableContext context = new MutableContext("user-1"); + context.add("region", "us-east-1"); + context.add("tier", "gold"); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + assertThat(result.attrs, hasEntry("region", "us-east-1")); + assertThat(result.truncatedReason, equalTo(null)); + } + + @Test + public void testCopyPrunedContextMultipleReasonsAreSortedAndDeduplicated() { + final char[] longChars = new char[DDEvaluator.MAX_VALUE_LENGTH + 1]; + java.util.Arrays.fill(longChars, 'x'); + final char[] longKeyChars = new char[DDEvaluator.MAX_KEY_LENGTH + 1]; + java.util.Arrays.fill(longKeyChars, 'k'); + final MutableContext context = new MutableContext(); + context.add("keep", "ok"); + context.add("dropValue", new String(longChars)); + context.add(new String(longKeyChars), "dropKey"); + + final DDEvaluator.CopyResult result = DDEvaluator.copyPrunedContext(context); + + // Both max_key_length and max_value_length fired; sorted alphabetically, no duplicates. + assertThat(result.truncatedReason, equalTo("max_key_length,max_value_length")); + } + @Test public void testCanonicalFixturesArePresent() throws IOException { assertThat(canonicalTestCases().size(), greaterThan(0)); diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java new file mode 100644 index 00000000000..3b7910d4712 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java @@ -0,0 +1,720 @@ +package datadog.trace.api.openfeature; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.FlagValueType; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.MutableStructure; +import dev.openfeature.sdk.Reason; +import dev.openfeature.sdk.Value; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link FlagEvalLoggingHook}: context capture, non-blocking enqueue, eval-time + * metadata, absent-variant detection, and killswitch-via-writer-null behaviour. + */ +class FlagEvalLoggingHookTest { + + @BeforeEach + void enableFlagEvaluationEnqueue() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + + @AfterEach + void resetFlagEvaluationEnqueue() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + // Clear any dispatched UFC so an observeFullEvaluationData value can't leak into other tests + // that share the static gateway. + FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + } + + // ---- helpers ---- + + /** + * Creates a writer that captures the enqueued event for assertion. Uses an anonymous class since + * FlagEvaluationWriter has multiple abstract methods. + */ + private FlagEvaluationWriter capturingWriter(final AtomicReference ref) { + return new FlagEvaluationWriter() { + @Override + public void enqueue(final FlagEvalEvent event) { + ref.set(event); + } + + @Override + public boolean hasCapacityForEnqueue() { + return true; + } + + @Override + public void countPreQueueOverflow() {} + + @Override + public void countContextTruncated(final String reason) {} + + @Override + public void start() {} + + @Override + public void close() {} + }; + } + + private static FlagEvalLoggingHook hookWithWriter(final FlagEvaluationWriter writer) { + return new FlagEvalLoggingHook<>(writer); + } + + private static FlagEvaluationDetails details( + final String flagKey, + final Object value, + final String variant, + final String reason, + final ImmutableMetadata metadata) { + final FlagEvaluationDetails.FlagEvaluationDetailsBuilder builder = + FlagEvaluationDetails.builder().flagKey(flagKey).value(value).reason(reason); + if (variant != null) { + builder.variant(variant); + } + if (metadata != null) { + builder.flagMetadata(metadata); + } + return builder.build(); + } + + private static HookContext hookCtxWithTargetingKey( + final String flagKey, final String targetingKey) { + final MutableContext ctx = new MutableContext(targetingKey); + return HookContext.builder() + .flagKey(flagKey) + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .build(); + } + + // ---- test: hook calls writer.enqueue once with flagKey, variant, allocationKey ---- + + @Test + void finallyAfterEnqueuesEventWithAllBasicFields() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details( + "my-flag", + "on-value", + "on", + Reason.TARGETING_MATCH.name(), + ImmutableMetadata.builder().addString("allocationKey", "alloc-1").build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get(), "writer.enqueue must be called once"); + final FlagEvalEvent e = captured.get(); + assertEquals("my-flag", e.flagKey); + assertEquals("on", e.variant, "variant must be the OpenFeature variant key"); + assertEquals("alloc-1", e.allocationKey); + } + + // ---- variant comes from details.getVariant(), NOT details.getValue() ---- + + @Test + void variantIsTheVariantKeyNotTheEvaluatedValue() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + // value and variant DIFFER, so a value-vs-variant mistake is detectable. + final FlagEvaluationDetails det = + details( + "g1-flag", + "the-evaluated-value", // value + "the-variant-key", // variant + Reason.TARGETING_MATCH.name(), + null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "the-variant-key", + captured.get().variant, + "variant must be sourced from details.getVariant(), not details.getValue()"); + } + + // ---- test: evalTimeMs from metadata "__dd_eval_timestamp_ms" ---- + + @Test + void evalTimeMsComesFromMetadataWhenPresent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final long expectedTimestamp = 1_700_000_000_000L; + final FlagEvaluationDetails det = + details( + "ts-flag", + "v", + "v", + Reason.SPLIT.name(), + ImmutableMetadata.builder() + .addString("allocationKey", "a") + .addLong("__dd_eval_timestamp_ms", expectedTimestamp) + .build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + expectedTimestamp, + captured.get().evalTimeMs, + "evalTimeMs must come from __dd_eval_timestamp_ms metadata when present"); + } + + // ---- test: evalTimeMs falls back to System.currentTimeMillis() when absent ---- + + @Test + void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final long before = System.currentTimeMillis(); + final FlagEvaluationDetails det = + details("ts-flag", "v", "v", Reason.SPLIT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + final long after = System.currentTimeMillis(); + assertNotNull(captured.get()); + final long ts = captured.get().evalTimeMs; + assertTrue( + ts >= before && ts <= after, + "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); + } + + // ---- test: absent variant -> variant is null -> runtime default ---- + + @Test + void absentVariantProducesNullVariant() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + // A runtime default returns the default value but no variant. + final FlagEvaluationDetails det = + details("def-flag", "default-value", null, Reason.DEFAULT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().variant, "Absent variant must stay null (runtime default)"); + } + + // ---- test: error message captured from details (error object support) ---- + + @Test + void errorMessageCapturedFromDetailsUnderConsentOn() { + // With observeFullEvaluationData=true the provider's raw message is preserved verbatim so + // operators keep the diagnostic detail they opted in to. + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.TYPE_MISMATCH) + .errorMessage("value does not match declared type") + .flagMetadata(consentOnMetadata()) + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "value does not match declared type", + captured.get().errorMessage, + "errorMessage must be captured from the evaluation details under consent-on"); + } + + @Test + void errorMessageReplacedByErrorCodeUnderConsentOff() { + // Defense-in-depth: even if a provider hands us a raw message under consent-off (a bug in the + // provider, or a third-party provider that doesn't distinguish consent tiers), the hook must + // substitute the ErrorCode name so raw PII from exception messages never reaches the wire. + // Uses a PII-looking marker exactly like the wire-level guards in FlagEvaluationWriterImplTest. + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.TYPE_MISMATCH) + .errorMessage("For input string: \"jane.doe@datadoghq.com\"") + .flagMetadata(consentOffMetadata()) + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "TYPE_MISMATCH", + captured.get().errorMessage, + "consent-off must replace the raw message with the ErrorCode name"); + assertFalse( + captured.get().errorMessage.contains("jane.doe@datadoghq.com"), + "raw PII must never survive into the enqueued event under consent-off"); + } + + @Test + void errorMessageDroppedWhenConsentOffAndNoErrorCode() { + // Edge case: no ErrorCode available (unusual — providers should set one for ERROR reason). + // Consent-off drops the message and there's nothing to substitute, so the enqueued event has + // no error message at all. This is the strictest privacy-preserving outcome. + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorMessage("For input string: \"jane.doe@datadoghq.com\"") + .flagMetadata(consentOffMetadata()) + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().errorMessage); + } + + // ---- test: error code used as fallback message when error message is empty ---- + + @Test + void errorCodeUsedAsFallbackWhenMessageEmpty() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.FLAG_NOT_FOUND) + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "FLAG_NOT_FOUND", + captured.get().errorMessage, + "error code name must be used when no error message is present"); + } + + // ---- test: success path has no error message ---- + + @Test + void successPathHasNullErrorMessage() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details("ok-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().errorMessage, "success path must have no error message"); + } + + // ---- test: hook does NO aggregation on the hook thread ---- + + @Test + void finallyAfterOnlyCallsEnqueueAndCapacityCheckOnWriter() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + when(writer.hasCapacityForEnqueue()).thenReturn(true); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + // Capacity check gates the enqueue; exactly one enqueue call; no start/close. + verify(writer, times(1)).hasCapacityForEnqueue(); + verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); + verify(writer, never()).countPreQueueOverflow(); + verify(writer, never()).countContextTruncated(any()); + verify(writer, never()).close(); + verify(writer, never()).start(); + } + + @Test + void finallyAfterSkipsEnqueueAndCountsDropWhenQueueSaturated() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + when(writer.hasCapacityForEnqueue()).thenReturn(false); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + // Pre-queue guard fires: one capacity check, one drop count, and no enqueue at all. + verify(writer, times(1)).hasCapacityForEnqueue(); + verify(writer, times(1)).countPreQueueOverflow(); + verify(writer, never()).enqueue(any(FlagEvalEvent.class)); + } + + @Test + void countContextTruncatedCalledWhenContextIsTruncated() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + when(writer.hasCapacityForEnqueue()).thenReturn(true); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + // Context with one oversized string value — triggers max_value_length. + final char[] longChars = new char[DDEvaluator.MAX_VALUE_LENGTH + 1]; + java.util.Arrays.fill(longChars, 'x'); + final MutableContext ctx = new MutableContext("user-1"); + ctx.add("oversized", new String(longChars)); + final HookContext hookCtx = + HookContext.builder() + .flagKey("flag") + .type(dev.openfeature.sdk.FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .build(); + final FlagEvaluationDetails det = + details( + "flag", + "v", + "v", + dev.openfeature.sdk.Reason.TARGETING_MATCH.name(), + consentOnMetadata()); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + verify(writer, times(1)).countContextTruncated("max_value_length"); + verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); + } + + @Test + void countContextTruncatedNotCalledWhenNoTruncation() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + when(writer.hasCapacityForEnqueue()).thenReturn(true); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + final MutableContext ctx = new MutableContext("user-1"); + ctx.add("region", "us-east-1"); + final HookContext hookCtx = + HookContext.builder() + .flagKey("flag") + .type(dev.openfeature.sdk.FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .build(); + final FlagEvaluationDetails det = + details( + "flag", + "v", + "v", + dev.openfeature.sdk.Reason.TARGETING_MATCH.name(), + consentOnMetadata()); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + verify(writer, never()).countContextTruncated(any()); + verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); + } + + @Test + void enqueueDisabledIsNoOpBeforeWriterLookup() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + final AtomicReference writerResolved = new AtomicReference<>(false); + final FlagEvalLoggingHook hook = + new FlagEvalLoggingHook<>( + () -> { + writerResolved.set(true); + throw new AssertionError("writer should not be resolved when enqueue is disabled"); + }); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtxWithTargetingKey("flag", "user-1"), det, Collections.emptyMap()); + + assertFalse(writerResolved.get()); + } + + // ---- test: writer=null -> no-op (killswitch off / not yet started) ---- + + @Test + void writerNullIsNoOp() { + final FlagEvalLoggingHook hook = hookWithWriter(null); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + // Must not throw; nothing is enqueued + hook.finallyAfter(null, det, Collections.emptyMap()); + } + + @Test + void writerLookupLinkageErrorIsNoOp() { + final FlagEvalLoggingHook hook = + new FlagEvalLoggingHook<>( + () -> { + throw new NoSuchMethodError("old bootstrap"); + }); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + assertDoesNotThrow(() -> hook.finallyAfter(null, det, Collections.emptyMap())); + } + + // ---- test: details=null -> no-op ---- + + @Test + void detailsNullIsNoOp() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + // Should not throw + hook.finallyAfter(null, null, Collections.emptyMap()); + + verifyNoInteractions(writer); + } + + // ---- test: targetingKey extracted from evaluation context ---- + + @Test + void targetingKeyExtractedFromContext() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.SPLIT.name(), null); + + final HookContext hookCtx = hookCtxWithTargetingKey("ctx-flag", "user-42"); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "user-42", + captured.get().targetingKey, + "targetingKey must be extracted from the evaluation context"); + } + + @Test + void contextAttributesAreFlattenedAndConvertedInline() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final Map profile = new HashMap<>(); + profile.put("tier", "gold"); + final Map attributes = new HashMap<>(); + attributes.put("score", 42); + attributes.put("profile", profile); + final MutableContext context = + new MutableContext(Value.objectToValue(attributes).asStructure().asMap()); + context.setTargetingKey("user-42"); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), consentOnMetadata()); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + // Flatten now happens inline in DDEvaluator#copyPrunedContext on the hot path, so the event + // arrives with attrs already populated (no deferred supplier). + final Map attrs = captured.get().attrs; + assertFalse(attrs.isEmpty(), "hook must flatten context inline"); + assertEquals(42, attrs.get("score")); + assertEquals("gold", attrs.get("profile.tier")); + assertFalse(attrs.containsKey("targetingKey")); + assertTrue( + attrs.values().stream().noneMatch(Value.class::isInstance), + "context attrs must contain converted scalar values, not OpenFeature Value wrappers"); + } + + @Test + void contextAttributesUseEnqueueTimeSnapshot() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final MutableContext context = new MutableContext("user-42"); + context.add("region", "us-east-1"); + final MutableStructure profile = new MutableStructure(); + profile.add("tier", "gold"); + context.add("profile", profile); + final List cohorts = new ArrayList<>(); + cohorts.add(Value.objectToValue("beta")); + context.add("cohorts", cohorts); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), consentOnMetadata()); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + context.add("region", "eu-west-1"); + context.add("late", "ignored"); + profile.add("tier", "platinum"); + profile.add("late", "ignored"); + cohorts.set(0, Value.objectToValue("ga")); + cohorts.add(Value.objectToValue("late")); + + assertNotNull(captured.get()); + final Map attrs = captured.get().attrs; + assertEquals("us-east-1", attrs.get("region")); + assertEquals("gold", attrs.get("profile.tier")); + assertEquals("beta", attrs.get("cohorts[0]")); + assertFalse(attrs.containsKey("late")); + assertFalse(attrs.containsKey("profile.late")); + assertFalse(attrs.containsKey("cohorts[1]")); + } + + // ---- observeFullEvaluationData is read from evaluation metadata, never the gateway ---- + + @Test + void readsObserveFullEvaluationDataTrueFromEvaluationMetadata() { + assertTrue(enqueuedEventWithConsentMetadata(true).observeFullEvaluationData); + } + + @Test + void readsObserveFullEvaluationDataFalseFromEvaluationMetadata() { + assertFalse(enqueuedEventWithConsentMetadata(false).observeFullEvaluationData); + } + + @Test + void observeFullEvaluationDataDefaultsToFalseWhenMetadataAbsent() { + // No metadata at all: fail-closed toward privacy. + assertFalse(enqueuedEventWithConsentMetadata(null).observeFullEvaluationData); + } + + @Test + void protectedPathSkipsEvaluationContextCapture() { + // Consent off → the hook must not snapshot the evaluation context at all. Verified by mutating + // the context after finallyAfter returns and asserting the enqueued event still sees nothing. + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final MutableContext context = new MutableContext("user-1"); + context.add("region", "us-east-1"); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final ImmutableMetadata consentOff = + ImmutableMetadata.builder() + .addBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA, false) + .build(); + + hook.finallyAfter( + hookCtx, + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), consentOff), + Collections.emptyMap()); + context.add("region", "eu-west-1"); + + assertNotNull(captured.get()); + assertTrue(captured.get().attrs.isEmpty()); + } + + @Test + void ignoresGatewayConsentEvenWhenItDisagreesWithMetadata() { + // Gateway says on, metadata says off; hook must trust metadata. + FeatureFlaggingGateway.dispatch(observeConfig(true)); + try { + assertFalse(enqueuedEventWithConsentMetadata(false).observeFullEvaluationData); + } finally { + FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + } + } + + /** + * Fires the hook once for a simple targeted evaluation whose metadata carries the given consent + * value ({@code null} = key absent) and returns the enqueued event. + */ + private FlagEvalEvent enqueuedEventWithConsentMetadata(final Boolean consent) { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + final ImmutableMetadata metadata = + consent == null + ? null + : ImmutableMetadata.builder() + .addBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA, consent) + .build(); + hook.finallyAfter( + hookCtxWithTargetingKey("obs-flag", "user-1"), + details("obs-flag", "on", "on", Reason.TARGETING_MATCH.name(), metadata), + Collections.emptyMap()); + assertNotNull(captured.get(), "writer.enqueue must be called once"); + return captured.get(); + } + + private static ImmutableMetadata consentOnMetadata() { + return ImmutableMetadata.builder() + .addBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA, true) + .build(); + } + + private static ImmutableMetadata consentOffMetadata() { + return ImmutableMetadata.builder() + .addBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA, false) + .build(); + } + + private static ServerConfiguration observeConfig(final boolean observeFullEvaluationData) { + return new ServerConfiguration( + "2024-04-17T19:40:53.716Z", + "SERVER", + observeFullEvaluationData, + null, + Collections.emptyMap()); + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java similarity index 89% rename from products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java rename to products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java index 8ed17d91cbb..322a06d2de7 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java @@ -13,12 +13,12 @@ import java.util.Collections; import org.junit.jupiter.api.Test; -class FlagEvalHookTest { +class FlagEvalMetricsHookTest { @Test void finallyAfterRecordsBasicEvaluation() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -44,7 +44,7 @@ void finallyAfterRecordsBasicEvaluation() { @Test void finallyAfterRecordsErrorEvaluation() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -68,7 +68,7 @@ void finallyAfterRecordsErrorEvaluation() { @Test void finallyAfterHandlesNullFlagMetadata() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -87,7 +87,7 @@ void finallyAfterHandlesNullFlagMetadata() { @Test void finallyAfterHandlesNullVariantAndReason() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder().flagKey("my-flag").value("default").build(); @@ -100,7 +100,7 @@ void finallyAfterHandlesNullVariantAndReason() { @Test void finallyAfterNeverThrows() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); // Should not throw even with completely null inputs hook.finallyAfter(null, null, null); @@ -110,7 +110,7 @@ void finallyAfterNeverThrows() { @Test void finallyAfterIsNoOpWhenMetricsIsNull() { - FlagEvalHook hook = new FlagEvalHook(null); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(null); FlagEvaluationDetails details = FlagEvaluationDetails.builder() diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index ad239a5a70b..9623a4a479a 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -17,6 +17,8 @@ import static org.mockito.Mockito.when; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import datadog.trace.api.openfeature.Provider.Options; import dev.openfeature.sdk.Client; @@ -25,6 +27,9 @@ import dev.openfeature.sdk.EventDetails; import dev.openfeature.sdk.Features; import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.OpenFeatureAPI; import dev.openfeature.sdk.ProviderEvaluation; import dev.openfeature.sdk.ProviderEvent; @@ -33,6 +38,7 @@ import dev.openfeature.sdk.exceptions.FatalError; import dev.openfeature.sdk.exceptions.ProviderNotReadyError; import java.lang.reflect.Field; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -66,6 +72,8 @@ public void tearDown() { executor.shutdownNow(); OpenFeatureAPI.getInstance().shutdown(); FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test @@ -325,15 +333,81 @@ protected Class loadEvaluatorClass() throws ClassNotFoundException { } @Test - public void testGetProviderHooksReturnsFlagEvalHook() { + public void testGetProviderHooksReturnsFlagEvalMetricsHook() { + Provider provider = + new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)); + List hooks = provider.getProviderHooks(); + // Two hooks: OTel FlagEvalMetricsHook (index 0) + FlagEvalLoggingHook (index 1) + assertThat(hooks.size(), equalTo(2)); + assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); + assertThat(hooks.get(1) instanceof FlagEvalLoggingHook, equalTo(true)); + } + + @Test + public void testGetProviderHooksSkipsFlagEvalLoggingHookOnLinkageFailure() { + Provider provider = + new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)) { + @Override + Hook buildFlagEvalLoggingHook() { + throw new NoClassDefFoundError("old bootstrap"); + } + }; + + List hooks = provider.getProviderHooks(); + + assertThat(hooks.size(), equalTo(1)); + assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); + } + + @Test + public void testClientEvaluationRoutesThroughFlagEvalLoggingHook() throws Exception { + FeatureFlaggingGateway.dispatch(mock(ServerConfiguration.class)); + final AtomicReference captured = new AtomicReference<>(); + FeatureFlaggingGateway.setFlagEvalWriter(capturingWriter(captured)); + final Evaluator evaluator = mock(Evaluator.class); + when(evaluator.initialize(eq(10L), eq(SECONDS), any())).thenReturn(true); + when(evaluator.hasConfiguration()).thenReturn(true); + when(evaluator.evaluate(eq(String.class), eq("logged-flag"), eq("default"), any())) + .thenReturn( + ProviderEvaluation.builder() + .value("value") + .reason("STATIC") + .variant("variant-1") + .flagMetadata( + ImmutableMetadata.builder() + .addString("allocationKey", "allocation-1") + .addLong("__dd_eval_timestamp_ms", 1_700_000_000_000L) + .addBoolean(DDEvaluator.METADATA_OBSERVE_FULL_EVALUATION_DATA, true) + .build()) + .build()); + final OpenFeatureAPI api = OpenFeatureAPI.getInstance(); + api.setProviderAndWait(new Provider(new Options().initTimeout(10, SECONDS), evaluator)); + final MutableContext context = new MutableContext("user-1"); + context.add("region", "us-east-1"); + + final FlagEvaluationDetails details = + api.getClient().getStringDetails("logged-flag", "default", context); + + assertThat(details.getValue(), equalTo("value")); + final FlagEvalEvent event = captured.get(); + assertThat(event.flagKey, equalTo("logged-flag")); + assertThat(event.variant, equalTo("variant-1")); + assertThat(event.allocationKey, equalTo("allocation-1")); + assertThat(event.targetingKey, equalTo("user-1")); + assertThat(event.evalTimeMs, equalTo(1_700_000_000_000L)); + assertThat(event.attrs.get("region"), equalTo("us-east-1")); + } + + @Test + public void testGetProviderHooksReturnsFlagEvalMetricsHookWithAndWithoutSpanEnrichment() { final Evaluator evaluator = mock(Evaluator.class); final Provider providerWithoutSpanEnrichment = new Provider(new Options(), evaluator, Boolean.FALSE); final Provider providerWithSpanEnrichment = new Provider(new Options(), evaluator, Boolean.TRUE); - assertHasFlagEvalHook(providerWithoutSpanEnrichment); - assertHasFlagEvalHook(providerWithSpanEnrichment); + assertHasFlagEvalMetricsHook(providerWithoutSpanEnrichment); + assertHasFlagEvalMetricsHook(providerWithSpanEnrichment); } @Test @@ -348,11 +422,13 @@ public void testShutdownCleansUpEvaluator() throws Exception { provider.shutdown(); verify(evaluator).shutdown(); + // After shutdown, getProviderHooks still returns a list with both OTel + logging hooks + assertThat(provider.getProviderHooks().size(), equalTo(2)); } - private static void assertHasFlagEvalHook(final Provider provider) { + private static void assertHasFlagEvalMetricsHook(final Provider provider) { assertTrue( - provider.getProviderHooks().stream().anyMatch(FlagEvalHook.class::isInstance), + provider.getProviderHooks().stream().anyMatch(FlagEvalMetricsHook.class::isInstance), "flag evaluation metrics hook should be registered"); } @@ -402,4 +478,30 @@ private static String initializationState(final Provider provider) throws Except final AtomicReference state = (AtomicReference) stateField.get(provider); return state.get().toString(); } + + private static FlagEvaluationWriter capturingWriter(final AtomicReference ref) { + return new FlagEvaluationWriter() { + @Override + public void enqueue(final FlagEvalEvent event) { + ref.set(event); + } + + @Override + public boolean hasCapacityForEnqueue() { + return true; + } + + @Override + public void countPreQueueOverflow() {} + + @Override + public void countContextTruncated(final String reason) {} + + @Override + public void start() {} + + @Override + public void close() {} + }; + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index c8f5625c855..2a823bd32ef 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -1,6 +1,7 @@ package datadog.trace.api.featureflag; import datadog.trace.api.featureflag.exposure.ExposureEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -28,6 +29,17 @@ public interface SpanEnrichmentListener extends Consumer {} private static final AtomicReference CURRENT_CONFIG = new AtomicReference<>(); + /** + * The active EVP flagevaluation writer. Registered by {@code FlagEvaluationWriterImpl.start()} + * when the killswitch {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED} is on (default). Read by + * {@code FlagEvalLoggingHook} to route evaluations into the two-tier aggregator. {@code null} + * when the EVP path is disabled. + */ + private static final AtomicReference FLAG_EVAL_WRITER = + new AtomicReference<>(); + + private static volatile boolean flagEvalEnqueueEnabled = true; + private FeatureFlaggingGateway() {} public static void addConfigListener(final ConfigListener listener) { @@ -72,6 +84,39 @@ public static void dispatch(final ExposureEvent event) { EXPOSURE_LISTENERS.forEach(listener -> listener.accept(event)); } + /** + * Registers the active EVP flagevaluation writer. Called by {@code + * FlagEvaluationWriterImpl.start()} when the feature is enabled. Replaces any previously + * registered writer. + * + * @param writer the writer to register, or {@code null} to deregister + */ + public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { + FLAG_EVAL_WRITER.set(writer); + } + + /** + * Enables or disables enqueueing EVP flagevaluation events on the OpenFeature hook path. This is + * populated from {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED} at feature-flagging startup and + * cleared during shutdown before the writer drains. + */ + public static void setFlagEvaluationEnqueueEnabled(final boolean enabled) { + flagEvalEnqueueEnabled = enabled; + } + + /** + * Returns the active EVP flagevaluation writer, or {@code null} when disabled (killswitch off or + * not yet started). + */ + public static FlagEvaluationWriter getFlagEvalWriter() { + return FLAG_EVAL_WRITER.get(); + } + + /** Returns whether EVP flagevaluation hook events may be enqueued. */ + public static boolean isFlagEvaluationEnqueueEnabled() { + return flagEvalEnqueueEnabled; + } + public static void addSpanEnrichmentListener(final SpanEnrichmentListener listener) { SPAN_ENRICHMENT_LISTENERS.add(listener); } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java new file mode 100644 index 00000000000..f5f44b94d7b --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java @@ -0,0 +1,98 @@ +package datadog.trace.api.featureflag.flagevaluation; + +import java.util.Collections; +import java.util.Map; + +/** + * Lightweight data record capturing a single flag evaluation for EVP flagevaluation emission. + * + *

This is the currency passed from the FlagEvalLoggingHook (feature-flagging-api) to the + * FlagEvaluationWriter (feature-flagging-lib) via a non-blocking bounded queue. + * + *

Scalar fields and context attributes are captured at hook-fire time on the evaluation thread. + * No aggregation happens here. + */ +public final class FlagEvalEvent { + + /** The feature flag key. Never null. */ + public final String flagKey; + + /** + * The OpenFeature variant key selected for the evaluation. Null means the default value was + * returned (runtime default). + */ + public final String variant; + + /** The allocation key from flag metadata ("allocationKey"). May be null. */ + public final String allocationKey; + + /** The targeting key from the evaluation context. May be null. */ + public final String targetingKey; + + /** + * The evaluation error message when the evaluation failed, else null. Sourced from the + * OpenFeature evaluation details (error message, falling back to the error code). + */ + public final String errorMessage; + + /** + * Evaluation timestamp in milliseconds since epoch. Stamped at eval-entry time from flag metadata + * key __dd_eval_timestamp_ms, or falls back to hook-fire time when absent. This ensures + * first/last_evaluation reflect evaluation time, not hook-fire time. + */ + public final long evalTimeMs; + + /** + * Flattened evaluation context attributes. Used for the full-tier canonical context key. May be + * empty but never null. + */ + public final Map attrs; + + /** + * PII consent from the ServerConfiguration used by the evaluation. When false (privacy-preserving + * default), the targeting key is hashed and the per-evaluation context is omitted on emission. + */ + public final boolean observeFullEvaluationData; + + /** Convenience constructor; consent defaults to the privacy-preserving false. */ + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + this(flagKey, variant, allocationKey, targetingKey, null, evalTimeMs, false, attrs); + } + + /** Convenience constructor; consent defaults to the privacy-preserving false. */ + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final Map attrs) { + this(flagKey, variant, allocationKey, targetingKey, errorMessage, evalTimeMs, false, attrs); + } + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final boolean observeFullEvaluationData, + final Map attrs) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.evalTimeMs = evalTimeMs; + this.observeFullEvaluationData = observeFullEvaluationData; + this.attrs = attrs != null ? attrs : Collections.emptyMap(); + } +} diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java new file mode 100644 index 00000000000..65ffaf80591 --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java @@ -0,0 +1,46 @@ +package datadog.trace.api.featureflag.flagevaluation; + +/** + * Defines an EVP flagevaluation writer responsible for aggregating flag evaluation events and + * flushing them to the EVP proxy. + * + *

Implementations must use a background thread (serializing handler) for aggregation and + * transport. The enqueue method must be non-blocking and callable from the OpenFeature hook thread + * without backpressure. + */ +public interface FlagEvaluationWriter extends AutoCloseable { + + /** + * Non-blocking enqueue of a flag evaluation event. May silently drop the event if the internal + * bounded queue is full (best-effort, observable via drop counter). + */ + void enqueue(FlagEvalEvent event); + + /** + * Reports whether the internal queue currently has room for another event. Producers should + * consult this before performing expensive context-copy work so a saturated queue is observed as + * an O(1) read rather than a full snapshot followed by a discarded offer. + * + *

Best-effort: the worker can drain (or a peer producer can fill) between this check and the + * next enqueue, so callers must still tolerate offer failure. + */ + boolean hasCapacityForEnqueue(); + + /** Counts one queue-overflow drop without offering an event. */ + void countPreQueueOverflow(); + + /** + * Counts one evaluation whose context was truncated by copyPrunedContext. The reason string is + * the sorted, comma-separated set of cap names that fired (e.g. + * "max_key_length,max_value_length"). Each unique reason string is counted separately so + * telemetry can distinguish which caps are hot. + */ + void countContextTruncated(String reason); + + /** Starts the background serializing thread. Must be called once after construction. */ + void start(); + + /** Stops the background thread and releases resources. */ + @Override + void close(); +} diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java index 221fc74079a..caaa85a611f 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java @@ -5,16 +5,24 @@ public class ServerConfiguration { public final String createdAt; public final String format; + // Boxed on purpose. Moshi's reflective adapter for a primitive boolean field aborts the whole + // UFC parse when the JSON value is null or not a boolean; with a Boolean field it tolerates + // null (and other malformed values are still caught locally) so a malformed consent field + // doesn't strand a fresh pod on PROVIDER_NOT_READY. Read sites must use + // Boolean.TRUE.equals(...) so null falls to the privacy-preserving default. + public final Boolean observeFullEvaluationData; public final Environment environment; public final Map flags; public ServerConfiguration( final String createdAt, final String format, + final Boolean observeFullEvaluationData, final Environment environment, final Map flags) { this.createdAt = createdAt; this.format = format; + this.observeFullEvaluationData = observeFullEvaluationData; this.environment = environment; this.flags = flags; } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java index daaaf8d7001..887a153f0a1 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java @@ -39,6 +39,8 @@ void tearDown() { FeatureFlaggingGateway.removeActivationListener(activationListener); FeatureFlaggingGateway.removeExposureListener(exposureListener); FeatureFlaggingGateway.removeSpanEnrichmentListener(spanEnrichmentListener); + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java new file mode 100644 index 00000000000..cc614150178 --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java @@ -0,0 +1,54 @@ +package datadog.trace.api.featureflag.flagevaluation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlagEvalEventTest { + + @Test + void storesFieldsWithContextAttributes() { + final Map attrs = Collections.singletonMap("tier", "gold"); + + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", "on", "allocation-1", "target-1", 123L, attrs); + + assertEquals("my-flag", event.flagKey); + assertEquals("on", event.variant); + assertEquals("allocation-1", event.allocationKey); + assertEquals("target-1", event.targetingKey); + assertNull(event.errorMessage); + assertEquals(123L, event.evalTimeMs); + assertSame(attrs, event.attrs); + } + + @Test + void storesErrorMessageAndDefaultsNullContextAttributes() { + final Map attrs = null; + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", null, null, null, "type mismatch", 456L, attrs); + + assertEquals("type mismatch", event.errorMessage); + assertTrue(event.attrs.isEmpty()); + } + + @Test + void observeFullEvaluationDataDefaultsToFalseOnConvenienceConstructors() { + final Map attrs = Collections.emptyMap(); + assertFalse(new FlagEvalEvent("f", "on", "a", "t", 1L, attrs).observeFullEvaluationData); + assertFalse(new FlagEvalEvent("f", "on", "a", "t", null, 1L, attrs).observeFullEvaluationData); + } + + @Test + void storesExplicitObserveFullEvaluationData() { + final Map attrs = Collections.emptyMap(); + assertTrue( + new FlagEvalEvent("f", "on", "a", "t", null, 1L, true, attrs).observeFullEvaluationData); + } +} diff --git a/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java index bae6fda1890..1edfd5bcb26 100644 --- a/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java +++ b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java @@ -23,6 +23,14 @@ public class FeatureFlaggingConfig { public static final String EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED = "experimental.flagging.provider.span.enrichment.enabled"; + /** + * Killswitch for the EVP {@code flagevaluation} emission path. Default: enabled. Disabling it + * turns off EVP flag-evaluation counts while leaving the OTel {@code feature_flag.evaluations} + * metric path untouched. Maps to {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED}. + */ + public static final String FLAGGING_EVALUATION_COUNTS_ENABLED = + "flagging.evaluation.counts.enabled"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE = "feature.flags.configuration.source"; public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL = diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 425e217e822..6f4413b5ff7 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -1,6 +1,7 @@ plugins { `java-library` id("dd-trace-java.version-file") + id("me.champeau.jmh") } apply(from = "$rootDir/gradle/java.gradle") @@ -32,3 +33,14 @@ dependencies { testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } + +jmh { + jmhVersion = libs.versions.jmh.get() + duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE + if (project.hasProperty("jmhIncludes")) { + includes = listOf(project.property("jmhIncludes").toString()) + } + if (project.hasProperty("jmhProf")) { + profilers = listOf(project.property("jmhProf").toString()) + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationEnqueueContentionBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationEnqueueContentionBenchmark.java new file mode 100644 index 00000000000..8da76d4a88e --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationEnqueueContentionBenchmark.java @@ -0,0 +1,190 @@ +package com.datadog.featureflag; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.communication.BackendApiFactory; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import de.thetaphi.forbiddenapis.SuppressForbidden; +import java.util.HashMap; +import java.util.Map; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Group; +import org.openjdk.jmh.annotations.GroupThreads; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Producer-contention benchmark for {@link FlagEvaluationWriterImpl#enqueue}. + * + *

Every flag evaluation in every application thread calls {@code enqueue}, so producer-side + * scaling is the property that matters. {@link FlagEvaluationHotPathBenchmark#writerEnqueue} + * measures the single-threaded cost and cannot see contention; this benchmark measures how enqueue + * behaves as producer count grows. + * + *

Why groups rather than {@code @Threads}: the hand-off queue is a JCTools + * {@code mpscBlockingConsumerArrayQueue} - multi-producer, single-consumer. Calling {@code + * poll()} from several threads breaks that contract, so producer threads cannot drain their own + * events. Each group therefore pairs N producers with exactly one consumer. + * + *

Why the consumer drains in batches: with more producers than consumers the + * queue would otherwise saturate, after which {@code offer} fast-fails and the measurement turns + * into {@code AtomicLong} contention on the overflow counter instead of the enqueue path. The + * consumer polls up to {@link #DRAIN_BATCH} events per invocation to keep the queue off its + * capacity limit. Overflow is reported at the end of each iteration so a saturated - and therefore + * invalid - run is visible rather than silent. + * + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh + * -PjmhIncludes=FlagEvaluationEnqueueContentionBenchmark}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class FlagEvaluationEnqueueContentionBenchmark { + + /** Events the single consumer drains per invocation, so it can keep up with N producers. */ + static final int DRAIN_BATCH = 32; + + private static final int NUM_FLAGS = 100; + private static final int NUM_USERS = 50; + private static final int NUM_FIELDS = 10; + + private Map attrs; + private String[] flagKeys; + private String[] targetingKeys; + private FlagEvaluationWriterImpl writer; + + /** Per-producer cursor: a shared counter would add its own cache-line contention. */ + @State(Scope.Thread) + public static class ProducerCursor { + int cursor; + } + + @Setup(Level.Iteration) + public void setUp() { + // enqueue() no-ops unless the gateway gate is on; set it explicitly so the benchmark is not + // silently measuring an early return. + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + + attrs = new HashMap<>(); + for (int i = 0; i < NUM_FIELDS; i++) { + attrs.put("field" + i, "value"); + } + flagKeys = keys("bench-flag-", NUM_FLAGS); + targetingKeys = keys("bench-user-", NUM_USERS); + + final Config config = Config.get(); + final BackendApiFactory factory = new BackendApiFactory(config, null); + // Capacity well above what the batch-draining consumer should ever let build up. + writer = new FlagEvaluationWriterImpl(1 << 20, Long.MAX_VALUE, NANOSECONDS, factory, config); + } + + /** + * Reports queue overflow so a saturated run - where the consumer failed to keep up and the + * numbers no longer describe the enqueue path - is visible in the benchmark output. + */ + @org.openjdk.jmh.annotations.TearDown(Level.Iteration) + @SuppressForbidden + public void reportOverflow() { + final long dropped = writer.droppedQueueOverflow(); + if (dropped > 0) { + System.out.println( + "\nWARNING: queue overflowed " + + dropped + + " times - consumer could not keep up, enqueue timings for this iteration are" + + " measuring overflow accounting, not the enqueue path."); + } + } + + // ---- 1 producer: uncontended baseline ---- + + @Benchmark + @Group("producers1") + @GroupThreads(1) + public void enqueue1(final ProducerCursor c) { + writer.enqueue(nextEvent(c)); + } + + @Benchmark + @Group("producers1") + @GroupThreads(1) + public void drain1(final Blackhole blackhole) { + drain(blackhole); + } + + // ---- 4 producers ---- + + @Benchmark + @Group("producers4") + @GroupThreads(4) + public void enqueue4(final ProducerCursor c) { + writer.enqueue(nextEvent(c)); + } + + @Benchmark + @Group("producers4") + @GroupThreads(1) + public void drain4(final Blackhole blackhole) { + drain(blackhole); + } + + // ---- 16 producers ---- + + @Benchmark + @Group("producers16") + @GroupThreads(16) + public void enqueue16(final ProducerCursor c) { + writer.enqueue(nextEvent(c)); + } + + @Benchmark + @Group("producers16") + @GroupThreads(1) + public void drain16(final Blackhole blackhole) { + drain(blackhole); + } + + private void drain(final Blackhole blackhole) { + for (int i = 0; i < DRAIN_BATCH; i++) { + final FlagEvalEvent event = writer.pollQueuedEventForTest(); + if (event == null) { + break; + } + blackhole.consume(event); + } + } + + private FlagEvalEvent nextEvent(final ProducerCursor c) { + final int i = c.cursor++; + return new FlagEvalEvent( + flagKeys[Math.floorMod(i, flagKeys.length)], + "variant-" + Math.floorMod(i, 4), + "alloc-" + Math.floorMod(i, flagKeys.length), + targetingKeys[Math.floorMod(i, targetingKeys.length)], + null, + 1_700_000_000_000L + i, + attrs); + } + + private static String[] keys(final String prefix, final int count) { + final String[] out = new String[count]; + for (int i = 0; i < count; i++) { + out[i] = prefix + i; + } + return out; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java new file mode 100644 index 00000000000..e22d05bee07 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -0,0 +1,159 @@ +package com.datadog.featureflag; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.communication.BackendApiFactory; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.HashMap; +import java.util.Map; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Writer-side benchmark for EVP {@code flagevaluation} recording: the bounded-queue enqueue and the + * worker-thread aggregation of already-captured events. + * + *

Scope limitation: this benchmark starts from a pre-built FlagEvalEvent + * holding a ready-made flat attribute map. It therefore does NOT cover the OpenFeature hook's own + * inline capture cost - in particular the bounded evaluation-context copy that + * FlagEvalLoggingHook.finallyAfter performs synchronously on the evaluation thread under + * consent-on. That cost is measured separately by FlagEvalHookHotPathBenchmark in + * feature-flagging-api, where the hook and the OpenFeature context types live. Do not read + * writerEnqueue as the total per-evaluation cost paid by the caller. + * + *

Events are built with observeFullEvaluationData=true so the aggregator actually canonicalizes + * the context. Under the consent-off default it drops attrs and skips canonicalization, which makes + * every field-count profile measure the same scalar-only work. + * + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh + * -PjmhIncludes=FlagEvaluationHotPathBenchmark}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class FlagEvaluationHotPathBenchmark { + + @Param({ + "typical/100flags_50users_10fields", + "stress/10flags_1000users_250fields", + "scale/2500flags_500users_20fields" + }) + public String profile; + + private Map attrs; + private String[] flagKeys; + private String[] targetingKeys; + private int cursor; + private FlagEvaluationWriterImpl writer; + private FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + + @Setup(Level.Iteration) + public void setUp() { + final Profile p = Profile.fromName(profile); + + attrs = new HashMap<>(); + for (int i = 0; i < p.numFields; i++) { + attrs.put("field" + i, "value"); + } + flagKeys = keys("bench-flag-", p.numFlags); + targetingKeys = keys("bench-user-", p.numUsers); + cursor = 0; + + final Config config = Config.get(); + final BackendApiFactory factory = new BackendApiFactory(config, null); + final Map ddContext = new HashMap<>(); + ddContext.put("service", "bench-service"); + handler = FlagEvaluationWriterImpl.createHandlerForTest(factory, ddContext); + + // Capacity large enough that the benchmark never overflows within a measurement window. + writer = new FlagEvaluationWriterImpl(1 << 20, Long.MAX_VALUE, NANOSECONDS, factory, config); + } + + /** + * Queue-mechanics cost only: event allocation plus the non-blocking bounded-queue offer. Excludes + * the hook's inline context snapshot - see the class javadoc. + */ + @Benchmark + public void writerEnqueue(final Blackhole blackhole) { + final FlagEvalEvent event = nextEvent(); + writer.enqueue(event); + blackhole.consume(writer.pollQueuedEventForTest()); + blackhole.consume(event); + } + + /** Worker-thread cost: materialize context, prune, canonicalize, and aggregate. */ + @Benchmark + public void workerAggregate(final Blackhole blackhole) { + final FlagEvalEvent event = nextEvent(); + handler.aggregateEvent(event); + if ((cursor % 10_000) == 0) { + handler.clearAggregationForTest(); + } + blackhole.consume(handler.fullTierSizeForTest()); + } + + private FlagEvalEvent nextEvent() { + final int i = cursor++; + // observeFullEvaluationData=true is required for this benchmark to mean anything: under the + // consent-off default the aggregator drops attrs and skips canonicalContextKey entirely, so + // every field-count profile would collapse to the same scalar-only cost. + return new FlagEvalEvent( + flagKeys[Math.floorMod(i, flagKeys.length)], + "variant-" + Math.floorMod(i, 4), + "alloc-" + Math.floorMod(i, flagKeys.length), + targetingKeys[Math.floorMod(i, targetingKeys.length)], + null, + 1_700_000_000_000L + i, + true, + attrs); + } + + private static String[] keys(final String prefix, final int count) { + final String[] out = new String[count]; + for (int i = 0; i < count; i++) { + out[i] = prefix + i; + } + return out; + } + + private static final class Profile { + private final int numFlags; + private final int numUsers; + private final int numFields; + + private Profile(final int numFlags, final int numUsers, final int numFields) { + this.numFlags = numFlags; + this.numUsers = numUsers; + this.numFields = numFields; + } + + private static Profile fromName(final String name) { + if ("typical/100flags_50users_10fields".equals(name)) { + return new Profile(100, 50, 10); + } + if ("stress/10flags_1000users_250fields".equals(name)) { + return new Profile(10, 1_000, 250); + } + if ("scale/2500flags_500users_20fields".equals(name)) { + return new Profile(2_500, 500, 20); + } + throw new IllegalArgumentException("unknown benchmark profile: " + name); + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index 9932a20256b..46b3fe5a486 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -5,25 +5,19 @@ import static datadog.trace.util.AgentThreadFactory.newAgentThread; import static java.util.concurrent.TimeUnit.SECONDS; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.Moshi; import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; -import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.exposure.ExposureEvent; import datadog.trace.api.featureflag.exposure.ExposuresRequest; -import datadog.trace.api.intake.Intake; import datadog.trace.api.internal.VisibleForTesting; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import okhttp3.RequestBody; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +27,7 @@ public class ExposureWriterImpl implements ExposureWriter { private static final int DEFAULT_CAPACITY = 1 << 16; // 65536 elements private static final int DEFAULT_FLUSH_INTERVAL_IN_SECONDS = 1; private static final int FLUSH_THRESHOLD = 100; + private static final String EXPOSURES_ROUTE = "exposures"; private final MessagePassingBlockingQueue queue; private final Thread serializerThread; @@ -48,21 +43,13 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con final SharedCommunicationObjects sco, final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); - final Map context = new HashMap<>(4); - context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); - if (config.getEnv() != null) { - context.put("env", config.getEnv()); - } - if (config.getVersion() != null) { - context.put("version", config.getVersion()); - } final ExposureSerializingHandler serializer = new ExposureSerializingHandler( new BackendApiFactory(config, sco), queue, flushInterval, timeUnit, - context, + FeatureFlagEvpContext.from(config), this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EXPOSURE_PROCESSOR, serializer); } @@ -101,10 +88,7 @@ private static class ExposureSerializingHandler implements Runnable { private final long ticksRequiredToFlush; private long lastTicks; - private final JsonAdapter jsonAdapter; - private final BackendApiFactory backendApiFactory; - private BackendApi evp; - + private final FeatureFlagEvpPublisher evpPublisher; private final Map context; private final ExposureCache cache; @@ -120,8 +104,7 @@ public ExposureSerializingHandler( final Runnable errorCallback) { this.queue = queue; this.cache = new LRUExposureCache(queue.capacity()); - this.jsonAdapter = new Moshi.Builder().build().adapter(ExposuresRequest.class); - this.backendApiFactory = backendApiFactory; + this.evpPublisher = new FeatureFlagEvpPublisher<>(backendApiFactory, ExposuresRequest.class); this.context = context; this.lastTicks = System.nanoTime(); @@ -134,8 +117,7 @@ public ExposureSerializingHandler( @Override public void run() { - evp = backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM); - if (evp == null) { + if (!evpPublisher.start()) { errorCallback.run(); throw new IllegalArgumentException("EVP Proxy not available"); } @@ -179,19 +161,17 @@ protected void flushIfNecessary() { return; } if (shouldFlush()) { - final String requestBodyJson; + final byte[] payload; try { final ExposuresRequest exposures = new ExposuresRequest(this.context, this.buffer); - requestBodyJson = jsonAdapter.toJson(exposures); + payload = evpPublisher.serialize(exposures); } catch (RuntimeException e) { LOGGER.error(EXCLUDE_TELEMETRY, "Could not serialize exposures; dropping batch", e); this.buffer.clear(); return; } try { - final RequestBody requestBody = - RequestBody.create(okhttp3.MediaType.parse("application/json"), requestBodyJson); - evp.post("exposures", requestBody, stream -> null, null, false); + evpPublisher.post(EXPOSURES_ROUTE, payload); this.buffer.clear(); } catch (Exception e) { LOGGER.debug("Could not submit exposures", e); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java new file mode 100644 index 00000000000..c964efa6c7f --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java @@ -0,0 +1,22 @@ +package com.datadog.featureflag; + +import datadog.trace.api.Config; +import java.util.HashMap; +import java.util.Map; + +final class FeatureFlagEvpContext { + + private FeatureFlagEvpContext() {} + + static Map from(final Config config) { + final Map context = new HashMap<>(4); + context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); + if (config.getEnv() != null) { + context.put("env", config.getEnv()); + } + if (config.getVersion() != null) { + context.put("version", config.getVersion()); + } + return context; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java new file mode 100644 index 00000000000..c871b18165e --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java @@ -0,0 +1,65 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.intake.Intake; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import okhttp3.MediaType; +import okhttp3.RequestBody; + +final class FeatureFlagEvpPublisher { + + private static final MediaType JSON = MediaType.parse("application/json"); + + private final BackendApiFactory backendApiFactory; + private final boolean responseCompression; + private final JsonAdapter jsonAdapter; + private BackendApi evp; + + FeatureFlagEvpPublisher(final BackendApiFactory backendApiFactory, final Class requestType) { + this(backendApiFactory, requestType, true); + } + + FeatureFlagEvpPublisher( + final BackendApiFactory backendApiFactory, + final Class requestType, + final boolean responseCompression) { + this.backendApiFactory = backendApiFactory; + this.responseCompression = responseCompression; + this.jsonAdapter = new Moshi.Builder().build().adapter(requestType); + } + + boolean start() { + if (evp == null) { + evp = backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM, responseCompression); + } + return evp != null; + } + + void post(final String route, final T request) throws IOException { + post(route, serialize(request)); + } + + byte[] serialize(final T request) { + return utf8Bytes(jsonAdapter.toJson(request)); + } + + void post(final String route, final byte[] json) throws IOException { + if (!start()) { + throw new IllegalStateException("EVP Proxy not available"); + } + final RequestBody requestBody = RequestBody.create(JSON, json); + evp.post(route, requestBody, stream -> null, null, false); + } + + static byte[] utf8Bytes(final String json) { + try { + return json.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError("UTF-8 must be available", e); + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java new file mode 100644 index 00000000000..5d0eeaa0e82 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java @@ -0,0 +1,454 @@ +package com.datadog.featureflag; + +import static datadog.trace.util.HashingUtils.addToHash; +import static datadog.trace.util.HashingUtils.hash; + +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +final class FlagEvaluationAggregator { + + // Design assumptions — document the scale we sized for + static final int EXPECTED_FLAG_COUNT = 2_500; + static final int EXPECTED_FULL_BUCKETS_PER_FLAG = 50; + static final int EXPECTED_USERS_PER_FLAG = 1_000; + static final int PER_FLAG_HEADROOM_MULTIPLIER = 10; + static final int EXPECTED_DEGRADED_BUCKETS_PER_FLAG = 10; + + // Derived sizing — show the math behind the bucket caps below + static final int FULL_BUCKET_SIZING_BASIS = + EXPECTED_FLAG_COUNT * EXPECTED_FULL_BUCKETS_PER_FLAG; // 125_000 + static final int PER_FLAG_BUCKET_SIZING_BASIS = + PER_FLAG_HEADROOM_MULTIPLIER * EXPECTED_USERS_PER_FLAG; // 10_000 + static final int DEGRADED_BUCKET_SIZING_BASIS = + EXPECTED_FLAG_COUNT * EXPECTED_DEGRADED_BUCKETS_PER_FLAG; // 25_000 + + // Enforced bucket caps + static final int GLOBAL_CAP = 131_072; // nearest power of two above FULL_BUCKET_SIZING_BASIS + static final int PER_FLAG_CAP = PER_FLAG_BUCKET_SIZING_BASIS; + static final int DEGRADED_CAP = 32_768; // nearest power of two above DEGRADED_BUCKET_SIZING_BASIS + + private static final byte CTX_TAG_STRING = 's'; + private static final byte CTX_TAG_BOOL = 'b'; + private static final byte CTX_TAG_INT = 'i'; + private static final byte CTX_TAG_LONG = 'l'; + private static final byte CTX_TAG_FLOAT = 'f'; + private static final byte CTX_TAG_DOUBLE = 'd'; + private static final byte CTX_TAG_OTHER = 'o'; + + final Map fullTier = new HashMap<>(); + final Map degradedTier = new HashMap<>(); + final Map perFlagCount = new HashMap<>(); + final AtomicLong droppedDegradedOverflow = new AtomicLong(0); + final AtomicInteger globalFullCount = new AtomicInteger(0); + + void aggregate(final FlagEvalEvent event) { + final boolean isDefault = event.variant == null; + final boolean observeFullEvaluationData = event.observeFullEvaluationData; + // On the protected path the context is dropped on emit, so it must not fragment buckets or be + // stored — otherwise a high-cardinality field (request_id, timestamp) blows out PER_FLAG_CAP + // and spills every subsequent evaluation into the degraded tier. + final Map prunedAttrs = observeFullEvaluationData ? event.attrs : null; + final String ctxKey = observeFullEvaluationData ? canonicalContextKey(prunedAttrs) : ""; + final FullKey fullKey = buildFullKey(event, ctxKey); + + EvalBucket bucket = fullTier.get(fullKey); + if (bucket != null) { + bucket.merge(event.evalTimeMs, isDefault); + bucket.observeFullEvaluationData &= observeFullEvaluationData; + return; + } + + final int flagCount = perFlagCount.getOrDefault(event.flagKey, 0); + if (globalFullCount.get() < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { + fullTier.put( + fullKey, + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + event.targetingKey, + event.errorMessage, + event.evalTimeMs, + isDefault, + prunedAttrs, + observeFullEvaluationData)); + globalFullCount.incrementAndGet(); + perFlagCount.put(event.flagKey, flagCount + 1); + return; + } + + final DegradedKey degradedKey = buildDegradedKey(event); + bucket = degradedTier.get(degradedKey); + if (bucket != null) { + bucket.merge(event.evalTimeMs, isDefault); + bucket.observeFullEvaluationData &= observeFullEvaluationData; + return; + } + + if (degradedTier.size() < DEGRADED_CAP) { + degradedTier.put( + degradedKey, + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + null, + event.errorMessage, + event.evalTimeMs, + isDefault, + null, + observeFullEvaluationData)); + return; + } + + droppedDegradedOverflow.incrementAndGet(); + } + + boolean isEmpty() { + return fullTier.isEmpty() && degradedTier.isEmpty(); + } + + int fullTierSize() { + return fullTier.size(); + } + + long degradedEvaluationCount() { + long count = 0; + for (final EvalBucket bucket : degradedTier.values()) { + count += bucket.count; + } + return count; + } + + int bucketCount() { + return fullTier.size() + degradedTier.size(); + } + + Iterable fullBuckets() { + return fullTier.values(); + } + + Iterable degradedBuckets() { + return degradedTier.values(); + } + + void clear() { + fullTier.clear(); + degradedTier.clear(); + perFlagCount.clear(); + globalFullCount.set(0); + } + + AggregatedState snapshot() { + return new AggregatedState( + new HashMap<>(fullTier), new HashMap<>(degradedTier), droppedDegradedOverflow.get()); + } + + void simulateFullTierAtCap() { + for (int i = globalFullCount.get(); i < GLOBAL_CAP; i++) { + final String key = "synthetic-full-" + i; + fullTier.put( + new FullKey(key, "on", "alloc", false, null, null, "", false), + new EvalBucket(key, "on", "alloc", null, null, 1L, false, null, false)); + globalFullCount.incrementAndGet(); + perFlagCount.merge(key, 1, Integer::sum); + } + } + + void simulateDegradedTierAtCap() { + for (int i = degradedTier.size(); i < DEGRADED_CAP; i++) { + final String key = "synthetic-dg-" + i; + degradedTier.put( + new DegradedKey(key, "on", "alloc", false, null), + new EvalBucket(key, "on", "alloc", null, null, 1L, false, null, false)); + } + } + + void addDegradedBucketForTest( + final String flagKey, + final String variant, + final String allocationKey, + final String errorMessage, + final long evalTimeMs) { + degradedTier.put( + new DegradedKey(flagKey, variant, allocationKey, variant == null, errorMessage), + new EvalBucket( + flagKey, + variant, + allocationKey, + null, + errorMessage, + evalTimeMs, + variant == null, + null, + false)); + } + + private static FullKey buildFullKey(final FlagEvalEvent event, final String ctxKey) { + return new FullKey( + event.flagKey, + event.variant, + event.allocationKey, + event.variant == null, + event.errorMessage, + event.targetingKey, + ctxKey, + event.observeFullEvaluationData); + } + + private static DegradedKey buildDegradedKey(final FlagEvalEvent event) { + return new DegradedKey( + event.flagKey, + event.variant, + event.allocationKey, + event.variant == null, + event.errorMessage); + } + + static String canonicalContextKey(final Map prunedAttrs) { + if (prunedAttrs == null || prunedAttrs.isEmpty()) { + return ""; + } + final Map sorted = + (prunedAttrs instanceof TreeMap) ? prunedAttrs : new TreeMap<>(prunedAttrs); + final StringBuilder sb = new StringBuilder(); + for (final Map.Entry entry : sorted.entrySet()) { + appendLengthDelimited(sb, entry.getKey()); + appendContextValue(sb, entry.getValue()); + } + return sb.toString(); + } + + private static final String HEX_ZEROS = "00000000"; + + private static void appendLengthDelimited(final StringBuilder sb, final String s) { + // 8-char zero-padded hex length prefix, allocation-lean (no String.format on the hot path). + final String hexLength = Integer.toHexString(s.length()); + sb.append(HEX_ZEROS, 0, 8 - hexLength.length()); + sb.append(hexLength); + sb.append(s); + } + + private static void appendContextValue(final StringBuilder sb, final Object v) { + if (v instanceof Boolean) { + sb.append((char) CTX_TAG_BOOL); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Long) { + sb.append((char) CTX_TAG_LONG); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Integer) { + sb.append((char) CTX_TAG_INT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Float) { + sb.append((char) CTX_TAG_FLOAT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Double) { + sb.append((char) CTX_TAG_DOUBLE); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof String) { + sb.append((char) CTX_TAG_STRING); + appendLengthDelimited(sb, (String) v); + } else { + sb.append((char) CTX_TAG_OTHER); + appendLengthDelimited(sb, v == null ? "" : v.toString()); + } + } + + static class EvalBucket { + long count; + long firstEvalMs; + long lastEvalMs; + boolean runtimeDefaultUsed; + String flagKey; + String variant; + String allocationKey; + String targetingKey; + String errorMessage; + Map prunedAttrs; + // Consent to emit raw PII. For full-tier buckets this is uniform (consent is a FullKey + // dimension) and the AND-fold on merge is defensive. For degraded-tier buckets consent is NOT + // a key dimension — mixed-consent events merge here — so the AND-fold produces false whenever + // any consent-off event lands in the bucket. That's benign because the degraded wire path + // drops the targeting key and context regardless of consent, so this field has no downstream + // effect for degraded rows. + boolean observeFullEvaluationData; + + EvalBucket( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final boolean runtimeDefaultUsed, + final Map prunedAttrs, + final boolean observeFullEvaluationData) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.firstEvalMs = evalTimeMs; + this.lastEvalMs = evalTimeMs; + this.count = 1; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.prunedAttrs = prunedAttrs; + this.observeFullEvaluationData = observeFullEvaluationData; + } + + int prunedContextFieldCount() { + return prunedAttrs == null ? 0 : prunedAttrs.size(); + } + + void merge(final long evalTimeMs, final boolean isDefault) { + count++; + if (evalTimeMs < firstEvalMs) { + firstEvalMs = evalTimeMs; + } + if (evalTimeMs > lastEvalMs) { + lastEvalMs = evalTimeMs; + } + if (isDefault) { + runtimeDefaultUsed = true; + } + } + } + + static final class FullKey { + private final String flagKey; + private final String variant; + private final String allocationKey; + private final boolean runtimeDefaultUsed; + private final String errorMessage; + private final String targetingKey; + private final String contextKey; + // Part of the key so consent-on and consent-off evaluations never share a bucket. The + // serializer branches on this to hash the targeting key and drop the context, so events with + // different consent produce different wire rows and belong in different buckets. + private final boolean observeFullEvaluationData; + + FullKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final String targetingKey, + final String contextKey, + final boolean observeFullEvaluationData) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.errorMessage = errorMessage; + this.targetingKey = targetingKey; + this.contextKey = contextKey; + this.observeFullEvaluationData = observeFullEvaluationData; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FullKey)) { + return false; + } + final FullKey fullKey = (FullKey) o; + return runtimeDefaultUsed == fullKey.runtimeDefaultUsed + && observeFullEvaluationData == fullKey.observeFullEvaluationData + && Objects.equals(flagKey, fullKey.flagKey) + && Objects.equals(variant, fullKey.variant) + && Objects.equals(allocationKey, fullKey.allocationKey) + && Objects.equals(errorMessage, fullKey.errorMessage) + && Objects.equals(targetingKey, fullKey.targetingKey) + && Objects.equals(contextKey, fullKey.contextKey); + } + + @Override + public int hashCode() { + // HashingUtils avoids the Object[] allocation and boolean boxing that Objects.hash performs + // on this hot bucket-lookup path. + int result = hash(flagKey, variant, allocationKey); + result = addToHash(result, runtimeDefaultUsed); + result = addToHash(result, errorMessage); + result = addToHash(result, targetingKey); + result = addToHash(result, contextKey); + return addToHash(result, observeFullEvaluationData); + } + } + + static final class DegradedKey { + // Unlike FullKey, consent is NOT a bucket dimension here: the wire serializer for degraded rows + // (FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket with isFullTier=false) drops the + // targeting key and context unconditionally, so two degraded buckets differing only in consent + // would emit byte-identical JSON with evaluation_count split — halving effective DEGRADED_CAP + // for zero wire fidelity. Mixed-consent events merge into one bucket; the AND-fold on + // EvalBucket.observeFullEvaluationData still runs but has no downstream effect for degraded + // rows. + private final String flagKey; + private final String variant; + private final String allocationKey; + private final boolean runtimeDefaultUsed; + private final String errorMessage; + + DegradedKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.errorMessage = errorMessage; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DegradedKey)) { + return false; + } + final DegradedKey that = (DegradedKey) o; + return runtimeDefaultUsed == that.runtimeDefaultUsed + && Objects.equals(flagKey, that.flagKey) + && Objects.equals(variant, that.variant) + && Objects.equals(allocationKey, that.allocationKey) + && Objects.equals(errorMessage, that.errorMessage); + } + + @Override + public int hashCode() { + // HashingUtils avoids the Object[] allocation and boolean boxing that Objects.hash performs + // on this hot bucket-lookup path. + int result = hash(flagKey, variant, allocationKey); + result = addToHash(result, runtimeDefaultUsed); + return addToHash(result, errorMessage); + } + } + + static class AggregatedState { + final Map fullTier; + final Map degradedTier; + final long droppedDegradedOverflow; + + AggregatedState( + final Map fullTier, + final Map degradedTier, + final long droppedDegradedOverflow) { + this.fullTier = fullTier; + this.degradedTier = degradedTier; + this.droppedDegradedOverflow = droppedDegradedOverflow; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java new file mode 100644 index 00000000000..3a8734fd516 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java @@ -0,0 +1,294 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +final class FlagEvaluationPayloads { + + private static final byte[] PAYLOAD_SUFFIX = FeatureFlagEvpPublisher.utf8Bytes("]}"); + private static final byte[] JSON_COMMA = FeatureFlagEvpPublisher.utf8Bytes(","); + + /** + * Wire prefix identifying a privacy-preserving, hashed targeting key. Emitted for full-tier rows + * when {@code observeFullEvaluationData} is off. The suffix is the lower-case hex SHA-256 of the + * UTF-8 targeting key (see {@link ULeb128Encoder#hashTargetingKey}). This is a cross-SDK wire + * contract - keep it in sync with the other server SDKs and the UFC/EVP spec. + */ + private static final String HASHED_TARGETING_KEY_PREFIX = "sha256_"; + + private static final JsonAdapter EVENT_JSON_ADAPTER; + private static final JsonAdapter> CONTEXT_JSON_ADAPTER; + + static { + final Moshi moshi = new Moshi.Builder().build(); + EVENT_JSON_ADAPTER = moshi.adapter(FlagEvaluationEvent.class); + final Type contextType = Types.newParameterizedType(Map.class, String.class, String.class); + CONTEXT_JSON_ADAPTER = moshi.adapter(contextType); + } + + private FlagEvaluationPayloads() {} + + static class FlagEvaluationsRequest { + public final Map context; + public final List flagEvaluations; + + FlagEvaluationsRequest( + final Map context, final List flagEvaluations) { + this.context = context; + this.flagEvaluations = flagEvaluations; + } + } + + static EncodedPayloads buildPayloads( + final List events, + final Map context, + final int payloadSizeLimitBytes) { + final byte[] prefix = payloadPrefix(context); + EncodedPayloadBuilder current = new EncodedPayloadBuilder(prefix); + final List payloads = new ArrayList<>(); + long droppedPayloadLimit = 0; + long degradedPayloadLimit = 0; + + for (final FlagEvaluationEvent event : events) { + byte[] eventBytes = encodeEvent(event); + + if (!current.canAdd(eventBytes, payloadSizeLimitBytes) && !current.isEmpty()) { + payloads.add(current.toByteArray()); + current = new EncodedPayloadBuilder(prefix); + } + + if (current.canAdd(eventBytes, payloadSizeLimitBytes)) { + current.add(eventBytes); + continue; + } + + final FlagEvaluationEvent degraded = event.withoutTargetingKeyAndContext(); + if (degraded != null) { + eventBytes = encodeEvent(degraded); + if (!current.canAdd(eventBytes, payloadSizeLimitBytes) && !current.isEmpty()) { + payloads.add(current.toByteArray()); + current = new EncodedPayloadBuilder(prefix); + } + if (current.canAdd(eventBytes, payloadSizeLimitBytes)) { + current.add(eventBytes); + degradedPayloadLimit += event.evaluation_count; + continue; + } + } + + droppedPayloadLimit += event.evaluation_count; + } + + if (!current.isEmpty()) { + payloads.add(current.toByteArray()); + } + return new EncodedPayloads(payloads, droppedPayloadLimit, degradedPayloadLimit); + } + + private static byte[] payloadPrefix(final Map context) { + return FeatureFlagEvpPublisher.utf8Bytes( + "{\"context\":" + CONTEXT_JSON_ADAPTER.toJson(context) + ",\"flagEvaluations\":["); + } + + private static byte[] encodeEvent(final FlagEvaluationEvent event) { + return FeatureFlagEvpPublisher.utf8Bytes(EVENT_JSON_ADAPTER.toJson(event)); + } + + static final class EncodedPayloads { + final List bodies; + final long droppedPayloadLimit; + final long degradedPayloadLimit; + + private EncodedPayloads( + final List bodies, + final long droppedPayloadLimit, + final long degradedPayloadLimit) { + this.bodies = bodies; + this.droppedPayloadLimit = droppedPayloadLimit; + this.degradedPayloadLimit = degradedPayloadLimit; + } + } + + private static final class EncodedPayloadBuilder { + private final byte[] prefix; + private final List events = new ArrayList<>(); + private int eventBytes; + + private EncodedPayloadBuilder(final byte[] prefix) { + this.prefix = prefix; + } + + private boolean isEmpty() { + return events.isEmpty(); + } + + private boolean canAdd(final byte[] event, final int payloadSizeLimitBytes) { + return sizeWith(event) <= payloadSizeLimitBytes; + } + + private int sizeWith(final byte[] event) { + return prefix.length + PAYLOAD_SUFFIX.length + eventBytes + event.length + events.size(); + } + + private void add(final byte[] event) { + events.add(event); + eventBytes += event.length; + } + + private byte[] toByteArray() { + final int size = prefix.length + PAYLOAD_SUFFIX.length + eventBytes; + final ByteArrayOutputStream out = new ByteArrayOutputStream(size + events.size()); + out.write(prefix, 0, prefix.length); + for (int i = 0; i < events.size(); i++) { + if (i > 0) { + out.write(JSON_COMMA, 0, JSON_COMMA.length); + } + final byte[] event = events.get(i); + out.write(event, 0, event.length); + } + out.write(PAYLOAD_SUFFIX, 0, PAYLOAD_SUFFIX.length); + return out.toByteArray(); + } + } + + static class FlagEvaluationEvent { + public final long timestamp; + public final FlagKeyObject flag; + public final long first_evaluation; + public final long last_evaluation; + public final long evaluation_count; + public final KeyObject variant; + public final KeyObject allocation; + public final String targeting_key; + public final Boolean runtime_default_used; + public final EventContext context; + public final ErrorObject error; + + FlagEvaluationEvent( + final long timestamp, + final String flagKey, + final long firstEvalMs, + final long lastEvalMs, + final long count, + final String variant, + final String allocation, + final String targetingKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final Map evaluationAttrs) { + this.timestamp = timestamp; + this.flag = new FlagKeyObject(flagKey); + this.first_evaluation = firstEvalMs; + this.last_evaluation = lastEvalMs; + this.evaluation_count = count; + this.variant = (variant != null && !variant.isEmpty()) ? new KeyObject(variant) : null; + this.allocation = + (allocation != null && !allocation.isEmpty()) ? new KeyObject(allocation) : null; + this.targeting_key = targetingKey; + this.runtime_default_used = runtimeDefaultUsed ? Boolean.TRUE : null; + this.context = + (evaluationAttrs != null && !evaluationAttrs.isEmpty()) + ? new EventContext(evaluationAttrs) + : null; + this.error = + (errorMessage != null && !errorMessage.isEmpty()) ? new ErrorObject(errorMessage) : null; + } + + static FlagEvaluationEvent fromBucket( + final FlagEvaluationAggregator.EvalBucket bucket, + final boolean isFullTier, + final boolean observeFullEvaluationData, + final long flushTimeMs) { + final boolean includeRawContext = isFullTier && observeFullEvaluationData; + return new FlagEvaluationEvent( + flushTimeMs, + bucket.flagKey, + bucket.firstEvalMs, + bucket.lastEvalMs, + bucket.count, + bucket.variant, + bucket.allocationKey, + resolveTargetingKey(bucket.targetingKey, isFullTier, observeFullEvaluationData), + bucket.runtimeDefaultUsed, + bucket.errorMessage, + includeRawContext ? bucket.prunedAttrs : null); + } + + private static String resolveTargetingKey( + final String rawTargetingKey, + final boolean isFullTier, + final boolean observeFullEvaluationData) { + if (!isFullTier || rawTargetingKey == null) { + return null; + } + if (observeFullEvaluationData) { + return rawTargetingKey; + } + return HASHED_TARGETING_KEY_PREFIX + ULeb128Encoder.hashTargetingKey(rawTargetingKey); + } + + FlagEvaluationEvent withoutTargetingKeyAndContext() { + if (targeting_key == null && context == null) { + return null; + } + return new FlagEvaluationEvent( + timestamp, + flag.key, + first_evaluation, + last_evaluation, + evaluation_count, + keyOf(variant), + keyOf(allocation), + null, + Boolean.TRUE.equals(runtime_default_used), + messageOf(error), + null); + } + } + + private static String keyOf(final KeyObject object) { + return object == null ? null : object.key; + } + + private static String messageOf(final ErrorObject object) { + return object == null ? null : object.message; + } + + static class KeyObject { + public final String key; + + KeyObject(final String key) { + this.key = key; + } + } + + static class FlagKeyObject { + public final String key; + + FlagKeyObject(final String key) { + this.key = key; + } + } + + static class ErrorObject { + public final String message; + + ErrorObject(final String message) { + this.message = message; + } + } + + static class EventContext { + public final Map evaluation; + + EventContext(final Map evaluation) { + this.evaluation = evaluation; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java new file mode 100644 index 00000000000..e15666aa10a --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -0,0 +1,631 @@ +package com.datadog.featureflag; + +import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_EVALUATION_PROCESSOR; +import static datadog.trace.util.AgentThreadFactory.newAgentThread; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.common.queue.MessagePassingBlockingQueue; +import datadog.common.queue.Queues; +import datadog.communication.BackendApiFactory; +import datadog.communication.EvpProxy; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import datadog.trace.api.telemetry.CoreMetricCollector; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * EVP flagevaluation writer for Java. + * + *

Uses the same EVP publisher path as ExposureWriterImpl, with two-tier aggregation replacing + * the single-exposure buffer. Routes to the Agent-advertised EVP proxy endpoint for + * /api/v2/flagevaluation. + * + *

Two-tier aggregation contract: Full key: (flagKey, variant, allocationKey, runtimeDefault, + * errorMessage, targetingKey, canonical-context-key). Degraded key: (flagKey, variant, + * allocationKey, runtimeDefault, errorMessage) - no targetingKey/context. Canonical context key: + * sorted entries, type-tagged length-delimited encoding - NOT a hash (collision-safe, comparable + * string identity). Context pruning: deterministic (sort before cut), <=256 fields, string values + * <=256 chars; the pruned attributes are what gets aggregated and serialized. Caps: + * globalCap=131072, perFlagCap=10000, degradedCap=32768. Eval-time: min/max of + * firstEvalMs/lastEvalMs across events in the same bucket. Runtime default: absent variant means + * runtimeDefaultUsed=true. Flush interval: 10 seconds. Queue: bounded MessagePassingBlockingQueue + * (capacity 2^16), non-blocking offer; on overflow the event is dropped and the + * droppedQueueOverflow counter is incremented and surfaced on flush. Enqueue: lock-free. Producers + * contend only on the MPSC queue, never on a monitor, so evaluation threads do not serialize + * against each other. Shutdown: close() drains the queue and performs a final flush before the + * worker thread exits. Because enqueue is lock-free, a producer can still offer during shutdown; + * close() sweeps the queue once the worker has been joined, counting any remainder as a closed drop + * so shutdown loss is observable rather than silent. + */ +public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlagEvaluationWriterImpl.class); + + static final int DEFAULT_CAPACITY = 1 << 12; // 4096 elements, per cross-SDK RFC + static final int FLUSH_INTERVAL_SECONDS = 10; + + static final int FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES = EvpProxy.PAYLOAD_SIZE_LIMIT_BYTES; + static final String FLAG_EVALUATION_DROPPED_METRIC = "flagevaluation.rows.dropped"; + static final String FLAG_EVALUATION_DEGRADED_METRIC = "flagevaluation.rows.degraded"; + static final String FLAG_EVALUATION_SPLITS_METRIC = "flagevaluation.payload.splits"; + static final String FLAG_EVALUATION_CONTEXT_TRUNCATED_METRIC = "flagevaluation.context.truncated"; + static final String DROP_REASON_QUEUE_OVERFLOW = "queue_overflow"; + static final String DROP_REASON_CLOSED = "closed"; + static final String DROP_REASON_DEGRADED_CAP = "degraded_cap"; + static final String DROP_REASON_PAYLOAD_LIMIT = "payload_limit"; + static final String DEGRADED_REASON_CARDINALITY_CAP = "cardinality_cap"; + static final String DEGRADED_REASON_PAYLOAD_LIMIT = "payload_limit"; + private static final String FLAG_EVALUATION_ROUTE = "flagevaluation"; + private static final CoreMetricCollector CORE_METRICS = CoreMetricCollector.getInstance(); + + private final MessagePassingBlockingQueue queue; + private final FlagEvaluationSerializingHandler serializer; + private final Thread serializerThread; + private final Object lifecycleLock = new Object(); + private final AtomicBoolean closed = new AtomicBoolean(false); + + private static void countMetric(final String metricName, final long value, final String reason) { + if (value <= 0) { + return; + } + CORE_METRICS.count(metricName, value, reason == null ? null : "reason:" + reason); + } + + /** + * Observable counter for events dropped because the bounded hand-off queue was full when the hook + * tried to enqueue (backpressure). Incremented on the hook thread, surfaced on flush. + */ + private final AtomicLong droppedQueueOverflow = new AtomicLong(0); + + /** + * Per-reason-tag counters for evaluations whose context was truncated by copyPrunedContext. Keyed + * by the sorted comma-separated reason string (e.g. "max_key_length,max_value_length"). + * Incremented on the hook thread, drained and emitted on flush. + */ + private final ConcurrentHashMap contextTruncatedCounts = + new ConcurrentHashMap<>(); + + public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { + this(DEFAULT_CAPACITY, FLUSH_INTERVAL_SECONDS, SECONDS, sco, config); + } + + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final SharedCommunicationObjects sco, + final Config config) { + this(capacity, flushInterval, timeUnit, new BackendApiFactory(config, sco), config); + } + + /** Package-private constructor allowing a BackendApiFactory to be injected for tests. */ + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final BackendApiFactory backendApiFactory, + final Config config) { + this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); + this.serializer = + new FlagEvaluationSerializingHandler( + backendApiFactory, + queue, + flushInterval, + timeUnit, + FeatureFlagEvpContext.from(config), + droppedQueueOverflow, + contextTruncatedCounts, + this::close); + this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); + } + + @Override + public void start() { + synchronized (lifecycleLock) { + if (closed.get()) { + return; + } + // Register with the gateway so FlagEvalLoggingHook can route evaluations to this writer + FeatureFlaggingGateway.setFlagEvalWriter(this); + this.serializerThread.start(); + } + } + + /** Test seam: starts the worker thread WITHOUT registering with the global gateway. */ + void startForTest() { + synchronized (lifecycleLock) { + if (closed.get()) { + return; + } + this.serializerThread.start(); + } + } + + /** Test seam: current full-tier bucket count in the worker's aggregator. */ + int aggregatorFullTierSizeForTest() { + return serializer.aggregator.fullTierSize(); + } + + @Override + public void close() { + final boolean workerRunning; + synchronized (lifecycleLock) { + if (!closed.compareAndSet(false, true)) { + return; + } + // Disable and deregister from the gateway so no new events are enqueued. + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + FeatureFlaggingGateway.setFlagEvalWriter(null); + workerRunning = this.serializerThread.isAlive(); + if (workerRunning) { + // Ask the worker to drain the queue and final-flush, then interrupt to wake it from poll(). + serializer.requestShutdown(); + this.serializerThread.interrupt(); + } + } + if (Thread.currentThread() == this.serializerThread) { + return; + } + if (workerRunning) { + try { + // Bounded wait for the worker's final flush so queued events are not lost on shutdown. + this.serializerThread.join(TimeUnit.SECONDS.toMillis(5)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + // enqueue() is lock-free, so a producer that passed the closed check before close() ran can + // still land an event after the worker's final drain. Sweep the remainder so that loss is + // counted rather than silently stranded in a queue nobody polls again. Only safe once the + // worker is gone: the queue is single-consumer, so sweeping alongside a live worker would + // break that contract. If join() timed out the worker is still draining, so skip the sweep. + if (!this.serializerThread.isAlive()) { + sweepAndCountResidualEvents(); + } + } + + /** + * Counts events left in the queue after the worker has exited. Must only be called when the + * serializer thread is not alive - the queue permits a single consumer. + */ + private void sweepAndCountResidualEvents() { + long residual = 0; + while (queue.poll() != null) { + residual++; + } + countMetric(FLAG_EVALUATION_DROPPED_METRIC, residual, DROP_REASON_CLOSED); + } + + @Override + public void enqueue(final FlagEvalEvent event) { + if (event == null) { + return; + } + if (isClosedOrEnqueueDisabled()) { + countClosedDrop(); + return; + } + // Deliberately lock-free: the hand-off queue is MPSC by design, so serializing producers on a + // monitor here would negate that and turn every evaluation in every application thread into + // contention on one lock. A producer that passed the check above can still offer after close() + // has started; that residue is accounted for by the worker's bounded post-drain passes and by + // close()'s post-join sweep, so shutdown loss stays observable. + // + // Safe publication of the event (including the context snapshot built by the hook) comes from + // the queue's own offer/poll ordering, not from any monitor held here. + // + // Non-blocking offer. Count overflow so loss is observable rather than silent; the count is + // surfaced on the next flush. The hook's pre-queue guard (see FlagEvalLoggingHook) samples the + // queue depth before doing any context-copy work, so a saturated queue costs an + // AtomicInteger.get(); the offer here still races with the worker and can legitimately fail. + if (!queue.offer(event)) { + droppedQueueOverflow.incrementAndGet(); + } + } + + /** + * Reports whether the async hand-off queue currently has room for another event. Producers + * consult this before performing any expensive context-copy work so a saturated queue is observed + * as an O(1) read rather than a full snapshot followed by a discarded offer. Best-effort only: + * the worker can drain (or a peer producer can fill) between this check and the subsequent + * enqueue, so callers must still tolerate offer failure. + */ + public boolean hasCapacityForEnqueue() { + return queue.size() < queue.capacity(); + } + + /** Counts one queue-overflow drop without offering an event. */ + public void countPreQueueOverflow() { + droppedQueueOverflow.incrementAndGet(); + } + + @Override + public void countContextTruncated(final String reason) { + contextTruncatedCounts.computeIfAbsent(reason, k -> new AtomicLong(0)).incrementAndGet(); + } + + private boolean isClosedOrEnqueueDisabled() { + return closed.get() || !FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled(); + } + + private void countClosedDrop() { + // Count the drop for any early exit from enqueue, whether closed=true or the gate was disabled + // by the surrounding subsystem. FeatureFlaggingSystem.stop() flips the gate before this + // writer's close() runs, so an in-flight enqueue could race that flip and see gate=false while + // closed=false. Counting on either condition keeps shutdown-loss observable. + countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); + } + + /** Returns the count of events dropped due to queue-overflow backpressure (observable). */ + long droppedQueueOverflow() { + return droppedQueueOverflow.get(); + } + + /** Test seam: returns one queued event without starting the worker. */ + FlagEvalEvent pollQueuedEventForTest() { + return queue.poll(); + } + + /** Test seam: flushes serializer state without starting the worker. */ + void flushForTest() { + serializer.flush(); + } + + // ---- Serializing handler (background thread logic) ---- + + static class FlagEvaluationSerializingHandler implements Runnable { + private final MessagePassingBlockingQueue queue; + private final long ticksRequiredToFlush; + + @SuppressFBWarnings( + value = "AT_NONATOMIC_64BIT_PRIMITIVE", + justification = "the field is confined to the single serializer thread") + private long lastTicks; + + private final FeatureFlagEvpPublisher + evpPublisher; + final Map context; + private final AtomicLong droppedQueueOverflow; + private final ConcurrentHashMap contextTruncatedCounts; + private final Runnable errorCallback; + private final int payloadSizeLimitBytes; + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + // Shutdown coordination: set by close(), drives a final drain+flush before the worker exits. + private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); + private final CountDownLatch finalFlushDone = new CountDownLatch(1); + + FlagEvaluationSerializingHandler( + final BackendApiFactory backendApiFactory, + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final Map context, + final AtomicLong droppedQueueOverflow, + final ConcurrentHashMap contextTruncatedCounts, + final Runnable errorCallback) { + this( + backendApiFactory, + queue, + flushInterval, + timeUnit, + context, + droppedQueueOverflow, + contextTruncatedCounts, + errorCallback, + FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } + + FlagEvaluationSerializingHandler( + final BackendApiFactory backendApiFactory, + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final Map context, + final AtomicLong droppedQueueOverflow, + final ConcurrentHashMap contextTruncatedCounts, + final Runnable errorCallback, + final int payloadSizeLimitBytes) { + this.queue = queue; + this.evpPublisher = + new FeatureFlagEvpPublisher<>( + backendApiFactory, FlagEvaluationPayloads.FlagEvaluationsRequest.class, false); + this.context = context; + this.droppedQueueOverflow = droppedQueueOverflow; + this.contextTruncatedCounts = contextTruncatedCounts; + this.payloadSizeLimitBytes = payloadSizeLimitBytes; + this.lastTicks = System.nanoTime(); + this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); + this.errorCallback = errorCallback; + LOGGER.debug("starting flag evaluation serializer"); + } + + /** Signals the worker to drain the queue and perform a final flush before exiting. */ + void requestShutdown() { + shutdownRequested.set(true); + } + + @Override + public void run() { + if (!evpPublisher.start()) { + finalFlushDone.countDown(); + errorCallback.run(); + throw new IllegalArgumentException("EVP Proxy not available"); + } + try { + runDutyCycle(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + // On exit (interrupt or shutdown request), drain everything still buffered and flush it so + // queued events are not lost on shutdown. + // + // close() interrupts this thread to break it out of poll(). The final flush does socket + // I/O, and OkHttp fails fast on a thread whose interrupt flag is set, so clear the flag + // for the duration of the drain and restore it afterwards. Without this the flush that + // close() exists to guarantee throws IOException, the aggregated rows are discarded by + // the clear() in flush(), and the loss is invisible to the drop counters because those + // rows already left the queue. + final boolean wasInterrupted = Thread.interrupted(); + try { + drainAndFlush(); + } finally { + finalFlushDone.countDown(); + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + } + LOGGER.debug("flag evaluation processor worker exited."); + } + + private void runDutyCycle() throws InterruptedException { + final Thread thread = Thread.currentThread(); + while (!thread.isInterrupted() && !shutdownRequested.get()) { + final FlagEvalEvent event = queue.poll(100, TimeUnit.MILLISECONDS); + if (event != null) { + aggregateEvent(event); + } + flushIfNecessary(); + } + } + + void drainAndFlush() { + FlagEvalEvent event; + while ((event = queue.poll()) != null) { + aggregateEvent(event); + } + flush(); + } + + // ---- Aggregation logic ---- + + /** Routes an event into the full tier or degraded tier, or drops and counts on overflow. */ + void aggregateEvent(final FlagEvalEvent event) { + try { + aggregator.aggregate(event); + } catch (LinkageError | RuntimeException e) { + LOGGER.debug("Could not aggregate flag evaluation event", e); + } + } + + // ---- Flush logic ---- + + void flushIfNecessary() { + if (shouldFlush()) { + flush(); + } + } + + void flush() { + // Surface backpressure (queue-overflow) drops as an observable warning even when there is + // nothing else to flush. + final long qDrops = droppedQueueOverflow.getAndSet(0); + countMetric(FLAG_EVALUATION_DROPPED_METRIC, qDrops, DROP_REASON_QUEUE_OVERFLOW); + if (qDrops > 0) { + LOGGER.warn( + "flag evaluation queue full - dropped {} evaluation(s) under backpressure" + + " (best-effort telemetry)", + qDrops); + } + final long dgDrops = aggregator.droppedDegradedOverflow.getAndSet(0); + countMetric(FLAG_EVALUATION_DROPPED_METRIC, dgDrops, DROP_REASON_DEGRADED_CAP); + if (dgDrops > 0) { + LOGGER.warn( + "degraded aggregation tier full - dropped {} evaluation(s); raise degraded cap" + + " (best-effort telemetry)", + dgDrops); + } + + // Drain per-reason context-truncation counters and emit one metric per unique reason tag. + for (final Map.Entry entry : contextTruncatedCounts.entrySet()) { + final long count = entry.getValue().getAndSet(0); + if (count > 0) { + countMetric(FLAG_EVALUATION_CONTEXT_TRUNCATED_METRIC, count, entry.getKey()); + } + } + + if (aggregator.isEmpty()) { + return; + } + try { + countMetric( + FLAG_EVALUATION_DEGRADED_METRIC, + aggregator.degradedEvaluationCount(), + DEGRADED_REASON_CARDINALITY_CAP); + final List events = buildEventList(); + if (events.isEmpty()) { + return; + } + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(events, context, payloadSizeLimitBytes); + countMetric( + FLAG_EVALUATION_DROPPED_METRIC, + payloads.droppedPayloadLimit, + DROP_REASON_PAYLOAD_LIMIT); + countMetric( + FLAG_EVALUATION_DEGRADED_METRIC, + payloads.degradedPayloadLimit, + DEGRADED_REASON_PAYLOAD_LIMIT); + if (payloads.bodies.size() > 1) { + countMetric(FLAG_EVALUATION_SPLITS_METRIC, payloads.bodies.size() - 1, null); + } + if (payloads.droppedPayloadLimit > 0) { + LOGGER.warn( + "flag evaluation payload too large - dropped {} evaluation(s)" + + " (best-effort telemetry)", + payloads.droppedPayloadLimit); + } + for (final byte[] payload : payloads.bodies) { + evpPublisher.post(FLAG_EVALUATION_ROUTE, payload); + } + } catch (Exception e) { + LOGGER.error("Could not submit flag evaluations", e); + } finally { + // Best-effort: always clear the aggregator after a flush attempt. Retaining buckets across + // flushes on encode failure would let one unserializable value (for example a NaN Double a + // customer put in the context) permanently block every subsequent flush. + aggregator.clear(); + lastTicks = System.nanoTime(); + } + } + + private List buildEventList() { + final long flushTimeMs = System.currentTimeMillis(); + // Consent is read per bucket from the value each event snapshotted at evaluation time, not + // from the gateway here: CURRENT_CONFIG may have been overwritten by a later RC update since + // these evaluations happened, and reading it at flush would apply the wrong config's consent. + final List events = + new ArrayList<>(aggregator.bucketCount()); + for (final FlagEvaluationAggregator.EvalBucket bucket : aggregator.fullBuckets()) { + events.add( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, true, bucket.observeFullEvaluationData, flushTimeMs)); + } + for (final FlagEvaluationAggregator.EvalBucket bucket : aggregator.degradedBuckets()) { + events.add( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, false, bucket.observeFullEvaluationData, flushTimeMs)); + } + return events; + } + + private boolean shouldFlush() { + if (aggregator.isEmpty() && droppedQueueOverflow.get() == 0) { + return false; + } + final long nanoTime = System.nanoTime(); + final long ticks = nanoTime - lastTicks; + if (ticks > ticksRequiredToFlush) { + lastTicks = nanoTime; + return true; + } + return false; + } + } + + // ---- Test-seam inner class (package-private) ---- + + /** + * Test-accessible handler that exposes {@link #drainAndAggregate()} and {@link #flush()} without + * starting a real background thread. + */ + static class SerializingHandlerForTest extends FlagEvaluationSerializingHandler { + + SerializingHandlerForTest(final BackendApiFactory factory, final Map context) { + this(factory, context, FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } + + SerializingHandlerForTest( + final BackendApiFactory factory, + final Map context, + final int payloadSizeLimitBytes) { + super( + factory, + Queues.mpscBlockingConsumerArrayQueue(DEFAULT_CAPACITY), + Long.MAX_VALUE, // effectively never auto-flush + TimeUnit.NANOSECONDS, + context, + new AtomicLong(0), + new ConcurrentHashMap<>(), + () -> {}, + payloadSizeLimitBytes); + } + + private final List staged = new ArrayList<>(); + + /** Adds an event to the staged list (simulates hook enqueue). */ + void add(final FlagEvalEvent event) { + staged.add(event); + } + + /** Aggregates all staged events and returns the current aggregation state. */ + FlagEvaluationAggregator.AggregatedState drainAndAggregate() { + for (final FlagEvalEvent e : staged) { + aggregateEvent(e); + } + staged.clear(); + return aggregator.snapshot(); + } + + /** Simulates filling the full tier to GLOBAL_CAP by injecting synthetic distinct buckets. */ + void simulateFullTierAtCap() { + aggregator.simulateFullTierAtCap(); + } + + /** + * Simulates filling the degraded tier to DEGRADED_CAP by injecting synthetic distinct buckets. + */ + void simulateDegradedTierAtCap() { + aggregator.simulateDegradedTierAtCap(); + } + + void addDroppedDegradedOverflowForTest(final long count) { + aggregator.droppedDegradedOverflow.addAndGet(count); + } + + void addDegradedBucketForTest( + final String flagKey, + final String variant, + final String allocationKey, + final String errorMessage, + final long evalTimeMs) { + aggregator.addDegradedBucketForTest( + flagKey, variant, allocationKey, errorMessage, evalTimeMs); + } + + void clearAggregationForTest() { + aggregator.clear(); + } + + int fullTierSizeForTest() { + return aggregator.fullTierSize(); + } + } + + /** Factory method for test use - creates a SerializingHandlerForTest. */ + static SerializingHandlerForTest createHandlerForTest( + final BackendApiFactory factory, final Map context) { + return new SerializingHandlerForTest(factory, context); + } + + static SerializingHandlerForTest createHandlerForTest( + final BackendApiFactory factory, + final Map context, + final int payloadSizeLimitBytes) { + return new SerializingHandlerForTest(factory, context, payloadSizeLimitBytes); + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java index 428378b3055..e1912302935 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java @@ -41,7 +41,8 @@ + "test-only seams for injecting the root-span resolver and interceptor registrar. The " + "singleton itself is required (PR #11658 review) so the single, unremovable trace " + "interceptor is registered exactly once and survives subsystem start/stop.") -public final class SpanEnrichmentWriter implements FeatureFlaggingGateway.SpanEnrichmentListener { +public final class SpanEnrichmentWriter + implements FeatureFlaggingGateway.SpanEnrichmentListener, AutoCloseable { private static final Logger log = LoggerFactory.getLogger(SpanEnrichmentWriter.class); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java index 8951f20f866..f89ac1ddc38 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java @@ -40,6 +40,7 @@ final class UniversalFlagConfigParser implements ConfigurationDeserializer V1_ADAPTER = MOSHI.adapter(ServerConfiguration.class); @@ -126,6 +127,50 @@ public void toJson(@Nonnull final JsonWriter writer, @Nullable final MapOnly applies to Boolean.class (not primitive boolean), so mandatory primitive-boolean fields + * (e.g. Flag.enabled) keep their strict parse. + */ + static final class LenientBooleanAdapter extends JsonAdapter { + + static final Factory FACTORY = + new Factory() { + @Nullable + @Override + public JsonAdapter create( + @Nonnull final Type type, + @Nonnull final Set annotations, + @Nonnull final Moshi moshi) { + if (!annotations.isEmpty() || type != Boolean.class) { + return null; + } + return new LenientBooleanAdapter(); + } + }; + + @Nullable + @Override + public Boolean fromJson(@Nonnull final JsonReader reader) throws IOException { + if (reader.peek() == JsonReader.Token.BOOLEAN) { + return reader.nextBoolean(); + } + // null and every wrong-typed value collapse to null so the caller falls back to its default + // rather than the enclosing config being rejected wholesale. + reader.skipValue(); + return null; + } + + @Override + public void toJson(@Nonnull final JsonWriter writer, @Nullable final Boolean value) + throws IOException { + throw new UnsupportedOperationException("Reading only adapter"); + } + } + static final class InstantAdapter extends JsonAdapter { @Nullable diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java new file mode 100644 index 00000000000..379dd49e444 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java @@ -0,0 +1,71 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.intake.Intake; +import okhttp3.RequestBody; +import org.junit.jupiter.api.Test; + +class FeatureFlagEvpPublisherTest { + + @Test + void defaultPublisherRequestsResponseCompression() { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM, true)).thenReturn(backendApi); + + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class); + + publisher.start(); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM, true); + verifyNoMoreInteractions(factory); + } + + @Test + void responseCompressionCanBeDisabled() throws Exception { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM, false)).thenReturn(backendApi); + + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class, false); + + publisher.post("flagevaluation", new TestRequest("value")); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM, false); + verify(backendApi) + .post(eq("flagevaluation"), any(RequestBody.class), any(), isNull(), eq(false)); + } + + @Test + void postThrowsWhenEvpBackendApiCannotBeCreated() { + final BackendApiFactory factory = mock(BackendApiFactory.class); + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class); + + assertFalse(publisher.start()); + assertThrows( + IllegalStateException.class, + () -> publisher.post("flagevaluation", FeatureFlagEvpPublisher.utf8Bytes("{}"))); + } + + static class TestRequest { + public final String value; + + TestRequest(final String value) { + this.value = value; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java new file mode 100644 index 00000000000..06faeadefea --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java @@ -0,0 +1,425 @@ +package com.datadog.featureflag; + +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlagEvaluationAggregatorTest { + + @Test + void identicalEventsAggregateIntoOneBucketWithCount2() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("flag-a", "on", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("flag-a", "on", "alloc1", "user-1", 2000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(1, state.fullTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertEquals(2, bucket.count); + assertEquals(1000L, bucket.firstEvalMs); + assertEquals(2000L, bucket.lastEvalMs); + assertTrue(bucket.firstEvalMs <= bucket.lastEvalMs); + } + + @Test + void differentValueTypesProduceDifferentBuckets() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map attrsInt = new HashMap<>(); + attrsInt.put("score", 1); + final Map attrsStr = new HashMap<>(); + attrsStr.put("score", "1"); + + aggregator.aggregate(event("flag-b", "on", "alloc1", "user-1", 1000L, true, attrsInt)); + aggregator.aggregate(event("flag-b", "on", "alloc1", "user-1", 1000L, true, attrsStr)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(2, state.fullTier.size()); + } + + @Test + void nulCharactersInKeyFieldsDoNotCollide() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("a\0b", "c", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("a", "b\0c", "alloc1", "user-1", 1000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(2, state.fullTier.size()); + } + + @Test + void globalCapOverflowRoutesToDegradedTier() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.simulateFullTierAtCap(); + aggregator.aggregate(simpleEvent("extra-flag", "on")); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertTrue(state.degradedTier.size() > 0); + assertEquals(0, state.droppedDegradedOverflow); + } + + @Test + void degradedCapOverflowIncrementsDroppedCounter() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.simulateFullTierAtCap(); + aggregator.simulateDegradedTierAtCap(); + aggregator.aggregate(simpleEvent("drop-flag", "on")); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertTrue(state.droppedDegradedOverflow > 0); + } + + @Test + void perFlagCapOverflowRoutesToDegradedTierAndMergesSameDegradedKey() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + aggregator.perFlagCount.put("hot-flag", FlagEvaluationAggregator.PER_FLAG_CAP); + + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-2", 2000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(0, state.fullTier.size()); + assertEquals(1, state.degradedTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = + state.degradedTier.values().iterator().next(); + assertEquals(2, bucket.count); + assertEquals(1000L, bucket.firstEvalMs); + assertEquals(2000L, bucket.lastEvalMs); + } + + @Test + void mixedConsentDegradedEvaluationsMergeIntoOneBucket() { + // Mirror of mixedConsentEvaluationsForSameSubjectLandInDistinctBuckets, but for the degraded + // tier: the wire serializer for degraded rows drops the targeting key and context regardless + // of consent, so two events differing only in consent emit byte-identical JSON. They must + // share a bucket, otherwise DEGRADED_CAP is effectively halved for zero wire fidelity gain. + // The AND-fold on EvalBucket.observeFullEvaluationData still runs but the value has no + // downstream effect for degraded rows. + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + aggregator.perFlagCount.put("hot-flag", FlagEvaluationAggregator.PER_FLAG_CAP); + + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-1", 1000L, true, emptyMap())); + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-2", 2000L, false, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(0, state.fullTier.size()); + assertEquals(1, state.degradedTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = + state.degradedTier.values().iterator().next(); + assertEquals(2, bucket.count); + // AND-fold collapses to consent-off; benign for degraded rows but a documented invariant. + assertFalse(bucket.observeFullEvaluationData); + } + + @Test + void absentVariantSetsRuntimeDefaultUsed() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("flag-c", null, "alloc1", "user-1", 1000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(1, state.fullTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertTrue(bucket.runtimeDefaultUsed); + } + + @Test + void aggregatorStoresPrunedAttrsVerbatim() { + // Hot-path hook (DDEvaluator#copyPrunedContext) now delivers an already-pruned map. + // The aggregator no longer re-prunes; it stores what it is given. + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map preprunedAttrs = new HashMap<>(); + for (int i = 0; i < 100; i++) { + preprunedAttrs.put("key" + i, "v" + i); + } + + aggregator.aggregate(event("flag-d", "on", "alloc1", "user-1", 1000L, true, preprunedAttrs)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertEquals(100, bucket.prunedContextFieldCount()); + assertEquals(100, bucket.prunedAttrs.size()); + } + + @Test + void emptyContextInputsProduceEmptyCanonicalKey() { + assertEquals("", FlagEvaluationAggregator.canonicalContextKey(null)); + assertEquals("", FlagEvaluationAggregator.canonicalContextKey(emptyMap())); + } + + @Test + void canonicalContextKeyEncodesSupportedValueTypes() { + final Map attrs = new HashMap<>(); + attrs.put("bool", true); + attrs.put("double", 1.5d); + attrs.put("float", 1.25f); + attrs.put("int", 1); + attrs.put("long", 2L); + attrs.put("null", null); + attrs.put("object", new StringBuilder("other")); + attrs.put("string", "value"); + + final String key = FlagEvaluationAggregator.canonicalContextKey(attrs); + + assertEquals(key, FlagEvaluationAggregator.canonicalContextKey(new HashMap<>(attrs))); + assertTrue(key.contains("bool")); + assertTrue(key.contains("string")); + assertTrue(key.contains("other")); + } + + @Test + void aggregatorStoresPrePrunedAttrsWithoutRePruning() { + // Value-length pruning moved to DDEvaluator#copyPrunedContext (hot path). The aggregator + // stores what it is given, so any long value present in the input remains present. + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map preprunedAttrs = new HashMap<>(); + preprunedAttrs.put("short-val", "ok"); + + aggregator.aggregate(event("flag-e", "on", "alloc1", "user-1", 1000L, true, preprunedAttrs)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertTrue(bucket.prunedAttrs.containsKey("short-val")); + } + + @Test + void capSizingUsesNamedScaleConstants() { + assertEquals(125_000, FlagEvaluationAggregator.FULL_BUCKET_SIZING_BASIS); + assertEquals(10_000, FlagEvaluationAggregator.PER_FLAG_BUCKET_SIZING_BASIS); + assertEquals(25_000, FlagEvaluationAggregator.DEGRADED_BUCKET_SIZING_BASIS); + assertEquals(131_072, FlagEvaluationAggregator.GLOBAL_CAP); + assertEquals(10_000, FlagEvaluationAggregator.PER_FLAG_CAP); + assertEquals(32_768, FlagEvaluationAggregator.DEGRADED_CAP); + } + + @Test + void flagEvalEventDoesNotCarryReason() { + final boolean hasReasonField = + Arrays.stream( + datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent.class + .getDeclaredFields()) + .anyMatch(field -> field.getName().equals("reason")); + + assertFalse(hasReasonField); + } + + @Test + void evalBucketTracksBoundsDefaultStateAndNullContextFieldCount() { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "bucket-flag", "on", "alloc1", "user-1", null, 1000L, false, null, false); + + assertEquals(0, bucket.prunedContextFieldCount()); + + bucket.merge(900L, true); + bucket.merge(1100L, false); + bucket.merge(1000L, false); + + assertEquals(4, bucket.count); + assertEquals(900L, bucket.firstEvalMs); + assertEquals(1100L, bucket.lastEvalMs); + assertTrue(bucket.runtimeDefaultUsed); + } + + @Test + void fullKeyEqualityUsesEveryDimension() { + final FlagEvaluationAggregator.FullKey base = + fullKey("flag", "on", "alloc", false, "error", "user", "ctx"); + final FlagEvaluationAggregator.FullKey same = + fullKey("flag", "on", "alloc", false, "error", "user", "ctx"); + + assertEquals(base, base); + assertEquals(base, same); + assertEquals(base.hashCode(), same.hashCode()); + assertNotEquals(base, null); + assertNotEquals(base, "not-a-key"); + assertNotEquals(base, fullKey("other", "on", "alloc", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "off", "alloc", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "other", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", true, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "other", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "error", "other", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "error", "user", "other")); + } + + @Test + void degradedKeyEqualityUsesEveryDimension() { + final FlagEvaluationAggregator.DegradedKey base = + degradedKey("flag", "on", "alloc", false, "error"); + final FlagEvaluationAggregator.DegradedKey same = + degradedKey("flag", "on", "alloc", false, "error"); + + assertEquals(base, base); + assertEquals(base, same); + assertEquals(base.hashCode(), same.hashCode()); + assertNotEquals(base, null); + assertNotEquals(base, "not-a-key"); + assertNotEquals(base, degradedKey("other", "on", "alloc", false, "error")); + assertNotEquals(base, degradedKey("flag", "off", "alloc", false, "error")); + assertNotEquals(base, degradedKey("flag", "on", "other", false, "error")); + assertNotEquals(base, degradedKey("flag", "on", "alloc", true, "error")); + assertNotEquals(base, degradedKey("flag", "on", "alloc", false, "other")); + } + + @Test + void mixedConsentEvaluationsForSameSubjectLandInDistinctBuckets() { + // Consent is part of FullKey: two evaluations that differ only in consent produce different + // wire rows (raw vs hashed targeting key, context vs no context) and belong in different + // buckets. Merging them would silently downgrade the consent-on row to the protected shape. + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + aggregator.aggregate(event("fold-flag", "on", "alloc1", "user-1", 1000L, true, emptyMap())); + aggregator.aggregate(event("fold-flag", "on", "alloc1", "user-1", 2000L, false, emptyMap())); + + assertEquals(2, aggregator.fullTierSize()); + int onCount = 0; + int offCount = 0; + for (final FlagEvaluationAggregator.EvalBucket bucket : + aggregator.snapshot().fullTier.values()) { + if (bucket.observeFullEvaluationData) { + onCount++; + } else { + offCount++; + } + } + assertEquals(1, onCount); + assertEquals(1, offCount); + } + + @Test + void sameConsentEvaluationsForSameSubjectMergeIntoOneBucket() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + aggregator.aggregate(event("fold-flag", "on", "alloc1", "user-1", 1000L, true, emptyMap())); + aggregator.aggregate(event("fold-flag", "on", "alloc1", "user-1", 2000L, true, emptyMap())); + + final FlagEvaluationAggregator.EvalBucket bucket = + aggregator.snapshot().fullTier.values().iterator().next(); + assertEquals(2, bucket.count); + assertTrue(bucket.observeFullEvaluationData); + } + + @Test + void protectedPathCollapsesDifferingContextIntoOneBucket() { + // Same subject, different request-id contexts, consent off: the context is dropped on emit so + // it must not fragment full-tier buckets or the per-flag cap blows out under real traffic. + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map ctx1 = new HashMap<>(); + ctx1.put("request_id", "req-1"); + final Map ctx2 = new HashMap<>(); + ctx2.put("request_id", "req-2"); + final Map ctx3 = new HashMap<>(); + ctx3.put("request_id", "req-3"); + + aggregator.aggregate(event("checkout", "on", "alloc1", "alice", 1000L, false, ctx1)); + aggregator.aggregate(event("checkout", "on", "alloc1", "alice", 2000L, false, ctx2)); + aggregator.aggregate(event("checkout", "on", "alloc1", "alice", 3000L, false, ctx3)); + + assertEquals(1, aggregator.fullTierSize()); + final FlagEvaluationAggregator.EvalBucket bucket = + aggregator.snapshot().fullTier.values().iterator().next(); + assertEquals(3, bucket.count); + assertFalse(bucket.observeFullEvaluationData); + assertEquals(0, bucket.prunedContextFieldCount()); + } + + @Test + void protectedPathSeparatesDifferentSubjects() { + // Different targeting keys must still fall into distinct buckets on the protected path — the + // (hashed) targeting key stays part of the aggregation identity. + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + aggregator.aggregate(event("checkout", "on", "alloc1", "alice", 1000L, false, emptyMap())); + aggregator.aggregate(event("checkout", "on", "alloc1", "bob", 2000L, false, emptyMap())); + + assertEquals(2, aggregator.fullTierSize()); + } + + @Test + void fullPathStillSplitsBucketsOnDifferingContext() { + // Consent-on preserves the previous behaviour: distinct contexts remain distinct buckets so + // each raw context is emitted verbatim. + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map ctx1 = new HashMap<>(); + ctx1.put("plan", "pro"); + ctx1.put("request_id", "req-1"); + final Map ctx2 = new HashMap<>(); + ctx2.put("plan", "pro"); + ctx2.put("request_id", "req-2"); + + aggregator.aggregate(event("checkout", "on", "alloc1", "alice", 1000L, true, ctx1)); + aggregator.aggregate(event("checkout", "on", "alloc1", "alice", 2000L, true, ctx2)); + + assertEquals(2, aggregator.fullTierSize()); + } + + private static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final boolean observeFullEvaluationData, + final Map attrs) { + return new FlagEvalEvent( + flagKey, + variant, + allocationKey, + targetingKey, + null, + evalTimeMs, + observeFullEvaluationData, + attrs); + } + + private static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + return new FlagEvalEvent(flagKey, variant, allocationKey, targetingKey, evalTimeMs, attrs); + } + + private static FlagEvalEvent simpleEvent(final String flagKey, final String variant) { + return event(flagKey, variant, "alloc1", "user-1", 1000L, emptyMap()); + } + + private static FlagEvaluationAggregator.FullKey fullKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final String targetingKey, + final String contextKey) { + return new FlagEvaluationAggregator.FullKey( + flagKey, + variant, + allocationKey, + runtimeDefaultUsed, + errorMessage, + targetingKey, + contextKey, + false); + } + + private static FlagEvaluationAggregator.DegradedKey degradedKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage) { + return new FlagEvaluationAggregator.DegradedKey( + flagKey, variant, allocationKey, runtimeDefaultUsed, errorMessage); + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java new file mode 100644 index 00000000000..d4ca517fffd --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -0,0 +1,453 @@ +package com.datadog.featureflag; + +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlagEvaluationPayloadsTest { + + private static final long EVAL_MS = 1_760_000_000_000L; + private static final Map CONTEXT = context(); + private static final JsonAdapter> JSON_MAP; + + static { + final Moshi moshi = new Moshi.Builder().build(); + final Type type = Types.newParameterizedType(Map.class, String.class, Object.class); + JSON_MAP = moshi.adapter(type); + } + + @Test + void fullTierPayloadUsesWorkerWireShape() throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event("my-flag", "on", "alloc-x", "user-1", 1, attrs)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertObjectWithKey(ev.get("variant"), "on"); + assertObjectWithKey(ev.get("allocation"), "alloc-x"); + assertObjectWithKey(ev.get("flag"), "my-flag"); + final Map ctx = (Map) ev.get("context"); + assertNotNull(ctx); + final Map evalAttrs = (Map) ctx.get("evaluation"); + assertNotNull(evalAttrs); + assertEquals("us-east-1", evalAttrs.get("region")); + assertFalse(ev.containsKey("reason")); + } + + @Test + void eventFromFullBucketUsesFlushTimeAndEvaluationBounds() throws Exception { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "ts-flag", "on", "alloc1", "user-1", null, EVAL_MS, false, emptyMap(), true); + bucket.merge(EVAL_MS + 10, false); + final long flushTimeMs = EVAL_MS + 5_000; + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, true, true, flushTimeMs)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertEquals((double) flushTimeMs, ((Number) ev.get("timestamp")).doubleValue()); + assertEquals((double) EVAL_MS, ((Number) ev.get("first_evaluation")).doubleValue()); + assertEquals((double) (EVAL_MS + 10), ((Number) ev.get("last_evaluation")).doubleValue()); + assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); + } + + @Test + void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "dg-flag", "on", "alloc1", null, null, EVAL_MS, false, null, false); + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, false, true, EVAL_MS)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void fullTierWithObserveFullEvaluationDataTrueEmitsRawTargetingKeyAndContext() throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "pii-flag", + "on", + "alloc1", + "jane.doe@datadoghq.com", + null, + EVAL_MS, + false, + attrs, + true); + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, true, true, EVAL_MS)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertEquals("jane.doe@datadoghq.com", ev.get("targeting_key")); + final Map ctx = (Map) ev.get("context"); + assertNotNull(ctx); + final Map evalAttrs = (Map) ctx.get("evaluation"); + assertNotNull(evalAttrs); + assertEquals("us-east-1", evalAttrs.get("region")); + } + + @Test + void fullTierWithObserveFullEvaluationDataFalseHashesTargetingKeyAndOmitsContext() + throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "pii-flag", + "on", + "alloc1", + "jane.doe@datadoghq.com", + null, + EVAL_MS, + false, + attrs, + false); + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, true, false, EVAL_MS)), + CONTEXT, + 1_000_000); + final String rawJson = + new String(payloads.bodies.get(0), java.nio.charset.StandardCharsets.UTF_8); + + // The raw wire bytes must carry the hashed key and must not leak the raw PII value or the + // per-event evaluation context — these are the exact properties system-tests asserts over the + // wire. (The batch envelope has its own top-level "context" field, so we guard on the nested + // "evaluation" key instead, which only appears inside a per-event context object.) + assertTrue( + rawJson.contains( + "sha256_b4698f9b6d186781fa8dc59e533578fa2d8379a46b1cf6db85cda6aa9c99e51b")); + assertFalse(rawJson.contains("jane.doe@datadoghq.com")); + assertFalse(rawJson.contains("\"evaluation\":")); + + final Map ev = firstEvent(parse(payloads.bodies.get(0))); + assertEquals( + "sha256_b4698f9b6d186781fa8dc59e533578fa2d8379a46b1cf6db85cda6aa9c99e51b", + ev.get("targeting_key")); + assertFalse(ev.containsKey("context")); + } + + @Test + void splitPayloadsByEncodedSize() throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 180)); + final java.util.ArrayList events = + new java.util.ArrayList<>(); + for (int i = 0; i < 4; i++) { + events.add(event("split-flag-" + i, "on", "alloc1", "user-" + i, 1, attrs)); + } + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(events, CONTEXT, 1_100); + + assertTrue(payloads.bodies.size() > 1); + int eventCount = 0; + for (final byte[] body : payloads.bodies) { + assertTrue(body.length <= 1_100); + eventCount += eventCount(parse(body)); + } + assertEquals(4, eventCount); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void oversizedFullPayloadRowIsDegradedBeforeDrop() throws Exception { + final Map attrs = new HashMap<>(); + for (int i = 0; i < 4; i++) { + attrs.put("payload-" + i, repeat('x', 200)); + } + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event("oversized-full", "on", "alloc1", "user-1", 2, attrs)), + CONTEXT, + 512); + + assertEquals(1, payloads.bodies.size()); + assertTrue(payloads.bodies.get(0).length <= 512); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(2, payloads.degradedPayloadLimit); + final Map ev = firstEvent(parse(payloads.bodies.get(0))); + assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void oversizedFullPayloadRowStartsNewPayloadWhenDegradedRowFitsByItself() throws Exception { + final FlagEvaluationPayloads.FlagEvaluationEvent first = + event("first-flag", "on", "alloc1", "user-1", 1, emptyMap()); + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 400)); + final FlagEvaluationPayloads.FlagEvaluationEvent second = + event("second-flag", "on", "alloc1", "user-2", 3, attrs); + final FlagEvaluationPayloads.FlagEvaluationEvent degradedSecond = + second.withoutTargetingKeyAndContext(); + assertNotNull(degradedSecond); + + final int firstPayloadSize = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList(first), CONTEXT, 1_000_000) + .bodies + .get(0) + .length; + final int degradedPayloadSize = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList(degradedSecond), CONTEXT, 1_000_000) + .bodies + .get(0) + .length; + final int limit = Math.max(firstPayloadSize, degradedPayloadSize); + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(Arrays.asList(first, second), CONTEXT, limit); + + assertEquals(2, payloads.bodies.size()); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(3, payloads.degradedPayloadLimit); + assertEquals(1, eventCount(parse(payloads.bodies.get(0)))); + final Map ev = firstEvent(parse(payloads.bodies.get(1))); + assertObjectWithKey(ev.get("flag"), "second-flag"); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void oversizedDegradedPayloadRowIsDropped() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event(repeat('f', 512), "on", "alloc1", null, 2, emptyMap())), + CONTEXT, + 128); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(2, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void oversizedFullPayloadRowIsDroppedWhenDegradedRowStillExceedsLimit() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event(repeat('f', 512), "on", "alloc1", "user-1", 2, emptyMap())), + CONTEXT, + 128); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(2, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void errorPayloadSerializesErrorObject() throws Exception { + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + "err-flag", + EVAL_MS, + EVAL_MS, + 1, + null, + null, + null, + true, + "type mismatch", + null)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + final Map error = (Map) ev.get("error"); + assertNotNull(error); + assertEquals("type mismatch", error.get("message")); + assertEquals(Boolean.TRUE, ev.get("runtime_default_used")); + } + + @Test + void emptyEventListProducesNoPayloads() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(java.util.Collections.emptyList(), CONTEXT, 1_000_000); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void requestDtoStoresContextAndEvents() { + final List events = + java.util.Collections.singletonList( + event("dto-flag", "on", "alloc1", "user-1", 1, emptyMap())); + + final FlagEvaluationPayloads.FlagEvaluationsRequest request = + new FlagEvaluationPayloads.FlagEvaluationsRequest(CONTEXT, events); + + assertEquals(CONTEXT, request.context); + assertEquals(events, request.flagEvaluations); + } + + @Test + void degradingEventWithMissingOptionalFieldsKeepsOptionalObjectsAbsent() { + final FlagEvaluationPayloads.FlagEvaluationEvent degraded = + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + "default-flag", + EVAL_MS, + EVAL_MS, + 1, + null, + null, + "user-1", + true, + null, + null) + .withoutTargetingKeyAndContext(); + + assertNotNull(degraded); + assertNull(degraded.variant); + assertNull(degraded.allocation); + assertNull(degraded.targeting_key); + assertNull(degraded.context); + assertNull(degraded.error); + assertEquals(Boolean.TRUE, degraded.runtime_default_used); + } + + @Test + void emptyOptionalStringsAreTreatedAsAbsentWhenContextIsPresent() { + final Map attrs = new HashMap<>(); + attrs.put("tier", "gold"); + final FlagEvaluationPayloads.FlagEvaluationEvent event = + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, "empty-optionals", EVAL_MS, EVAL_MS, 1, "", "", null, false, "", attrs); + + assertNull(event.variant); + assertNull(event.allocation); + assertNull(event.error); + assertNotNull(event.context); + + final FlagEvaluationPayloads.FlagEvaluationEvent degraded = + event.withoutTargetingKeyAndContext(); + assertNotNull(degraded); + assertNull(degraded.targeting_key); + assertNull(degraded.context); + } + + private static FlagEvaluationPayloads.FlagEvaluationEvent event( + final String flagKey, + final String variant, + final String allocation, + final String targetingKey, + final long count, + final Map attrs) { + return new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + flagKey, + EVAL_MS, + EVAL_MS, + count, + variant, + allocation, + targetingKey, + false, + null, + attrs); + } + + private static Map firstPayload( + final FlagEvaluationPayloads.EncodedPayloads payloads) throws Exception { + assertEquals(1, payloads.bodies.size()); + return parse(payloads.bodies.get(0)); + } + + private static Map parse(final byte[] body) throws Exception { + return JSON_MAP.fromJson(new String(body, java.nio.charset.StandardCharsets.UTF_8)); + } + + @SuppressWarnings("unchecked") + private static Map firstEvent(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events); + assertFalse(events.isEmpty()); + return (Map) events.get(0); + } + + @SuppressWarnings("unchecked") + private static int eventCount(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events); + return events.size(); + } + + private static void assertObjectWithKey(final Object object, final String expectedKey) { + assertTrue(object instanceof Map); + assertEquals(expectedKey, ((Map) object).get("key")); + } + + private static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + java.util.Arrays.fill(chars, c); + return new String(chars); + } + + private static Map context() { + final Map context = new HashMap<>(); + context.put("service", "test-service"); + return context; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java new file mode 100644 index 00000000000..4a65a81a8bf --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -0,0 +1,239 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.telemetry.CoreMetricCollector; +import datadog.trace.api.telemetry.MetricCollector; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import okhttp3.RequestBody; +import okio.Buffer; + +final class FlagEvaluationTestSupport { + + static final long REALISTIC_EVAL_MS = 1_760_000_000_000L; + static final JsonAdapter> JSON_MAP; + + static { + final Moshi moshi = new Moshi.Builder().build(); + final Type type = Types.newParameterizedType(Map.class, String.class, Object.class); + JSON_MAP = moshi.adapter(type); + } + + private FlagEvaluationTestSupport() {} + + static void clearCoreMetrics() { + CoreMetricCollector.getInstance().drain(); + } + + static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + return new FlagEvalEvent(flagKey, variant, allocationKey, targetingKey, evalTimeMs, attrs); + } + + static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final boolean observeFullEvaluationData, + final Map attrs) { + return new FlagEvalEvent( + flagKey, + variant, + allocationKey, + targetingKey, + null, + evalTimeMs, + observeFullEvaluationData, + attrs); + } + + static FlagEvalEvent errorEvent( + final String flagKey, final String errorMessage, final long evalTimeMs) { + return new FlagEvalEvent( + flagKey, null, null, null, errorMessage, evalTimeMs, java.util.Collections.emptyMap()); + } + + static FlagEvalEvent simpleEvent(final String flagKey, final String variant) { + return event(flagKey, variant, "alloc1", "user-1", 1000L, java.util.Collections.emptyMap()); + } + + static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } + + static TestWriterSetup buildTestWriter(final BackendApi mockEvp) { + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + + final Map context = new HashMap<>(); + context.put("service", "test-service"); + + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context); + + return new TestWriterSetup(handler, mockEvp, factory); + } + + static TestWriterSetup buildTestWriter( + final BackendApi mockEvp, final int payloadSizeLimitBytes) { + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + + final Map context = new HashMap<>(); + context.put("service", "test-service"); + + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context, payloadSizeLimitBytes); + + return new TestWriterSetup(handler, mockEvp, factory); + } + + static CapturedJson flushAndCapture(final TestWriterSetup setup) throws Exception { + final List captured = flushAndCaptureAll(setup); + assertEquals(1, captured.size(), "Expected exactly one posted payload"); + return captured.get(0); + } + + static List flushAndCaptureAll(final TestWriterSetup setup) throws Exception { + final List captured = new ArrayList<>(); + when(setup.mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured.add(inv.getArgument(1)); + return null; + }); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + final List json = new ArrayList<>(); + for (final RequestBody body : captured) { + json.add(readJson(body)); + } + return json; + } + + static CapturedJson readJson(final RequestBody body) throws Exception { + assertNotNull(body, "RequestBody must have been posted"); + final Buffer buf = new Buffer(); + body.writeTo(buf); + final String raw = buf.readUtf8(); + return new CapturedJson(raw, JSON_MAP.fromJson(raw)); + } + + static Map flushAndCaptureJson(final TestWriterSetup setup) throws Exception { + return flushAndCapture(setup).parsed; + } + + static long metricSum( + final Collection metrics, + final String metricName, + final String tag) { + long sum = 0; + for (final MetricCollector.Metric metric : metrics) { + if (!metricName.equals(metric.metricName)) { + continue; + } + if (tag == null) { + if (!metric.tags.isEmpty()) { + continue; + } + } else if (!metric.tags.contains(tag)) { + continue; + } + sum += metric.value.longValue(); + } + return sum; + } + + static datadog.trace.api.Config cfg() { + final datadog.trace.api.Config config = mock(datadog.trace.api.Config.class); + when(config.getServiceName()).thenReturn("test-service"); + return config; + } + + static void assertObjectWithKey(final Object o, final String expectedKey, final String msg) { + assertTrue(o instanceof Map, msg + " (must be a JSON object, not a bare string)"); + assertEquals(expectedKey, ((Map) o).get("key"), msg); + } + + @SuppressWarnings("unchecked") + static Map firstEvent(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events, "flagEvaluations array must be present"); + assertFalse(events.isEmpty(), "flagEvaluations must not be empty"); + return (Map) events.get(0); + } + + @SuppressWarnings("unchecked") + static int eventCount(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events, "flagEvaluations array must be present"); + return events.size(); + } + + @SuppressWarnings("unchecked") + static Map eventForFlag(final Map batch, final String flagKey) { + final List events = (List) batch.get("flagEvaluations"); + for (final Object o : events) { + final Map ev = (Map) o; + final Map flag = (Map) ev.get("flag"); + if (flag != null && flagKey.equals(flag.get("key"))) { + return ev; + } + } + return null; + } + + static class TestWriterSetup { + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + final BackendApi mockEvp; + final BackendApiFactory factory; + + TestWriterSetup( + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler, + final BackendApi mockEvp, + final BackendApiFactory factory) { + this.handler = handler; + this.mockEvp = mockEvp; + this.factory = factory; + } + } + + static class CapturedJson { + final String raw; + final Map parsed; + + CapturedJson(final String raw, final Map parsed) { + this.raw = raw; + this.parsed = parsed; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java new file mode 100644 index 00000000000..b354fdbb6c4 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -0,0 +1,783 @@ +package com.datadog.featureflag; + +import static com.datadog.featureflag.FlagEvaluationTestSupport.JSON_MAP; +import static com.datadog.featureflag.FlagEvaluationTestSupport.buildTestWriter; +import static com.datadog.featureflag.FlagEvaluationTestSupport.cfg; +import static com.datadog.featureflag.FlagEvaluationTestSupport.clearCoreMetrics; +import static com.datadog.featureflag.FlagEvaluationTestSupport.event; +import static com.datadog.featureflag.FlagEvaluationTestSupport.eventForFlag; +import static com.datadog.featureflag.FlagEvaluationTestSupport.flushAndCapture; +import static com.datadog.featureflag.FlagEvaluationTestSupport.flushAndCaptureJson; +import static com.datadog.featureflag.FlagEvaluationTestSupport.metricSum; +import static com.datadog.featureflag.FlagEvaluationTestSupport.repeat; +import static com.datadog.featureflag.FlagEvaluationTestSupport.simpleEvent; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.common.queue.MessagePassingBlockingQueue; +import datadog.common.queue.Queues; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import datadog.trace.api.intake.Intake; +import datadog.trace.api.telemetry.CoreMetricCollector; +import datadog.trace.api.telemetry.MetricCollector; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import okhttp3.RequestBody; +import okio.Buffer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class FlagEvaluationWriterImplTest { + + @BeforeEach + void clearCoreMetricsBefore() { + clearCoreMetrics(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + + @AfterEach + void clearCoreMetricsAfter() { + clearCoreMetrics(); + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + // Reset the dispatched UFC state so observeFullEvaluationData can't leak into other tests. + FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + } + + @Test + void degradedCapOverflowTelemetryIsEmittedOnFlush() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.addDroppedDegradedOverflowForTest(3); + setup.handler.flush(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 3, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_DEGRADED_CAP)); + } + + @Test + void startRegistersWriterAndCloseDeregistersIt() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.start(); + assertEquals(writer, FeatureFlaggingGateway.getFlagEvalWriter()); + + writer.close(); + writer.close(); + writer.start(); + + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } + + @Test + void queueOverflowIncrementsObservableDropCounter() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(2, 10L, TimeUnit.SECONDS, factory, cfg()); + + for (int i = 0; i < 100; i++) { + writer.enqueue(simpleEvent("of-flag", "on")); + } + + assertTrue(writer.droppedQueueOverflow() > 0); + final long queueDrops = writer.droppedQueueOverflow(); + writer.flushForTest(); + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + queueDrops, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_QUEUE_OVERFLOW)); + } + + @Test + void enqueueAfterCloseIsDroppedAndCounted() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.close(); + writer.enqueue(simpleEvent("closed-flag", "on")); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CLOSED)); + assertNull(writer.pollQueuedEventForTest()); + } + + @Test + void enqueueDisabledDropsAndCountsAsClosedDrop() { + // FeatureFlaggingSystem.stop() flips the gate before this writer's close() runs. Producers + // that race the gate flip must count the drop, otherwise shutdown loss stays invisible. + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + writer.enqueue(simpleEvent("disabled-flag", "on")); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CLOSED)); + assertNull(writer.pollQueuedEventForTest()); + } + + @Test + void closeSweepsAndCountsEventsLeftInTheQueue() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + // The worker is never started, so nothing drains these; close() must account for them rather + // than leave them silently stranded. Stands in for the narrow window where a lock-free + // producer offers after the worker's final drain. + writer.enqueue(simpleEvent("residual-flag-1", "on")); + writer.enqueue(simpleEvent("residual-flag-2", "on")); + writer.close(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 2, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CLOSED)); + assertNull(writer.pollQueuedEventForTest()); + } + + @Test + void enqueueIgnoresNullEvent() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.enqueue(null); + + assertNull(writer.pollQueuedEventForTest()); + assertEquals(0, writer.droppedQueueOverflow()); + } + + @Test + void enqueueDoesNotAggregateOnTheCallingThread() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.enqueue(simpleEvent("g2-flag", "on")); + writer.enqueue(simpleEvent("g2-flag", "on")); + + assertEquals(0, writer.aggregatorFullTierSizeForTest()); + assertEquals(0, writer.droppedQueueOverflow()); + } + + @Test + void handlerRunFailsFastWhenEvpProxyIsUnavailable() { + final BackendApiFactory factory = mock(BackendApiFactory.class); + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context()); + + assertThrows(IllegalArgumentException.class, handler::run); + } + + @Test + void flushIfNecessarySkipsEmptyStateAndWaitsForInterval() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.flushIfNecessary(); + setup.handler.add(simpleEvent("pending-flag", "on")); + setup.handler.drainAndAggregate(); + setup.handler.flushIfNecessary(); + + assertEquals(1, setup.handler.fullTierSizeForTest()); + } + + @Test + void flushIfNecessaryDoesNotReturnEarlyWhenOnlyQueueDropsArePending() { + final AtomicLong queueDrops = new AtomicLong(1); + final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = + new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( + mock(BackendApiFactory.class), + Queues.mpscBlockingConsumerArrayQueue(16), + Long.MAX_VALUE, + TimeUnit.NANOSECONDS, + context(), + queueDrops, + new java.util.concurrent.ConcurrentHashMap<>(), + () -> {}); + + handler.flushIfNecessary(); + + assertEquals(1, queueDrops.get()); + } + + @Test + @SuppressWarnings("unchecked") + void workerHandlesEmptyPolls() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final MessagePassingBlockingQueue queue = + mock(MessagePassingBlockingQueue.class); + when(queue.poll(100, TimeUnit.MILLISECONDS)) + .thenAnswer( + invocation -> { + Thread.currentThread().interrupt(); + return null; + }); + final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = + new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( + factory, + queue, + Long.MAX_VALUE, + TimeUnit.NANOSECONDS, + context(), + new AtomicLong(0), + new java.util.concurrent.ConcurrentHashMap<>(), + () -> {}); + + handler.run(); + + assertTrue(Thread.interrupted()); + } + + @Test + void degradedBucketsAreSerializedWithoutTargetingKeyOrContext() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.addDegradedBucketForTest("degraded-flag", "on", "alloc1", null, 1000L); + + final Map json = flushAndCaptureJson(setup); + + final Map ev = eventForFlag(json, "degraded-flag"); + assertNotNull(ev); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void testHandlerCanSimulateAndClearDegradedTierAtCap() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.simulateDegradedTierAtCap(); + setup.handler.clearAggregationForTest(); + setup.handler.add(simpleEvent("after-clear", "on")); + setup.handler.drainAndAggregate(); + + assertEquals(1, setup.handler.fullTierSizeForTest()); + } + + @Test + void payloadLimitDropsAreCountedOnFlush() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp, 128); + setup.handler.add(event(repeat('f', 512), "on", "alloc1", "user-1", 1000L, emptyMap())); + + setup.handler.drainAndAggregate(); + setup.handler.flush(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_PAYLOAD_LIMIT)); + } + + @Test + void finalFlushRunsWithoutTheInterruptFlagSet() throws Exception { + // close() interrupts the worker to break it out of poll(). The final flush does socket I/O, + // and OkHttp fails fast on an interrupted thread, so the flag must be clear by the time the + // publisher is called. A mock publisher ignores the flag, so assert on it directly. + final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); + final boolean[] interruptedDuringPost = {true}; + final BackendApi mockEvp = mock(BackendApi.class); + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + interruptedDuringPost[0] = Thread.currentThread().isInterrupted(); + posted.countDown(); + return null; + }); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, factory, cfg()); + + writer.startForTest(); + writer.enqueue(simpleEvent("interrupt-flag", "on")); + writer.close(); + + assertTrue(posted.await(5, TimeUnit.SECONDS)); + assertFalse( + interruptedDuringPost[0], + "final flush must not run on an interrupted thread; OkHttp fails fast and the drained" + + " rows are lost without being counted"); + } + + @Test + void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { + final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); + final RequestBody[] captured = {null}; + final BackendApi mockEvp = mock(BackendApi.class); + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured[0] = inv.getArgument(1); + posted.countDown(); + return null; + }); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, factory, cfg()); + + writer.startForTest(); + writer.enqueue(simpleEvent("shutdown-flag", "on")); + writer.close(); + + assertTrue(posted.await(5, TimeUnit.SECONDS)); + assertNotNull(captured[0]); + final Buffer buf = new Buffer(); + captured[0].writeTo(buf); + final Map json = JSON_MAP.fromJson(buf.readUtf8()); + assertNotNull(eventForFlag(json, "shutdown-flag")); + } + + @Test + void continuousTrafficFlushesWithoutWaitingForIdle() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(1 << 12, 1, TimeUnit.MILLISECONDS, factory, cfg()); + + writer.startForTest(); + boolean posted = false; + try { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (System.nanoTime() < deadline) { + writer.enqueue(simpleEvent("busy-flag", "on")); + try { + verify(mockEvp, atLeastOnce()) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + posted = true; + break; + } catch (AssertionError ignored) { + // Keep the worker busy until the deadline. + } + } + } finally { + writer.close(); + } + + assertTrue(posted); + } + + @Test + void flushPostsToFlagevaluationEndpoint() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.add(event("flag-f", "on", "alloc1", "user-1", 1000L, emptyMap())); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + + verify(setup.factory).createBackendApi(Intake.EVENT_PLATFORM, false); + verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + } + + @Test + void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { + final int limit = 1_100; + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp, limit); + final AtomicInteger posts = new AtomicInteger(); + doAnswer( + invocation -> { + if (posts.incrementAndGet() == 2) { + throw new IOException("boom"); + } + return null; + }) + .when(mockEvp) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + + for (int i = 0; i < 4; i++) { + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 180)); + setup.handler.add(event("split-failure-" + i, "on", "alloc1", "user-" + i, 1000L, attrs)); + } + + setup.handler.drainAndAggregate(); + setup.handler.flush(); + assertEquals(2, posts.get()); + setup.handler.flush(); + assertEquals(2, posts.get()); + } + + private static final String HASHED_JANE_DOE = + "sha256_b4698f9b6d186781fa8dc59e533578fa2d8379a46b1cf6db85cda6aa9c99e51b"; + + @Test + void observeFullEvaluationDataTrueEmitsRawTargetingKeyAndContext() throws Exception { + // Consent travels on the event (snapshotted by the hook at evaluation time); the writer honours + // it verbatim and never consults the gateway. + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.add(piiEvent(true)); + + final Map json = flushAndCapture(setup).parsed; + + final Map ev = eventForFlag(json, "pii-flag"); + assertNotNull(ev); + assertEquals("jane.doe@datadoghq.com", ev.get("targeting_key")); + final Map ctx = (Map) ev.get("context"); + assertNotNull(ctx); + final Map evalAttrs = (Map) ctx.get("evaluation"); + assertNotNull(evalAttrs); + assertEquals("us-east-1", evalAttrs.get("region")); + } + + @Test + void observeFullEvaluationDataFalseHashesTargetingKeyAndOmitsContext() throws Exception { + assertHashedTargetingKeyAndOmittedContext(piiEvent(false)); + } + + @Test + void flagEvalEventDefaultConsentHashesTargetingKeyAndOmitsContext() throws Exception { + // An event built without an explicit consent value defaults to the privacy-preserving false, so + // it must behave exactly like the explicit "false" case. This is the state the hook produces + // when no UFC has been dispatched (the gateway reports false). + assertHashedTargetingKeyAndOmittedContext(piiEventDefaultConsent()); + } + + @Test + void eventConsentFalseStaysHashedEvenWhenGatewayLaterReportsTrue() throws Exception { + // Regression guard: consent is decided by the value the event carried at evaluation time, never + // re-read from the gateway at flush. An event evaluated under consent=false must stay hashed + // even if a later RC update turns the gateway's consent on before the flush drains. + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.add(piiEvent(false)); + + // Flip the gateway's consent on before both aggregation and flush; the event's evaluation-time + // snapshot (false) must win at every downstream step, so neither may consult the gateway. + dispatchObserveFullEvaluationData(true); + setup.handler.drainAndAggregate(); + + final java.util.List captured = new java.util.ArrayList<>(); + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured.add(inv.getArgument(1)); + return null; + }); + setup.handler.flush(); + + assertEquals(1, captured.size()); + final FlagEvaluationTestSupport.CapturedJson json = + FlagEvaluationTestSupport.readJson(captured.get(0)); + final Map ev = eventForFlag(json.parsed, "pii-flag"); + assertNotNull(ev); + assertEquals(HASHED_JANE_DOE, ev.get("targeting_key")); + assertFalse(ev.containsKey("context")); + assertTrue(json.raw.contains(HASHED_JANE_DOE)); + assertFalse(json.raw.contains("jane.doe@datadoghq.com")); + } + + @Test + void eventConsentTrueStaysRawEvenWhenGatewayLaterReportsFalse() throws Exception { + // Symmetric guard: an event evaluated under consent=true must stay raw even if a later RC + // update + // turns the gateway's consent off before aggregation and flush. Together with the false-stays- + // hashed test this pins that neither aggregation nor flush ever consults the gateway. + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.add(piiEvent(true)); + + dispatchObserveFullEvaluationData(false); + setup.handler.drainAndAggregate(); + + final java.util.List captured = new java.util.ArrayList<>(); + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured.add(inv.getArgument(1)); + return null; + }); + setup.handler.flush(); + + assertEquals(1, captured.size()); + final Map ev = + eventForFlag(FlagEvaluationTestSupport.readJson(captured.get(0)).parsed, "pii-flag"); + assertNotNull(ev); + assertEquals("jane.doe@datadoghq.com", ev.get("targeting_key")); + final Map ctx = (Map) ev.get("context"); + assertNotNull(ctx); + assertNotNull(ctx.get("evaluation")); + } + + @Test + void consentOffPreservesErrorCodeSignalAndNeverLeaksPiiInErrorMessage() throws Exception { + // Upstream contract: the hook substitutes the ErrorCode name for the raw exception message + // under consent-off (see + // FlagEvalLoggingHookTest#errorMessageReplacedByErrorCodeUnderConsentOff). + // This wire-level guard pins that a properly-formed consent-off event (a) still surfaces the + // stable ErrorCode signal for operators and (b) never lets a PII-shaped string escape onto the + // wire. Mirrors the existing PII guards on the targeting_key axis. + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.add( + new FlagEvalEvent( + "err-flag", + null, + "alloc1", + "jane.doe@datadoghq.com", + "TYPE_MISMATCH", + 1000L, + false, + emptyMap())); + + final FlagEvaluationTestSupport.CapturedJson captured = flushAndCapture(setup); + + final Map ev = eventForFlag(captured.parsed, "err-flag"); + assertNotNull(ev); + final Map error = (Map) ev.get("error"); + assertNotNull(error, "error object must be present so operators keep the ErrorCode signal"); + assertEquals("TYPE_MISMATCH", error.get("message")); + assertEquals(HASHED_JANE_DOE, ev.get("targeting_key")); + assertFalse(captured.raw.contains("jane.doe@datadoghq.com")); + assertFalse( + captured.raw.contains("For input string"), + "no exception-message-shaped text may reach the wire under consent-off"); + } + + @Test + void encodeFailureClearsAggregatorSoLaterFlushesRecover() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + // Moshi rejects non-finite JSON numbers. A NaN in the context poisons buildPayloads for this + // bucket. Before the fix, the aggregator kept the bucket and every later flush re-threw. + final Map poison = new HashMap<>(); + poison.put("bad-number", Double.NaN); + setup.handler.add(event("poison-flag", "on", "alloc1", "user-1", 1000L, true, poison)); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + verify(mockEvp, org.mockito.Mockito.never()) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + + // The bucket must not survive the failed flush. A follow-up healthy event flushes cleanly. + setup.handler.add(simpleEvent("healthy-flag", "on")); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + } + + @Test + void scoConstructorCreatesUsableWriter() { + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(new SharedCommunicationObjects(), cfg()); + writer.enqueue(simpleEvent("sco-flag", "on")); + assertNotNull(writer.pollQueuedEventForTest()); + writer.close(); + } + + @Test + void countContextTruncatedAccumulatesPerReason() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.countContextTruncated("field_count"); + writer.countContextTruncated("field_count"); + writer.countContextTruncated("field_length"); + writer.flushForTest(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 2, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_CONTEXT_TRUNCATED_METRIC, + "reason:field_count")); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_CONTEXT_TRUNCATED_METRIC, + "reason:field_length")); + } + + @Test + void hasCapacityForEnqueueReflectsQueueSaturationAndCountsPreQueueOverflow() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); + final int capacity = 2; + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + capacity, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + assertTrue(writer.hasCapacityForEnqueue()); + + // Saturate the hand-off queue without starting the worker so no drain can free slots. + for (int i = 0; i < capacity; i++) { + writer.enqueue(simpleEvent("cap-flag-" + i, "on")); + } + assertFalse(writer.hasCapacityForEnqueue()); + + // Simulate a pre-queue overflow account by the hook and surface it on flush. + writer.countPreQueueOverflow(); + writer.flushForTest(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_QUEUE_OVERFLOW)); + + writer.close(); + } + + private void assertHashedTargetingKeyAndOmittedContext(final FlagEvalEvent piiEvent) + throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.add(piiEvent); + + final FlagEvaluationTestSupport.CapturedJson captured = flushAndCapture(setup); + + final Map ev = eventForFlag(captured.parsed, "pii-flag"); + assertNotNull(ev); + assertEquals(HASHED_JANE_DOE, ev.get("targeting_key")); + assertFalse(ev.containsKey("context")); + // The raw wire bytes must carry the hashed key and never leak the raw PII value or a per-event + // evaluation context (the batch envelope owns the top-level "context" key, so guard on the + // nested "evaluation" field instead). + assertTrue(captured.raw.contains(HASHED_JANE_DOE)); + assertFalse(captured.raw.contains("jane.doe@datadoghq.com")); + assertFalse(captured.raw.contains("\"evaluation\":")); + } + + private static FlagEvalEvent piiEvent(final boolean observeFullEvaluationData) { + return event( + "pii-flag", + "on", + "alloc1", + "jane.doe@datadoghq.com", + 1000L, + observeFullEvaluationData, + piiAttrs()); + } + + private static FlagEvalEvent piiEventDefaultConsent() { + return event("pii-flag", "on", "alloc1", "jane.doe@datadoghq.com", 1000L, piiAttrs()); + } + + private static Map piiAttrs() { + final Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + return attrs; + } + + private static void dispatchObserveFullEvaluationData(final boolean value) { + FeatureFlaggingGateway.dispatch( + new ServerConfiguration( + "2024-04-17T19:40:53.716Z", "SERVER", value, null, java.util.Collections.emptyMap())); + } + + private static Object lifecycleLock(final FlagEvaluationWriterImpl writer) throws Exception { + final Field field = FlagEvaluationWriterImpl.class.getDeclaredField("lifecycleLock"); + field.setAccessible(true); + return field.get(writer); + } + + private static void awaitThreadState(final Thread thread, final Thread.State state) + throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != state && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertEquals(state, thread.getState()); + } + + private static Map context() { + final Map context = new HashMap<>(); + context.put("service", "test-service"); + return context; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java index a31d47889ba..104adef4cbe 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java @@ -2,6 +2,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -10,6 +11,8 @@ import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.io.IOException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class JsonApiUfcResponseParserTest { @@ -73,10 +76,101 @@ void rejectsTrailingJson() { + "}}{}")); } + @Test + void observeFullEvaluationDataDefaultsToFalseWhenAbsent() throws Exception { + // Absent → Moshi leaves the boxed field null; the read site's Boolean.TRUE.equals(...) then + // resolves to the privacy-preserving default (consent-off). Either null-or-false is the + // documented invariant; assert the field never reads as true. + final ServerConfiguration configuration = parse(wrap(emptyConfig())); + assertNotNull(configuration); + assertFalse(Boolean.TRUE.equals(configuration.observeFullEvaluationData)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void observeFullEvaluationDataParsesExplicitValue(final boolean value) throws Exception { + final ServerConfiguration configuration = + parse(wrap(configWithObserveFullEvaluationData(value))); + assertNotNull(configuration); + assertEquals(value, configuration.observeFullEvaluationData); + } + + @Test + void observeFullEvaluationDataExplicitNullDefaultsToFalseWithoutRejectingConfig() + throws Exception { + // An explicit null (or a wrong-typed value) for this field must not abort the whole UFC parse. + // A pod that starts after a malformed UFC has no last-known-good, so aborting would strand + // every flag on PROVIDER_NOT_READY (its default value). We fail closed on privacy (consent + // stays false) but preserve availability: flags parse and evaluate. + final ServerConfiguration configuration = + parse(wrap(configWithRawObserveFullEvaluationData("null"))); + + assertNotNull(configuration); + assertFalse( + configuration.observeFullEvaluationData != null && configuration.observeFullEvaluationData); + assertNotNull(configuration.flags); + } + + @Test + void observeFullEvaluationDataWrongTypedStringDefaultsToFalse() throws Exception { + // Moshi tolerates a stringified boolean like "true" via nullSafe/boxed handling: it either + // parses as null or throws locally and leaves the field null. Either way, downstream reads + // via Boolean.TRUE.equals(...) treat it as consent-off. The rest of the config must parse. + final ServerConfiguration configuration = + parse(wrap(configWithRawObserveFullEvaluationData("\"true\""))); + + assertNotNull(configuration); + assertFalse( + configuration.observeFullEvaluationData != null && configuration.observeFullEvaluationData); + assertNotNull(configuration.flags); + } + + @Test + void observeFullEvaluationDataWrongTypedNumberDefaultsToFalse() throws Exception { + final ServerConfiguration configuration = + parse(wrap(configWithRawObserveFullEvaluationData("1"))); + + assertNotNull(configuration); + assertFalse( + configuration.observeFullEvaluationData != null && configuration.observeFullEvaluationData); + assertNotNull(configuration.flags); + } + private static ServerConfiguration parse(final String json) throws Exception { return JsonApiUfcResponseParser.INSTANCE.parse(json.getBytes(UTF_8)); } + private static String wrap(final String attributes) { + return "{\"data\":{\"type\":\"universal-flag-configuration\",\"attributes\":" + + attributes + + "}}"; + } + + private static String configWithObserveFullEvaluationData(final boolean value) { + return "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"observeFullEvaluationData\":" + + value + + "," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{}" + + "}"; + } + + private static String configWithRawObserveFullEvaluationData(final String rawJsonValue) { + // Emit the field with a caller-controlled raw JSON value (null / "true" / 1 / ...) so we can + // assert the parser's tolerance of malformed shapes without going through configWith's + // boolean-typed helper. + return "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"observeFullEvaluationData\":" + + rawJsonValue + + "," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{}" + + "}"; + } + private static String emptyConfig() { return "{" + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java index 6d14a28f796..c1da0a11dcc 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java @@ -242,6 +242,35 @@ void flagMapAdapterFactoryOnlyCreatesFlagMapAdapterForFlagMapType() { flagsType, singleton(mock(Annotation.class)), moshi)); } + @Test + void lenientBooleanAdapterFactoryOnlyCreatesAdapterForUnannotatedBoxedBoolean() { + final Moshi moshi = moshi(); + + final JsonAdapter adapter = + UniversalFlagConfigParser.LenientBooleanAdapter.FACTORY.create( + Boolean.class, emptySet(), moshi); + + assertNotNull(adapter); + assertTrue(adapter instanceof UniversalFlagConfigParser.LenientBooleanAdapter); + // Primitive boolean keeps Moshi's strict adapter so mandatory fields still reject bad values. + assertNull( + UniversalFlagConfigParser.LenientBooleanAdapter.FACTORY.create( + boolean.class, emptySet(), moshi)); + // A qualified Boolean belongs to whichever adapter declared the qualifier, not to this one. + assertNull( + UniversalFlagConfigParser.LenientBooleanAdapter.FACTORY.create( + Boolean.class, singleton(mock(Annotation.class)), moshi)); + } + + @Test + void lenientBooleanAdapterIsReadOnly() { + final UniversalFlagConfigParser.LenientBooleanAdapter adapter = + new UniversalFlagConfigParser.LenientBooleanAdapter(); + + assertThrows( + UnsupportedOperationException.class, () -> adapter.toJson(mock(JsonWriter.class), true)); + } + @Test void allowsNullFlagMap() throws Exception { final ServerConfiguration config = diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ULeb128EncoderTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ULeb128EncoderTest.java new file mode 100644 index 00000000000..f47fdeaef1c --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ULeb128EncoderTest.java @@ -0,0 +1,39 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ULeb128EncoderTest { + + @Test + void hashTargetingKeyMatchesCanonicalPiiVector() { + // Canonical vector shared across all SDK implementations — see system-tests PR #7316. + assertEquals( + "b4698f9b6d186781fa8dc59e533578fa2d8379a46b1cf6db85cda6aa9c99e51b", + ULeb128Encoder.hashTargetingKey("jane.doe@datadoghq.com")); + } + + @Test + void hashTargetingKeyPreservesWhitespaceExactly() { + assertNotEquals( + ULeb128Encoder.hashTargetingKey("jane.doe@datadoghq.com"), + ULeb128Encoder.hashTargetingKey(" jane.doe@datadoghq.com ")); + } + + @Test + void hashTargetingKeyPreservesCaseExactly() { + assertNotEquals( + ULeb128Encoder.hashTargetingKey("jane.doe@datadoghq.com"), + ULeb128Encoder.hashTargetingKey("JANE.DOE@DATADOGHQ.COM")); + } + + @Test + void hashTargetingKeyIsLowercase64CharHex() { + final String hash = ULeb128Encoder.hashTargetingKey("some-arbitrary-key"); + assertEquals(64, hash.length()); + assertTrue(hash.matches("[0-9a-f]{64}")); + } +}